diff --git a/.github/workflows/helm-e2e.yaml b/.github/workflows/helm-e2e.yaml index 5af10fa1f9..c7de0b1280 100644 --- a/.github/workflows/helm-e2e.yaml +++ b/.github/workflows/helm-e2e.yaml @@ -65,7 +65,7 @@ jobs: # the chart composes {registry}/{repository}/{component}, so the local # registry serves each image where the default repository expects it -- # the same path-preserving rule a production mirror follows. - for component in ateapi atecontroller atelet podcertcontroller atenet objectstore-plugin credential-provider/kubernetes-secrets; do + for component in ateapi ate-idp-server atecontroller atelet podcertcontroller atenet objectstore-plugin credential-provider/kubernetes-secrets; do KO_DOCKER_REPO="localhost:5001/kagent-dev/substrate/${component##*/}" \ ./hack/run-tool.sh ko build --bare --tags helm-e2e \ --platform linux/amd64 "./cmd/${component}" diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index da91d9e414..c2e9871f00 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -58,24 +58,46 @@ jobs: # we only test on amd64, make sure everything still builds for arm64 - name: build for arm64 run: GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build ./... - # gotestsum wraps go test to emit JUnit; the go test arguments are unchanged. + # gotestsum wraps go test to emit JUnit and the go test -json stream. + # the log only gets a line per package and failed test output, the full + # output is in the uploaded json + # NOTE: no -v with gotestsum, it replaces the -test.v=test2json that -json + # sets, which marks t.Error output for the failure report below - name: unit tests - run: hack/run-tool.sh gotestsum --junitfile "${ARTIFACTS}/unit.xml" - --jsonfile "${ARTIFACTS}/unit.json" --format standard-verbose - -- -race -v ./... + run: | + go -C tools/junittool run . register "${ARTIFACTS}/unit.xml" + hack/run-tool.sh gotestsum --junitfile "${ARTIFACTS}/unit.xml" \ + --jsonfile "${ARTIFACTS}/unit.json" \ + --format pkgname-and-test-fails --format-hide-empty-pkg \ + -- -race ./... # tools/apitool has its own module so we need to run it explicitly. # run-tool.sh resolves the repo root itself, so it works from the subdir. - name: apitool tests working-directory: tools/apitool - run: ../../hack/run-tool.sh gotestsum --junitfile "${ARTIFACTS}/apitool.xml" - --jsonfile "${ARTIFACTS}/apitool.json" --format standard-verbose - -- -race -v ./... + run: | + go -C ../junittool run . register "${ARTIFACTS}/apitool.xml" + ../../hack/run-tool.sh gotestsum --junitfile "${ARTIFACTS}/apitool.xml" \ + --jsonfile "${ARTIFACTS}/apitool.json" \ + --format pkgname-and-test-fails --format-hide-empty-pkg \ + -- -race ./... + # tools/junittool has its own module too + - name: junittool tests + working-directory: tools/junittool + run: | + go run . register "${ARTIFACTS}/junittool.xml" + ../../hack/run-tool.sh gotestsum --junitfile "${ARTIFACTS}/junittool.xml" \ + --jsonfile "${ARTIFACTS}/junittool.json" \ + --format pkgname-and-test-fails --format-hide-empty-pkg \ + -- -race ./... # Kept apart from the root module so it can move to its own repository. - name: gcp-secret-manager credential provider tests working-directory: internal/plugins/gcp-secret-manager - run: ../../../hack/run-tool.sh gotestsum --junitfile "${ARTIFACTS}/gcp-secret-manager.xml" - --jsonfile "${ARTIFACTS}/gcp-secret-manager.json" --format standard-verbose - -- -race -v ./... + run: | + go -C ../../../tools/junittool run . register "${ARTIFACTS}/gcp-secret-manager.xml" + ../../../hack/run-tool.sh gotestsum --junitfile "${ARTIFACTS}/gcp-secret-manager.xml" \ + --jsonfile "${ARTIFACTS}/gcp-secret-manager.json" \ + --format pkgname-and-test-fails --format-hide-empty-pkg \ + -- -race ./... # The root verifiers stop at the module's go.mod, so it gets its own run: # gofmt, go vet and the tidy check, then golangci-lint with the root config. - name: gcp-secret-manager credential provider verify @@ -104,12 +126,16 @@ jobs: - name: root-gated tests env: ROOT_JUNIT_FILE: ${{ github.workspace }}/_artifacts/root.xml - run: hack/run-root-tests.sh -race -v + run: hack/run-root-tests.sh -race # An exit code cannot distinguish "all passed" from "none ran". A registered # file that was never written fails too. - name: Require tests to have run if: always() run: go -C tools/junittool run . verify -manifest "${ARTIFACTS}/expected-junit.txt" + # list each failed test and its error in the job summary and as annotations + - name: Summarize test failures + if: always() + run: go -C tools/junittool run . report "${ARTIFACTS}" - name: Upload test results if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 @@ -253,7 +279,7 @@ jobs: env: E2E_PARALLELISM: 8 E2E_JUNIT_FILE: ${{ github.workspace }}/_artifacts/e2e-gvisor${{ env.LANE_SUFFIX }}.xml - run: hack/run-e2e-kind.sh -v -args --no-color --preview="${ATE_PREVIEW}" + run: hack/run-e2e-kind.sh -args --no-color --preview="${ATE_PREVIEW}" - name: Focus the node on micro-VM run: hack/e2e-focus-sandbox-class.sh microvm - name: Run E2E tests (micro-VM) @@ -263,7 +289,7 @@ jobs: E2E_SANDBOX_CLASS: microvm E2E_PARALLELISM: 8 E2E_JUNIT_FILE: ${{ github.workspace }}/_artifacts/e2e-microvm${{ env.LANE_SUFFIX }}.xml - run: hack/run-e2e-kind.sh -v -args --no-color --preview="${ATE_PREVIEW}" + run: hack/run-e2e-kind.sh -args --no-color --preview="${ATE_PREVIEW}" # Every lane must have reported tests; a `-run` filter that stops matching # otherwise leaves its lane green and empty. The set comes from the manifest # each lane registers itself in, so adding a lane needs no change here, and @@ -271,6 +297,9 @@ jobs: - name: Require every lane to have run tests if: always() run: go -C tools/junittool run . verify -manifest "${ARTIFACTS}/expected-junit.txt" + - name: Summarize test failures + if: always() + run: go -C tools/junittool run . report "${ARTIFACTS}" - name: Save Go build cache # Main only, so PRs restore the latest main build rather than each # storing its own copy. diff --git a/AGENTS.md b/AGENTS.md index 59a2e4a269..e9ce36462f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -102,7 +102,7 @@ The following is what Substrate currently offers. Keep this up to date when updating AGENTS.md. - **Workload Isolation**: The project uses `gVisor` (`runsc`) for sandboxing and security isolation of workloads on pods. -- **JWTs and JWKs**: Signing (`internal/localjwtauthority`), verification (`internal/oidcjwt`), and JWK set publishing (`internal/oidcdiscovery`) are built on the standard library's crypto packages. Extend those packages rather than adding go-jose or another JOSE library, so the code that decides whether a token is valid stays small and supports only the algorithms Substrate uses. +- **JWTs and JWKs**: Signing and JWK set encoding (`internal/localjwtauthority`) and verification (`internal/oidcjwt`) are built on the standard library's crypto packages. Extend those packages rather than adding go-jose or another JOSE library, so the code that decides whether a token is valid stays small and supports only the algorithms Substrate uses. - **Ingress Authentication**: `atenet-router --ingress-auth-mode=static-mtls` requires clients to present a SPIFFE certificate signed by `--ingress-client-ca-file` whose ID is in `--ingress-allowed-spiffe-ids`; that list is the whole authorization policy, so any listed client may reach any Actor. The default, `deprecated-insecure`, checks a certificate only when a client presents one, and also serves plaintext listeners that accept any client. Install with it via `ate-setup --ingress-auth-mode=static-mtls` (`manifests/ate-install/components/router-static-mtls`); the allowlist lives in `manifests/ate-install/atenet-router.yaml`. - **Control API Authorization**: `ate-api-server` checks every Control RPC against an embedded OpenFGA model (`cmd/ateapi/internal/authz/model.fga`), on by default. Every served unary RPC needs an entry in `defaultRPCPermissions` (`registry.go`): with enforcement on, an RPC without one is denied, and a test fails the build. Access is granted per global scope and atespace through AccessPolicy (`kubectl ate access-policy`); the install's bootstrap owner is the shared `ate-client` ServiceAccount that `kubectl ate` uses. See `docs/authorization.md` for roles and known gaps. diff --git a/README.md b/README.md index ccf43e918d..6d8469001e 100644 --- a/README.md +++ b/README.md @@ -268,6 +268,7 @@ We provide several sample applications demonstrating Agent Substrate's capabilit * `cmd/ateapi`: The core control plane API server exposing gRPC endpoints to manage actor and worker lifecycles. * `cmd/atelet`: A node-level DaemonSet that supervises physical worker pods, coordinates snapshotting, and manages state transfers. * `cmd/atecontroller`: A Kubernetes controller that reconciles WorkerPool custom resources. +* `cmd/ate-idp-server`: Serves the OpenID discovery document and key set that relying parties use to verify actor JWTs. * `cmd/atenet`: A combined networking controller providing Envoy routing and proxy sidecars. * `cmd/ateworker-gvisor`: An interior-pod helper running inside sandboxed worker pods to execute `runsc` checkpoint and restore commands. * `cmd/ateworker-microvm`: The micro-VM peer of `ateworker-gvisor`, running actors as cloud-hypervisor VMs. diff --git a/benchmarking/README.md b/benchmarking/README.md index d64187df9c..ae94e73bc4 100644 --- a/benchmarking/README.md +++ b/benchmarking/README.md @@ -118,13 +118,14 @@ Four flags control the optional post-run measurements described in [Benchmark output files](#benchmark-output-files): * `--cluster-facts` / `--no-cluster-facts`: read node capacity and worker pod - count from the Kubernetes API once the run ends, to derive density frontiers. + count from the Kubernetes API once the run ends, to derive density frontiers, + and worker pod restarts and evictions around the run, for `oom_events`. On by default. Pass `--no-cluster-facts` to skip Kubernetes API discovery. * `--prometheus-url`: the Prometheus to harvest server-side telemetry from. Defaults to the in-cluster service installed by [Optional: Prometheus + Grafana](#optional-prometheus--grafana). * `--atelet-lag-s`: how long to wait after the run before reading the - atelet's snapshot and active actor metrics. Defaults to 70. + atelet's metrics. Defaults to 70. * `--phase-breakdown` / `--no-phase-breakdown`: read the atelet and worker pod logs once the run ends and add the suspend/resume phase percentiles to `stats.jsonl`. On by default; needs `pods/log` access in `ate-system` and @@ -240,6 +241,35 @@ When the run ends, shutdown cancels cycles still in flight. Calls cut short this nothing, and the interrupted task records no `TaskCEL` or `TaskWallClock` row, so the end of a run does not show up as a burst of failures. +#### Sweperf Log Lines + +The boomer log also carries per-task facts for post-processing such as the goodput script in +`tools/goodput`, which decides from them whether a task was good. The benchmark judges +nothing. Durations are in milliseconds. + +* `Completed sweperf cycle`: `actor`, `cycle`, `cel_ms` (0 when the execute failed), + `wall_ms` (resume start to cycle end), `ok` (the execute succeeded, and so did the suspend + if there was one), `resume_ms` (as on the `ResumeActor` row; omitted for a no-op resume) + and `ttfe_ms` (resume start to the asynchronous ack; omitted without one, and for a no-op + resume even when its execute got one). +* `ResumeActor failed in step`: also carries `wall_ms`. The cycle is retried on the next + iteration, and the failed attempt's time counts toward the task's `wall_ms`. +* `sweperf task record`, one per finished task: `v` (1), `user` (boomer goroutine ID, unique + per boomer process), `actor`, `template`, `cycles` and `steps` (the cycle and trace step + counts the task ran, resolved when it started, so a mid-run change to + `--sweperf-num-cycles` or `--sweperf-total-steps` applies from the next task; `cycles` is + capped at `steps`), `task_start` (when the session started) and `end`, `startup_ms` + (`CreateActor` until the sandbox answers `/status`), `cel_ms` and `wall_ms` (as in `TaskCEL` + and `TaskWallClock`), `think_ms` (think time between the task's cycles), `resume_max_ms` + (slowest non-no-op resume), `ttfe_max_ms` (slowest `ttfe_ms`; omitted when no cycle after + the no-op resume got an asynchronous ack, so always for a one-cycle task), `retries` + (failed resumes), `failed`, and the first failure's `fail_step` (`resume`, `execute` or + `suspend`) and `fail_cycle`. +* `sweperf on_start failed`: also carries `user`, to tie a failed start to its user. +* `sweperf shutdown started`: the worker began tearing down. Startups and cycles in flight + are canceled: interrupted tasks log no task record, and startup failures logged after it + belong to the teardown, not the run. + ### Agent-Session Benchmark The agent-session benchmark (`--user-class agentsession`) emulates a fleet of @@ -312,8 +342,12 @@ deployment fails loudly instead of showing up as OOM-flaky steps. gap gets ±20% jitter so sessions don't move in lockstep. * `--resume-mode implicit|explicit` — implicit (default) lets the parked first request wake the actor; explicit issues ResumeActor before traffic. -* `--lifecycle-mode suspend|pause` — durable suspend (default) or node-local - pause between steps. +* `--lifecycle-mode suspend|pause|none` — durable suspend (default), + node-local pause between steps, or `none` to keep actors resident in memory + between steps without hibernating. +* `--agentsession-snapshot-steps` — comma-separated 1-based step numbers after + which the actor is hibernated (for example, `3,10,20,30,40,50,61`); between + all other steps the actor stays resident. #### Writing an agent-session script @@ -493,13 +527,14 @@ warm-up until they have all joined at the spawn rate. * `ResumeActorCold`: Latency of each measured resume. A resume that finds its actor already running fails, as no cold resume happened. -* `PauseActor`: Latency of the pause that makes the actor cold again. -* `PauseActorRecovery`: After a failed cycle, the pause that makes the actor - cold for the next one. It is load on the server, so it is recorded, but - apart from `PauseActor`. A recovery pause that finds the actor already - paused counts as a success. -* `CreateActor`, `ResumeActorWarmup`, `PauseActorWarmup`: A user's warm-up of - its actor, in the step that added the user. +* `SuspendActorAsync`: Latency of the async SuspendActor that makes the actor + cold again. +* `SuspendActorAsyncRecovery`: After a failed cycle, the async SuspendActor that + makes the actor cold for the next one. It is load on the server, so it is + recorded, but apart from `SuspendActorAsync`. A recovery suspend that finds + the actor already suspended counts as a success. +* `CreateActor`, `ResumeActorWarmup`, `SuspendActorAsyncWarmup`: A user's + warm-up of its actor, in the step that added the user. ### Isolate Mode @@ -696,6 +731,49 @@ actually did, independent of what the load generator reported. nodes: the collector keeps an exited atelet's last value for about 5 minutes, so a restart can briefly count twice, and an atelet scraped both directly and through a collector counts twice. +* `working_set.node`, `worker`, `atelet`: memory working set in GiB from + cAdvisor on the nodes hosting a worker pod, per worker pod (every actor on it + included), and per atelet on those nodes. Each has a percentile `summary` + (plus `max`) over the steady-state window, `count` of series seen in it, and + a `timeseries` every 10s over the whole run with the sum (`total_gb`) and + largest (`max_gb`). A spike shorter than 10s can be missed. +* `working_set.actor`, `per_actor`: the same shape for the actors' own working + set as the atelet reports it (`ate_actor_stats_memory_working_set_bytes`), + per atelet with every template summed, and the average per measured actor + (`ate_actor_stats_sampled_actors`) on each atelet; `count` is atelet + processes, keyed like `active_actors`. As `per_actor` is an average per + atelet, its `max` is the highest atelet average, not the largest actor, and + its `total_gb` has no meaning. A gVisor actor is read from its sandbox cgroup + on the host, a microVM actor from inside the guest. As with `active_actors`, + suspended actors fall out, but through the telemetry-meter a series that + stops being exported keeps its last value for up to 5m. Missing samples are + skipped, not counted as 0. The atelet samples every + `--actor-stats-poll-interval` (default 1m), so these move in steps of a + minute or more. `actor` matches one worker pod's actors only when the node + runs one worker pod. +* `oom_events.node`, `pod`: OOM kills over the run, from cAdvisor's + `container_oom_events_total`. `node` counts only global OOMs. `pod` counts + kills inside a worker pod (actor sandboxes and the worker container) and is + best effort: a non-zero value is real, but a kill whose cgroup is gone before + the scrape is missed. A node-allocatable OOM (`oom_memcg=/kubepods.slice`) + isn't global, so it isn't in `node`, and `pod` usually misses it; the kubelet + evictions that often come with it show in `evicted`. OOMs inside a microVM + guest are invisible and show up as task failures. Memory PSI (`node_psi` / + `pod_psi`) is the early warning. +* `oom_events.worker_restarts`, `worker_oomkilled_pods`, `evicted`, + `worker_pods_lost`: from the Kubernetes API, read before and after the run. + Restarts on a pod deleted during the run are lost. `worker_oomkilled_pods` + counts pods whose last restart was `OOMKilled`, not kills; a group kill of + the whole worker (`memory.oom.group=1`) shows here. `evicted` is kubelet + eviction under node pressure (usually memory), not a kernel OOM kill. It also + counts replacement pods the kubelet refused to start while the node was under + pressure, as they get the same `Evicted` reason. It comes from `Evicted` + events, which last about 1h, so a run longer than that can undercount. + `worker_pods_lost` is pods gone by the end for any reason (eviction, + scale-down, recycling), not an OOM count; a pod created and deleted within + the run isn't counted. A node OOM can also show as a worker `OOMKilled` + restart, so never add the fields. A field is `null` when its Kubernetes read + fails, and all are with `--no-cluster-facts`. Every distribution reports p50, p90, p95 and p99 over the steady-state window. The steady-state window runs from the first to the last Locust sample at 90% or @@ -708,14 +786,15 @@ every 10s. The atelet exports before Prometheus scrapes it, so the harvest waits `--atelet-lag-s` seconds (default 70, enough for the OTel SDK's 60s default -export and a 10s scrape) and reads the snapshot and `active_actors` windows -half that late, stamping `active_actors` points back by the same amount. The -window ends at the last full-load sample, so teardown suspends are left out. - -`metadata.start_ts` and `end_ts` bound the whole run, which the packing and -`active_actors` timeseries cover. `steady_start_ts` and `steady_end_ts` bound -the steady-state window. A flat subset of the numbers also goes into a -`server_summary` row in `stats.jsonl`. +export and a 10s scrape) and reads the snapshot, `active_actors` and actor +working set windows half that late, stamping `active_actors` and actor +working set points back by the same amount. The window ends at the last +full-load sample, so teardown suspends are left out. + +`metadata.start_ts` and `end_ts` bound the whole run, which the packing, +`active_actors` and `working_set` timeseries cover. `steady_start_ts` and +`steady_end_ts` bound the steady-state window. A flat subset of the numbers +also goes into a `server_summary` row in `stats.jsonl`. Neither the Kubernetes API nor Prometheus is required. If either is unreachable, or discovery was skipped, the affected fields are written as `null` and the run diff --git a/benchmarking/analysis/README.md b/benchmarking/analysis/README.md index 91076b3cfb..3109de03fb 100644 --- a/benchmarking/analysis/README.md +++ b/benchmarking/analysis/README.md @@ -72,18 +72,22 @@ A failed atelet operation still writes its record, marked with `error.type` ## Reading the report **Phase percentiles.** Per layer, operation, sandbox class, snapshot kind, -scope and phase: count, p50, p90, p95 and max. A `golden` restore downloads -the golden image and a `latest` one the actor's own, so they are separate -rows, as are gVisor and micro-VM operations. The atelet rows split a -checkpoint between `sandbox_assets`, `worker_checkpoint` and `persist`, and a +scope and phase: count, p50, p90, p95 and max. A `local` restore reads a +node-local snapshot and a `latest` one a durable snapshot from object +storage, so they are separate rows, as are gVisor and micro-VM operations. +A restore from a template's golden snapshot reports `latest`, like the +actor's own durable snapshot. The atelet rows split a +checkpoint between `sandbox_assets`, `worker_checkpoint`, `persist` and +`cleanup`, and a restore between `volume_mount`, `manifest_fetch`, `sandbox_assets`, `download`, `oci_unpack` and `worker_restore`. The ateworker rows split the `worker_*` phase further with the runtime's own phases: for micro-VM `prep` / `pause` / `snapshot` / `durable_dir` / `rootfs_upper` / `teardown` on a checkpoint and `prep` / … / `vm_restore` / `wakeup_probe` on a restore; -for gVisor `pause` / `checkpoint` / `durable_dir` / `resume` / `teardown` and -`prep` / `net_setup` / … / `pause_restore` / `app_restore` / `wakeup_probe` / -`activate` (the full lists are in `cmd/ateworker-*/phaselog.go`). The ateworker +for gVisor `pause` / `checkpoint` / `fs_checkpoint` / `resume` / `teardown` and +`prep` / `net_setup` / … / `pause_restore` / `app_restore` / +`wakeup_probe` / `activate` (the full lists are in +`cmd/ateworker-*/phaselog.go`). The ateworker records carry no sandbox class or kind of their own; a paired one takes both from its atelet record, so both layers split into the same rows. diff --git a/benchmarking/analysis/phase_report.py b/benchmarking/analysis/phase_report.py index 21d3860fdb..2d7c154f40 100755 --- a/benchmarking/analysis/phase_report.py +++ b/benchmarking/analysis/phase_report.py @@ -99,7 +99,7 @@ "tap", "vmm_launch", "vm_restore", "resume", "wakeup_probe", "activate", # atelet + ateworker checkpoint "pause", "checkpoint", "snapshot", "durable_dir", "rootfs_upper", - "worker_checkpoint", "teardown", "persist", + "fs_checkpoint", "worker_checkpoint", "teardown", "persist", "cleanup", UNATTRIBUTED, TOTAL, ] _PHASE_RANK = {name: i for i, name in enumerate(PHASE_ORDER)} @@ -113,9 +113,8 @@ CONCURRENT_CAPTURES = ("snapshot", "durable_dir", "rootfs_upper") # Slack on the window in which the ateworker record of an operation must land: -# atelet does a little work after the ateworker call returns before it writes its -# own record (unmounting, resetting the actor dirs), and the two clocks are -# the same node's but not the same goroutine's. +# atelet does a little untimed work between the ateworker call and its own +# record, and the two clocks are the same node's but not the same goroutine's. JOIN_SLACK_S = 1.0 @@ -177,7 +176,7 @@ def unattributed(source: str, op: str, sandbox_class: str, phases: dict[str, flo if total is None: return None if (source, op) == ("atelet", "checkpoint"): - spent = sum(phases.get(p, 0.0) for p in ("sandbox_assets", "worker_checkpoint", "persist")) + spent = sum(phases.get(p, 0.0) for p in ("sandbox_assets", "worker_checkpoint", "persist", "cleanup")) elif (source, op, sandbox_class) == ("ateworker", "checkpoint", MICROVM): spent = (phases.get("prep", 0.0) + phases.get("pause", 0.0) + max(phases.get(p, 0.0) for p in CONCURRENT_CAPTURES) @@ -295,9 +294,9 @@ def fmt_s(seconds: float) -> str: def report_phases(breakdowns: list[Breakdown], writer) -> list[dict]: """Per (source, op, sandbox class, kind, scope, phase) percentiles. A - golden restore (which downloads the golden image) and a latest restore, - or a gVisor and a micro-VM checkpoint, are different distributions and - must not be pooled. Returns the rows for CSV.""" + local restore and a latest one, or a gVisor and a micro-VM checkpoint, + are different distributions and must not be pooled. Returns the rows + for CSV.""" groups: dict[tuple, list[float]] = defaultdict(list) for b in breakdowns: if b.failed: @@ -335,14 +334,15 @@ def ateworker_window(op: Breakdown) -> tuple[float, float] | None: """When, relative to the atelet record's time, the ateworker record of the same operation was written. ateworker writes it as its RPC returns, i.e. at the end of the atelet worker_* phase: for a checkpoint that is before - persist runs, for a restore it is the last phase. An operation that never - reached the ateworker call has no window and nothing to pair with.""" + persist and cleanup run, for a restore it is the last phase. An operation + that never reached the ateworker call has no window and nothing to pair + with.""" total = op.phases.get(TOTAL) if op.ts is None or total is None or INNER_PHASE[op.op] not in op.phases: return None if op.op == "checkpoint": - return (op.ts - total - JOIN_SLACK_S, - op.ts - op.phases.get("persist", 0.0) + JOIN_SLACK_S) + after = op.phases.get("persist", 0.0) + op.phases.get("cleanup", 0.0) + return (op.ts - total - JOIN_SLACK_S, op.ts - after + JOIN_SLACK_S) return (op.ts - op.phases[INNER_PHASE[op.op]] - JOIN_SLACK_S, op.ts + JOIN_SLACK_S) diff --git a/benchmarking/analysis/test_phase_report.py b/benchmarking/analysis/test_phase_report.py index 386bbb4f72..40fde9b188 100644 --- a/benchmarking/analysis/test_phase_report.py +++ b/benchmarking/analysis/test_phase_report.py @@ -227,9 +227,10 @@ def test_prefixed_kubectl_lines_still_parse(self): class UnattributedTest(unittest.TestCase): def test_atelet_checkpoint_residual_is_total_minus_sequential_phases(self): - out = parse(ATELET_CHECKPOINT) + rec = dict(ATELET_CHECKPOINT, **{"ate.actor.checkpoint.duration.cleanup": 0.35}) + out = parse(rec) phase_report.join_layers(out.breakdowns) - self.assertAlmostEqual(out.breakdowns[0].phases["unattributed"], 6.0 - (0.01 + 1.14 + 4.48)) + self.assertAlmostEqual(out.breakdowns[0].phases["unattributed"], 6.0 - (0.01 + 1.14 + 4.48 + 0.35)) def test_microvm_ateworker_checkpoint_counts_the_concurrent_captures_once(self): # prep + pause + max(snapshot, rootfs_upper) + teardown; the two # captures overlapped, so only the slower one is spent wall time. @@ -290,14 +291,14 @@ def test_failed_records_are_flagged_and_excluded_from_percentiles(self): self.assertEqual(downloads[0]["count"], 1) # the failed 30s never entered def test_percentiles_split_by_sandbox_class_and_kind(self): - golden = dict(ATELET_RESTORE, **{"time": "2026-09-23T09:59:00.000000000Z", - "ate.snapshot.kind": "golden", "ate.sandbox.class": "gvisor", - "ate.actor.restore.duration.download": 9.0, - "ate.actor.restore.duration.total": 10.0}) + local = dict(ATELET_RESTORE, **{"time": "2026-09-23T09:59:00.000000000Z", + "ate.snapshot.kind": "local", "ate.sandbox.class": "gvisor", + "ate.actor.restore.duration.download": 9.0, + "ate.actor.restore.duration.total": 10.0}) latest = dict(ATELET_RESTORE, **{"ate.sandbox.class": "gvisor"}) - rows = phase_report.report_phases(parse(golden, latest).breakdowns, lambda _: None) + rows = phase_report.report_phases(parse(local, latest).breakdowns, lambda _: None) downloads = {(r["class"], r["kind"]): r["p50_ms"] for r in rows if r["phase"] == "download"} - self.assertEqual(downloads, {("gvisor", "golden"): 9000.0, ("gvisor", "latest"): 2400.0}) + self.assertEqual(downloads, {("gvisor", "local"): 9000.0, ("gvisor", "latest"): 2400.0}) def test_paired_ateworker_record_inherits_the_atelet_kind(self): out = parse(ATEWORKER_RESTORE, ATELET_RESTORE) @@ -317,6 +318,16 @@ def test_checkpoint_pairing_window_ends_where_persist_starts(self): atelet = next(b for b in out.breakdowns if b.source == "atelet") self.assertIs(pairs[id(atelet)], out.breakdowns[0]) + def test_checkpoint_pairing_window_excludes_cleanup(self): + # Cleanup runs after the ateworker record is written too: a record from + # inside the 0.35 s cleanup window belongs to a later cycle. + checkpoint = dict(ATELET_CHECKPOINT, **{"ate.actor.checkpoint.duration.cleanup": 0.35}) + during_cleanup = dict(ATEWORKER_CHECKPOINT, time="2026-09-23T10:00:56.400000000Z") + out = parse(ATEWORKER_CHECKPOINT, during_cleanup, checkpoint) + pairs = phase_report.pair_records(out.breakdowns) + atelet = next(b for b in out.breakdowns if b.source == "atelet") + self.assertIs(pairs[id(atelet)], out.breakdowns[0]) + def test_an_ateworker_record_pairs_with_at_most_one_operation(self): second = dict(ATELET_RESTORE, time="2026-09-23T10:00:05.900000000Z") out = parse(ATEWORKER_RESTORE, ATELET_RESTORE, second) diff --git a/benchmarking/automation/manifests/runner-job.yaml.tmpl b/benchmarking/automation/manifests/runner-job.yaml.tmpl index eb5e4acc5d..5303dcb2fb 100644 --- a/benchmarking/automation/manifests/runner-job.yaml.tmpl +++ b/benchmarking/automation/manifests/runner-job.yaml.tmpl @@ -81,6 +81,10 @@ rules: - apiGroups: [""] resources: ["pods/log"] verbs: ["get"] +# Worker pod events, for the evictions in oom_events (cluster_facts.py). +- apiGroups: [""] + resources: ["events"] + verbs: ["list"] --- # The atelet pod logs hold the other half of the phase breakdown. apiVersion: rbac.authorization.k8s.io/v1 diff --git a/benchmarking/automation/orchestrator.py b/benchmarking/automation/orchestrator.py index f6ed296e92..0940c4d79c 100644 --- a/benchmarking/automation/orchestrator.py +++ b/benchmarking/automation/orchestrator.py @@ -420,8 +420,8 @@ def teardown_substrate() -> None: def install_microvm_deps() -> None: """Stage kata/cloud-hypervisor assets and apply the cluster-wide - microvm SandboxConfig. Required before a microvm ActorTemplate can - boot; must run after deploy_substrate() (which installs the CRDs).""" + microvm SandboxConfig. Required before a microvm WorkerPool can run + actors; must run after deploy_substrate() (which installs the CRDs).""" run(["hack/install-microvm-deps.sh", "--install"]) diff --git a/benchmarking/locust/cluster_facts.py b/benchmarking/locust/cluster_facts.py index 8bb03d9a82..1f983565c5 100644 --- a/benchmarking/locust/cluster_facts.py +++ b/benchmarking/locust/cluster_facts.py @@ -19,7 +19,8 @@ API, then derives the actor-density frontiers (actors per node / vCPU / GB RAM and the actors-per-pod percentiles) for a completed trial. Also records the nodes' Kubernetes version, the worker pods' declared capacity and sandbox -class, the run window, and any caller-supplied --metadata. +class, the run window, and any caller-supplied --metadata, and reads worker +pod restarts and evictions around a run, for server telemetry's `oom_events`. """ import argparse @@ -389,3 +390,106 @@ def append_trial_summary( with open(jsonl_path, "a", encoding="utf-8") as f: f.write(json.dumps(summary_entry) + "\n") _log(logs, f"Appended trial_summary to {jsonl_path}") + + +def snapshot_worker_pods( + logs: TextIO | None = None, +) -> dict[str, dict[str, Any]] | None: + """Per worker pod uid: worker restarts, last exit OOMKilled, evicted. + + An evicted pod is usually gone by now; read_evicted_pods covers it. None if + the read failed. + """ + if not _load_kube_config(logs): + return None + try: + # Consistent read, so the snapshot right after the run is exact. + pods = client.CoreV1Api().list_namespaced_pod( + namespace=WORKER_POOL_NAMESPACE, + label_selector=WORKER_POOL_LABEL, + _request_timeout=API_TIMEOUT_SECONDS, + ).items + except Exception as e: + reason = getattr(e, "reason", e) + _log(logs, + f"Notice: could not list pods in {WORKER_POOL_NAMESPACE}: {reason}") + return None + out: dict[str, dict[str, Any]] = {} + for pod in pods: + # By name, so a sidecar can't skew it; a Pending pod may have none yet. + cs = next((c for c in pod.status.container_statuses or [] + if c.name == WORKER_CONTAINER), None) + last = cs.last_state.terminated if cs and cs.last_state else None + out[pod.metadata.uid] = { + "restarts": (cs.restart_count or 0) if cs else 0, + "last_oomkilled": bool(last and last.reason == "OOMKilled"), + "evicted": pod.status.reason == "Evicted", + } + return out + + +def read_evicted_pods( + since_ts: int, + logs: TextIO | None = None, +) -> set[str] | None: + """Uids of pods evicted at or after since_ts, from Evicted events. + + atecontroller deletes an evicted worker pod at once, so the pod is gone by + the next snapshot; its event stays for about 1h. Events can't be filtered by + label, so this covers every pod in the namespace, where only workers run. + None if the read failed. + """ + if not _load_kube_config(logs): + return None + try: + events = client.CoreV1Api().list_namespaced_event( + namespace=WORKER_POOL_NAMESPACE, + field_selector="reason=Evicted,involvedObject.kind=Pod", + _request_timeout=API_TIMEOUT_SECONDS, + ).items + except Exception as e: + reason = getattr(e, "reason", e) + _log(logs, + f"Notice: could not list events in {WORKER_POOL_NAMESPACE}: {reason}") + return None + uids: set[str] = set() + for ev in events: + # Kubelet sets last_timestamp; events.k8s.io writers set event_time. + t = ev.last_timestamp or ev.event_time or ev.metadata.creation_timestamp + if t and t.timestamp() >= since_ts and ev.involved_object.uid: + uids.add(ev.involved_object.uid) + return uids + + +def diff_worker_pods( + before: dict[str, dict[str, Any]] | None, + after: dict[str, dict[str, Any]] | None, + evicted_uids: set[str] | None, +) -> dict[str, int | None]: + """Worker restarts, OOMKilled pods, evictions and lost pods over the run. + + `worker_oomkilled_pods` counts pods, not kills: lastState keeps only the + last exit. A node OOM can also restart a worker. `evicted` is kubelet + eviction or admission rejection, not a kernel OOM kill: pods evicted in the + run, from events and from the snapshots, once per uid. `worker_pods_lost` + is pods gone by the second snapshot, for any reason. A field is None when + its source failed. + """ + restarts: int | None = None + oomkilled: int | None = None + lost: int | None = None + status_evicted: set[str] = set() + if before is not None and after is not None: + grown = oom = 0 + for uid, now in after.items(): + was = before.get(uid, {"restarts": 0, "evicted": False}) + grew = max(now["restarts"] - was["restarts"], 0) + grown += grew + oom += bool(grew and now["last_oomkilled"]) + if now["evicted"] and not was["evicted"]: + status_evicted.add(uid) + restarts, oomkilled = grown, oom + lost = len(before.keys() - after.keys()) + evicted = None if evicted_uids is None else len(evicted_uids | status_evicted) + return {"worker_restarts": restarts, "worker_oomkilled_pods": oomkilled, + "evicted": evicted, "worker_pods_lost": lost} diff --git a/benchmarking/locust/common/agentsession_config.py b/benchmarking/locust/common/agentsession_config.py index 67bb42e1e3..a6ea37fe3e 100644 --- a/benchmarking/locust/common/agentsession_config.py +++ b/benchmarking/locust/common/agentsession_config.py @@ -56,3 +56,16 @@ def add_agentsession_arguments(parser: LocustArgumentParser) -> None: ), include_in_web_ui=True, ) + parser.add_argument( + "--agentsession-snapshot-steps", + type=str, + default="", + env_var="LOCUST_AGENTSESSION_SNAPSHOT_STEPS", + help=( + "Comma-separated 1-based step numbers after which the actor is " + "hibernated (per --lifecycle-mode). When empty (default), the " + "actor is hibernated after every step unless --lifecycle-mode is " + "'none'." + ), + include_in_web_ui=True, + ) diff --git a/benchmarking/locust/common/boomer_config.py b/benchmarking/locust/common/boomer_config.py index e78c30ea60..6f9cfeac92 100644 --- a/benchmarking/locust/common/boomer_config.py +++ b/benchmarking/locust/common/boomer_config.py @@ -36,7 +36,11 @@ * serve_config_headless(): the same /boomer-config payload from a plain HTTP server, for a headless run whose values change while it runs. -Keep _FLAGS aligned with internal/benchmarking/boomer/dynconfig.payload. +_FLAGS is the set of keys the payload carries. The Go side takes the payload +as an arbitrary JSON object, and the user class a worker runs reads and +validates the keys it documents (the dynconfig.Typed codec it registers with +its userclass.Entry), so a flag added here needs a matching field in the +class that consumes it. """ import argparse @@ -74,6 +78,7 @@ "--agentsession-script": str, "--agentsession-script-file": str, "--agentsession-think-scale": float, + "--agentsession-snapshot-steps": str, "--total-actors": int, "--spawn-concurrency": int, "--actor-deadline": float, diff --git a/benchmarking/locust/common/lifecycle_mode.py b/benchmarking/locust/common/lifecycle_mode.py index 9d15560b6b..644a50df40 100644 --- a/benchmarking/locust/common/lifecycle_mode.py +++ b/benchmarking/locust/common/lifecycle_mode.py @@ -25,7 +25,8 @@ def add_lifecycle_mode_arguments(parser: LocustArgumentParser) -> None: "--lifecycle-mode", type=str, default="suspend", - choices=["suspend", "pause"], + choices=["suspend", "pause", "none"], help="Lifecycle mode: 'suspend' uses SuspendActor (persists snapshot to storage); " - "'pause' uses PauseActor (keeps snapshot on local node) (default: suspend)", + "'pause' uses an asynchronous SuspendActor (keeps snapshot on local node); " + "'none' keeps actors resident in memory between steps (default: suspend)", ) diff --git a/benchmarking/locust/runner.py b/benchmarking/locust/runner.py index 5f94e78dc2..be7503321e 100644 --- a/benchmarking/locust/runner.py +++ b/benchmarking/locust/runner.py @@ -46,7 +46,10 @@ from cluster_facts import ( EMPTY_FACTS, append_trial_summary, + diff_worker_pods, get_cluster_hardware_facts, + read_evicted_pods, + snapshot_worker_pods, ) from common.boomer_config import build_config_json from server_telemetry import extract_and_record_server_telemetry @@ -154,7 +157,8 @@ def parse_args() -> argparse.Namespace: default=True, help=( "Read node capacity and worker pod count from the Kubernetes API " - "after the run to derive density frontiers. Pass " + "after the run to derive density frontiers, and worker pod restarts " + "and evictions around it for oom_events. Pass " "--no-cluster-facts to skip Kubernetes API discovery" ), ) @@ -517,6 +521,32 @@ def collect_cluster_facts( return get_cluster_hardware_facts(logs) +def read_worker_pods( + args: argparse.Namespace, logs: TextIO +) -> dict[str, dict[str, Any]] | None: + """Worker pod state, or None when discovery is off or the read fails.""" + if not args.cluster_facts: + return None + try: + return snapshot_worker_pods(logs) + except Exception as e: + tee(logs, f"Warning: Failed to read worker pods: {e}") + return None + + +def read_evictions( + args: argparse.Namespace, since_ts: int, logs: TextIO +) -> set[str] | None: + """Pods evicted since since_ts, or None when discovery is off or it fails.""" + if not args.cluster_facts: + return None + try: + return read_evicted_pods(since_ts, logs) + except Exception as e: + tee(logs, f"Warning: Failed to read evictions: {e}") + return None + + def main() -> None: args = parse_args() now = datetime.now(timezone.utc) @@ -547,8 +577,13 @@ def main() -> None: traces.write("\t".join(TRACE_COLUMNS) + "\n") traces.flush() log_run_config(args, prefix, work_dir, logs) + # Worker pods around the run, for OOMKilled restarts and evictions. + pods_before = read_worker_pods(args, logs) exit_code = run_test(args, csv_prefix, logs, traces) run_end_ts = int(datetime.now(timezone.utc).timestamp()) + pods_after = read_worker_pods(args, logs) + # Right after the run, as Evicted events expire after about 1h. + evicted_uids = read_evictions(args, run_ts, logs) stats_generated = False if stats_csv.exists(): @@ -618,6 +653,7 @@ def main() -> None: test_name=args.name, logs=logs, lag_s=args.atelet_lag_s, + worker_pods=diff_worker_pods(pods_before, pods_after, evicted_uids), ) except Exception as e: tee(logs, f"Warning: Failed to harvest server telemetry: {e}") diff --git a/benchmarking/locust/server_telemetry.py b/benchmarking/locust/server_telemetry.py index 5ecd93ad00..ba9d6a90ac 100644 --- a/benchmarking/locust/server_telemetry.py +++ b/benchmarking/locust/server_telemetry.py @@ -16,7 +16,7 @@ Queries Prometheus over [T_start, T_end] and the steady-state window inside it to capture cluster packing, node and pod PSI stalls, snapshot sizes, latencies -and throughput, and running actors. +and throughput, running actors, memory working set and OOM kills. """ import csv @@ -26,6 +26,7 @@ import time import urllib.parse import urllib.request +from collections.abc import Callable from pathlib import Path from typing import Any, TextIO @@ -562,6 +563,194 @@ def _count_active_actors( } +# Live worker pods: the rate drops a deleted pod's stale series in ~1 min, not 5. +WORKER_PODS = ( + "rate(container_cpu_usage_seconds_total" + '{namespace="benchmark-workloads", container="worker"}[1m]) > 0' +) + +# Only nodes/pods with a live worker; max, not sum: a restart briefly has 2 series. +WORKING_SET_QUERIES = { + "node": ( + 'max by (instance) (container_memory_working_set_bytes{container="node"})' + f" and on (instance) count by (instance) ({WORKER_PODS})" + ), + "worker": ( + "max by (pod) (container_memory_working_set_bytes" + '{namespace="benchmark-workloads", container="worker"})' + f" and on (pod) ({WORKER_PODS})" + ), + "atelet": ( + "max by (pod, instance) (container_memory_working_set_bytes" + '{namespace="ate-system", container="atelet"})' + f" and on (instance) count by (instance) ({WORKER_PODS})" + ), +} + +# Actors' working set and running count per atelet, all templates summed; keyed +# like packing and active_actors. Not node names, so no worker filter: the +# atelet stops exporting on a node with no running actor. +ACTOR_WORKING_SET = ( + "sum by (instance, exported_instance) (ate_actor_stats_memory_working_set_bytes)" +) +ACTOR_COUNT = "sum by (instance, exported_instance) (ate_actor_stats_sampled_actors)" +WORKING_SET_KEYS = (*WORKING_SET_QUERIES, "actor", "per_actor") + +BYTES_PER_GIB = 2**30 + + +def _label_key(metric: dict[str, str]) -> tuple: + """Every label, so each series keeps its own key.""" + return tuple(sorted(metric.items())) + + +def _read_series( + prom_url: str, query: str, start_ts: int, end_ts: int, shift: int = 0, + key: Callable[[dict[str, str]], Any] = _label_key, +) -> dict[Any, dict[int, float]]: + """Samples per `key` every 10s, read `shift` late and moved back by it. + + Series sharing a key describe the same thing, so they merge by max. + """ + res = query_prometheus_range( + prom_url, query, start_ts + shift, end_ts + shift, step="10s" + ) + out: dict[Any, dict[int, float]] = {} + for series in res: + new = _series_samples(series) + if not new: + continue + samples = out.setdefault(key(series.get("metric", {})), {}) + for t, val in new.items(): + samples[t - shift] = max(samples.get(t - shift, val), val) + return out + + +def _summarize_gb( + series: dict[Any, dict[int, float]], + steady_start_ts: int, + steady_end_ts: int, +) -> dict[str, Any]: + """Steady percentiles, series seen in steady, and a 10s sum/max timeseries.""" + by_ts: dict[int, list[float]] = {} + steady_gb: list[float] = [] + steady_series = 0 + for samples in series.values(): + in_steady = False + for t, val in samples.items(): + gb = val / BYTES_PER_GIB + by_ts.setdefault(t, []).append(gb) + if steady_start_ts <= t <= steady_end_ts: + steady_gb.append(gb) + in_steady = True + steady_series += in_steady + return { + "summary": compute_percentiles(steady_gb), + "count": steady_series if by_ts else None, + "timeseries": [ + { + "timestamp": t, + "total_gb": round(sum(v), 4), + "max_gb": round(max(v), 4), + } + for t, v in sorted(by_ts.items()) + ], + } + + +def _measure_working_set( + prom_url: str, + start_ts: int, + end_ts: int, + steady_start_ts: int, + steady_end_ts: int, + lag_s: int = 0, +) -> dict[str, Any]: + """Working set GiB per node, worker, atelet and actor; null, not 0, with no series. + + `worker` is the whole worker pod, every actor on it included. `actor` is the + atelet's sum over the actors it measured, per atelet; `per_actor` divides it + by their count. The atelet samples about once a minute and exports late, so + both are read `lag_s // 2` late and moved back onto the run's 10s grid. + """ + def summarize(series: dict[Any, dict[int, float]]) -> dict[str, Any]: + return _summarize_gb(series, steady_start_ts, steady_end_ts) + + out: dict[str, Any] = { + key: summarize(_read_series(prom_url, query, start_ts, end_ts)) + for key, query in WORKING_SET_QUERIES.items() + } + shift = lag_s // 2 + actor = _read_series( + prom_url, ACTOR_WORKING_SET, start_ts, end_ts, shift, key=_process_key + ) + count = _read_series( + prom_url, ACTOR_COUNT, start_ts, end_ts, shift, key=_process_key + ) + out["actor"] = summarize(actor) + # Same atelet and timestamp; a missing or zero count is skipped. + out["per_actor"] = summarize({ + key: {t: v / count[key][t] for t, v in samples.items() + if count.get(key, {}).get(t)} + for key, samples in actor.items() + }) + return out + + +# A global OOM logs no oom_memcg, so cAdvisor counts it on `/` (container="node"). +OOM_NODE = 'container="node"' +# Kills in a worker pod: nested actor sandbox cgroups, then the worker container's. +OOM_POD = ( + r'pod="", id=~".*/cri-containerd-[^/]+\\.scope/.+"', + 'namespace="benchmark-workloads", container="worker"', +) + + +def _oom_growth_query(selector: str, start_ts: int, window_s: int) -> str: + """OOMs in the window, clamped per series; `or vector(0)` so [] = failed read.""" + s = f"container_oom_events_total{{{selector}}}" + peak = f"max_over_time({s}[{window_s}s])" + return f"sum(clamp_min({peak} - ({s} @ {start_ts} or {peak} * 0), 0)) or vector(0)" + + +# From the Kubernetes API, read in runner.py. +WORKER_POD_FIELDS = ( + "worker_restarts", "worker_oomkilled_pods", "evicted", "worker_pods_lost", +) + + +def _count_oom_events(prom_url: str, start_ts: int, end_ts: int) -> dict[str, Any]: + """OOM kills over the run; a failed read is null, not 0. + + `node` counts global OOMs only. `pod` is best effort: a kill counts only if + its cgroup lasts until the scrape, so it usually misses a node-allocatable + OOM and a group kill of the whole worker, but a non-zero value is real. + """ + window_s = max(end_ts - start_ts, 1) + out: dict[str, Any] = { + "node": None, + "pod": None, + "window": {"start_ts": start_ts, "end_ts": end_ts}, + } + if not query_prometheus_instant( + prom_url, "count(container_oom_events_total)", time_ts=end_ts + ): + return out # Metric not scraped. + + def grown(selector: str) -> int | None: + v = _parse_instant_float(query_prometheus_instant( + prom_url, _oom_growth_query(selector, start_ts, window_s), + time_ts=end_ts, + )) + return None if v is None else int(round(v)) + + out["node"] = grown(OOM_NODE) + # Two reads, as `@` takes a selector, not (A or B); half a read is no read. + pod = [grown(sel) for sel in OOM_POD] + out["pod"] = None if None in pod else sum(pod) + return out + + def harvest_server_telemetry( prom_url: str, start_ts: int, @@ -572,9 +761,9 @@ def harvest_server_telemetry( ) -> dict[str, Any]: """Harvests the ground truth metric streams from Prometheus. - Only the atelet-exported snapshot and active actor blocks are read late, by - `lag_s // 2`; packing and PSI aren't shifted and are read over the run - window as is. + Only the atelet-exported snapshot, active actor and actor working set + blocks are read late, by `lag_s // 2`; packing, PSI, the cAdvisor working + set and OOM counts aren't shifted and are read over the run window as is. """ steady_end = end_ts if steady_end_ts is None else steady_end_ts return { @@ -593,6 +782,10 @@ def harvest_server_telemetry( "active_actors": _count_active_actors( prom_url, start_ts, end_ts, steady_start_ts, steady_end, lag_s ), + "working_set": _measure_working_set( + prom_url, start_ts, end_ts, steady_start_ts, steady_end, lag_s + ), + "oom_events": _count_oom_events(prom_url, start_ts, end_ts), } @@ -608,11 +801,13 @@ def extract_and_record_server_telemetry( test_name: str, logs: TextIO | None = None, lag_s: int = 70, + worker_pods: dict[str, int | None] | None = None, ) -> None: """Entry point called by runner.py to query Prometheus and persist artifacts. Waits until `end_ts + lag_s` first, so the atelet's last export of the run has been scraped; lag_s must cover its export interval plus one scrape. + `worker_pods` (WORKER_POD_FIELDS) joins `oom_events`; null when not given. """ def log(msg: str) -> None: if logs: @@ -638,6 +833,9 @@ def log(msg: str) -> None: prom_url, start_ts, end_ts, steady_start, lag_s, steady_end_ts=steady_end, ) + telemetry.setdefault("oom_events", {}).update( + {k: (worker_pods or {}).get(k) for k in WORKER_POD_FIELDS} + ) full_artifact = { "metadata": { @@ -689,6 +887,14 @@ def log(msg: str) -> None: active = telemetry.get("active_actors", {}).get("summary", {}) for p in ps: measurements[f"active_actors_{p}"] = active.get(p) + working_set = telemetry.get("working_set", {}) + for block in WORKING_SET_KEYS: + ws = working_set.get(block, {}).get("summary", {}) + for p in (*ps, "max"): + measurements[f"working_set_{block}_{p}_gb"] = ws.get(p) + oom = telemetry.get("oom_events", {}) + for key in ("node", "pod", *WORKER_POD_FIELDS): + measurements[f"oom_events_{key}"] = oom.get(key) jsonl_row = { "timestamp": data_ts, diff --git a/benchmarking/locust/tests/ate_api.py b/benchmarking/locust/tests/ate_api.py index 635bb60399..44f84a575a 100644 --- a/benchmarking/locust/tests/ate_api.py +++ b/benchmarking/locust/tests/ate_api.py @@ -110,7 +110,10 @@ def invoke_target(self) -> None: try: with traced_grpc("SuspendActor", self.__class__.__name__) as metadata: _, metadata.call = self.stub.SuspendActor.with_call( - ateapi_pb2.SuspendActorRequest(actor=self.actor_ref), + ateapi_pb2.SuspendActorRequest( + actor=self.actor_ref, + options=ateapi_pb2.SuspendOptions(mode=ateapi_pb2.DURABILITY_MODE_SYNCHRONOUS), + ), metadata=metadata, ) except Exception: diff --git a/benchmarking/locust/tests/counter_demo.py b/benchmarking/locust/tests/counter_demo.py index deef074024..afeed39a70 100644 --- a/benchmarking/locust/tests/counter_demo.py +++ b/benchmarking/locust/tests/counter_demo.py @@ -106,7 +106,10 @@ def on_stop(self) -> None: try: with traced_grpc("SuspendActor", self.__class__.__name__) as metadata: _, metadata.call = self.stub.SuspendActor.with_call( - ateapi_pb2.SuspendActorRequest(actor=self.actor_ref), + ateapi_pb2.SuspendActorRequest( + actor=self.actor_ref, + options=ateapi_pb2.SuspendOptions(mode=ateapi_pb2.DURABILITY_MODE_SYNCHRONOUS), + ), metadata=metadata, ) except Exception as e: @@ -170,7 +173,10 @@ def run_and_suspend(self) -> None: try: with traced_grpc("SuspendActor", self.__class__.__name__) as metadata: _, metadata.call = self.stub.SuspendActor.with_call( - ateapi_pb2.SuspendActorRequest(actor=self.actor_ref), + ateapi_pb2.SuspendActorRequest( + actor=self.actor_ref, + options=ateapi_pb2.SuspendOptions(mode=ateapi_pb2.DURABILITY_MODE_SYNCHRONOUS), + ), metadata=metadata, ) except Exception: diff --git a/benchmarking/locust/tests/kernelmem.py b/benchmarking/locust/tests/kernelmem.py index 8e09ddccbf..2b4d583d3d 100644 --- a/benchmarking/locust/tests/kernelmem.py +++ b/benchmarking/locust/tests/kernelmem.py @@ -103,7 +103,10 @@ def workload_cycle(self) -> None: try: with traced_grpc("SuspendActor", self.__class__.__name__) as metadata: _, metadata.call = self.stub.SuspendActor.with_call( - ateapi_pb2.SuspendActorRequest(actor=self.actor_ref), + ateapi_pb2.SuspendActorRequest( + actor=self.actor_ref, + options=ateapi_pb2.SuspendOptions(mode=ateapi_pb2.DURABILITY_MODE_SYNCHRONOUS), + ), metadata=metadata, ) except Exception: diff --git a/benchmarking/locust/tests/sleep.py b/benchmarking/locust/tests/sleep.py index 162164473a..40ea7787fd 100644 --- a/benchmarking/locust/tests/sleep.py +++ b/benchmarking/locust/tests/sleep.py @@ -103,7 +103,10 @@ def workload_cycle(self) -> None: try: with traced_grpc("SuspendActor", self.__class__.__name__) as metadata: _, metadata.call = self.stub.SuspendActor.with_call( - ateapi_pb2.SuspendActorRequest(actor=self.actor_ref), + ateapi_pb2.SuspendActorRequest( + actor=self.actor_ref, + options=ateapi_pb2.SuspendOptions(mode=ateapi_pb2.DURABILITY_MODE_SYNCHRONOUS), + ), metadata=metadata, ) except Exception: diff --git a/benchmarking/locust/tests/usermem.py b/benchmarking/locust/tests/usermem.py index 78a257c674..b49d4ea182 100644 --- a/benchmarking/locust/tests/usermem.py +++ b/benchmarking/locust/tests/usermem.py @@ -103,7 +103,10 @@ def workload_cycle(self) -> None: try: with traced_grpc("SuspendActor", self.__class__.__name__) as metadata: _, metadata.call = self.stub.SuspendActor.with_call( - ateapi_pb2.SuspendActorRequest(actor=self.actor_ref), + ateapi_pb2.SuspendActorRequest( + actor=self.actor_ref, + options=ateapi_pb2.SuspendOptions(mode=ateapi_pb2.DURABILITY_MODE_SYNCHRONOUS), + ), metadata=metadata, ) except Exception: diff --git a/benchmarking/locust/unit_tests/test_boomer_config.py b/benchmarking/locust/unit_tests/test_boomer_config.py new file mode 100644 index 0000000000..93455361df --- /dev/null +++ b/benchmarking/locust/unit_tests/test_boomer_config.py @@ -0,0 +1,54 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for common/boomer_config.py.""" + +import json +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from common import boomer_config + + +class TestBoomerConfig(unittest.TestCase): + def test_build_config_json_lifecycle_mode_none(self): + raw = boomer_config.build_config_json([ + "--lifecycle-mode=none", + "--agentsession-script=personal-assistant-light", + ]) + parsed = json.loads(raw) + self.assertEqual(parsed["lifecycle_mode"], "none") + self.assertEqual( + parsed["agentsession_script"], "personal-assistant-light" + ) + + def test_build_config_json_snapshot_steps(self): + raw = boomer_config.build_config_json([ + "--lifecycle-mode", + "suspend", + "--agentsession-snapshot-steps", + "3,10,20,30,40,50,61", + ]) + parsed = json.loads(raw) + self.assertEqual(parsed["lifecycle_mode"], "suspend") + self.assertEqual( + parsed["agentsession_snapshot_steps"], "3,10,20,30,40,50,61" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarking/locust/unit_tests/test_cluster_facts.py b/benchmarking/locust/unit_tests/test_cluster_facts.py index 713748b009..13ad41b08b 100644 --- a/benchmarking/locust/unit_tests/test_cluster_facts.py +++ b/benchmarking/locust/unit_tests/test_cluster_facts.py @@ -27,6 +27,7 @@ import sys import tempfile import unittest +from datetime import datetime, timezone from pathlib import Path from types import SimpleNamespace from unittest import mock @@ -77,13 +78,29 @@ def pod(phase="Running", node_name="node-a", containers=()): status=SimpleNamespace(phase=phase)) -def fake_api(nodes=None, ns_pods=None): +def worker_pod(uid, restarts=0, last=None, evicted=False, statuses=True): + """A worker pod for the snapshot: the worker container's restarts and last exit reason.""" + worker = SimpleNamespace( + name="worker", restart_count=restarts, + last_state=SimpleNamespace( + terminated=SimpleNamespace(reason=last) if last else None)) + sidecar = SimpleNamespace(name="sidecar", restart_count=9, last_state=None) + return SimpleNamespace( + metadata=SimpleNamespace(uid=uid), + status=SimpleNamespace( + container_statuses=[sidecar, worker] if statuses else None, + reason="Evicted" if evicted else None)) + + +def fake_api(nodes=None, ns_pods=None, events=None): """Stand-in CoreV1Api; None raises 403 Forbidden.""" def forbidden(*_args, **_kwargs): raise ApiException(status=403, reason="Forbidden") api = mock.Mock() - for attr, items in (("list_node", nodes), ("list_namespaced_pod", ns_pods)): + lists = (("list_node", nodes), ("list_namespaced_pod", ns_pods), + ("list_namespaced_event", events)) + for attr, items in lists: if items is None: getattr(api, attr).side_effect = forbidden else: @@ -98,6 +115,13 @@ def discover(api): return cluster_facts.get_cluster_hardware_facts() +def snapshot(api): + with mock.patch.object(cluster_facts, "_load_kube_config", return_value=True), \ + mock.patch.object(cluster_facts.client, "CoreV1Api", return_value=api), \ + contextlib.redirect_stdout(io.StringIO()): + return cluster_facts.snapshot_worker_pods() + + def summarize(facts, directory, stats=STATS_HEADER + ",Aggregated,100,25\n", users=10, user_counts=None, actors_per_user=None, run_start=None, run_end=None, metadata=None): @@ -433,6 +457,101 @@ def test_per_rpc_failure_ratios(self): self.assertNotIn("resume_actor_failure_ratio", row["measurements"]) self.assertNotIn("suspend_actor_failure_ratio", row["measurements"]) + def test_worker_pod_snapshot(self): + api = fake_api(ns_pods=[ + worker_pod("a", restarts=2, last="OOMKilled"), + worker_pod("b", restarts=1, last="Error"), + worker_pod("p", statuses=False), # Pending, no statuses yet + worker_pod("e", evicted=True, statuses=False), + ]) + # worker container only: the sidecar's 9 restarts never show. + self.assertEqual(snapshot(api), { + "a": {"restarts": 2, "last_oomkilled": True, "evicted": False}, + "b": {"restarts": 1, "last_oomkilled": False, "evicted": False}, + "p": {"restarts": 0, "last_oomkilled": False, "evicted": False}, + "e": {"restarts": 0, "last_oomkilled": False, "evicted": True}, + }) + api.list_namespaced_pod.assert_called_once_with( + namespace="benchmark-workloads", + label_selector="ate.dev/worker-pool", + _request_timeout=5, + ) + # A denied read or no credentials is unknown, not an empty pool. + self.assertIsNone(snapshot(fake_api(ns_pods=None))) + with mock.patch.object(cluster_facts, "_load_kube_config", return_value=False): + self.assertIsNone(cluster_facts.snapshot_worker_pods()) + + def test_evicted_pods_from_events(self): + def event(uid, last=None, event_time=None, created=None): + def ts(s): + return datetime.fromtimestamp(s, timezone.utc) if s else None + return SimpleNamespace( + involved_object=SimpleNamespace(uid=uid), + last_timestamp=ts(last), event_time=ts(event_time), + metadata=SimpleNamespace(creation_timestamp=ts(created))) + + api = fake_api(events=[ + event("a", last=120), + event("a", last=150), # same pod again: counted once + event("old", last=90), # evicted before the run + event("b", event_time=110), # events.k8s.io: no last_timestamp + event("c", created=130), # only creation_timestamp + ]) + with mock.patch.object(cluster_facts, "_load_kube_config", return_value=True), \ + mock.patch.object(cluster_facts.client, "CoreV1Api", return_value=api), \ + contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(cluster_facts.read_evicted_pods(100), {"a", "b", "c"}) + api.list_namespaced_event.assert_called_once_with( + namespace="benchmark-workloads", + field_selector="reason=Evicted,involvedObject.kind=Pod", + _request_timeout=5, + ) + # A denied read is unknown, not zero evictions. + api.list_namespaced_event.side_effect = ApiException(status=403) + self.assertIsNone(cluster_facts.read_evicted_pods(100)) + + def test_worker_pod_diff(self): + def state(restarts=0, oom=False, evicted=False): + return {"restarts": restarts, "last_oomkilled": oom, "evicted": evicted} + + before = {"a": state(), "b": state(1, oom=True), "e": state(evicted=True), + "gone": state()} # deleted in the run: lost + after = { + "a": state(2, oom=True), # 2 restarts, last OOMKilled: 1 pod + "b": state(1, oom=True), # an OOMKilled from before the run + "e": state(evicted=True), # evicted before the run + "c": state(1, oom=True), # new pod, restarted once + "f": state(evicted=True), # evicted in the run, still listed + } + # "f" is in both sources, "g" was deleted before the second snapshot. + self.assertEqual(cluster_facts.diff_worker_pods(before, after, {"f", "g"}), { + "worker_restarts": 3, "worker_oomkilled_pods": 2, "evicted": 2, + "worker_pods_lost": 1}) + # Each field is null only when its own source failed. + self.assertEqual(cluster_facts.diff_worker_pods(None, after, {"g"}), { + "worker_restarts": None, "worker_oomkilled_pods": None, "evicted": 1, + "worker_pods_lost": None}) + self.assertEqual(cluster_facts.diff_worker_pods(before, after, None), { + "worker_restarts": 3, "worker_oomkilled_pods": 2, "evicted": None, + "worker_pods_lost": 1}) + + def test_worker_pod_reads_follow_cluster_facts_flag(self): + with mock.patch.object(runner, "snapshot_worker_pods") as read, \ + mock.patch.object(runner, "read_evicted_pods") as events: + off = parse("--no-cluster-facts") + self.assertIsNone(runner.read_worker_pods(off, io.StringIO())) + self.assertIsNone(runner.read_evictions(off, 100, io.StringIO())) + read.assert_not_called() + events.assert_not_called() + # A failed read never stops the run. + with mock.patch.object(runner, "snapshot_worker_pods", + side_effect=RuntimeError("boom")), \ + mock.patch.object(runner, "read_evicted_pods", + side_effect=RuntimeError("boom")), \ + contextlib.redirect_stdout(io.StringIO()): + self.assertIsNone(runner.read_worker_pods(parse(), io.StringIO())) + self.assertIsNone(runner.read_evictions(parse(), 100, io.StringIO())) + if __name__ == "__main__": unittest.main() diff --git a/benchmarking/locust/unit_tests/test_server_telemetry.py b/benchmarking/locust/unit_tests/test_server_telemetry.py index 811a1b32c8..47ed4539d4 100644 --- a/benchmarking/locust/unit_tests/test_server_telemetry.py +++ b/benchmarking/locust/unit_tests/test_server_telemetry.py @@ -40,8 +40,8 @@ "end_ts": 105, "steady_start_ts": 100} -def ranges(packing, node=(), pod=(), active=()): - """A query_prometheus_range fake answering packing, PSI and active actors. +def ranges(packing, node=(), pod=(), active=(), ws=(), actor_ws=()): + """A query_prometheus_range fake: packing, PSI, active actors, working set. Pod PSI only answers the pod-slice selector, so a query that loses the id regex (and would also sum the pause container) gets nothing back. Node PSI @@ -56,6 +56,11 @@ def fake(_url, query, *_args, **_kwargs): return packing if "ate_actor_stats_sampled_actors" in query: return list(active) + if "ate_actor_stats_memory_working_set_bytes" in query: + return list(actor_ws) + # Before the PSI branch: the node working set query has container="node". + if "container_memory_working_set_bytes" in query: + return list(ws) if 'container="node"' in query: return list(node) if worker_nodes in query and pod_slice in query else [] return list(pod) if pod_slice in query else [] @@ -103,18 +108,24 @@ def parse(*extra): } -def snapshot_prom(quantile=None, start=None, end=None, at=(100, 105)): +def snapshot_prom(quantile=None, start=None, end=None, at=(100, 105), oom=None): """A query_prometheus_instant fake answering by query and timestamp. Every quantile read at the window end returns `quantile`; `start` and `end` map a SUMS name to its value at the two instants in `at`. A window delta query subtracts its `@` value, or zero when that series is absent. Anything - else, including a read at the wrong time, returns no series. + else, including a read at the wrong time, returns no series. Every OOM + query returns `oom`, or with a dict, the value of the selector it holds. """ by_time = {at[0]: start or {}, at[1]: end or {}} def fake(_url, query, time_ts=None): - if query.startswith("histogram_quantile"): + if "container_oom_events_total" in query: + val = oom + if isinstance(oom, dict): + # The existence check has no selector, so it finds the metric. + val = next((v for k, v in oom.items() if k in query), "1") + elif query.startswith("histogram_quantile"): val = quantile if time_ts == at[1] else None else: name = next((n for n, s in SUMS.items() if s in query), None) @@ -242,7 +253,11 @@ def test_packing_and_checkpoint_math(self, mock_instant, mock_range): "values": [[100, "0.5"], [105, "0.5"]]} for p in "ab"] mock_range.side_effect = ranges( [partial, full, idle], node=quiet, pod=pods, - active=[{"metric": {"instance": "atelet-a"}, "values": [[135, "3"]]}]) + active=[{"metric": {"instance": "atelet-a"}, "values": [[135, "3"]]}], + ws=[{"metric": {"instance": "node-a"}, + "values": [[100, str(2**30)], [105, str(2**30)]]}], + actor_ws=[{"metric": {"instance": "atelet-a"}, + "values": [[135, str(6 * 2**30)]]}]) # The default 70s lag reads the snapshot window at 135 and 140. mock_instant.side_effect = snapshot_prom( "11.5", @@ -252,7 +267,7 @@ def test_packing_and_checkpoint_math(self, mock_instant, mock_range): end={"count": "150", "size": str(100 * MB), "bytes": str(100 * MB), "seconds": "50", "restore_sum": "4", "restore_count": "8", "checkpoint_count": "25"}, - at=(135, 140)) + at=(135, 140), oom="0") with tempfile.TemporaryDirectory() as td: out_json = Path(td) / "server_summary.json" @@ -270,20 +285,22 @@ def test_packing_and_checkpoint_math(self, mock_instant, mock_range): data_ts="2026-01-01", tag="unit", test_name="unit-run", + worker_pods={"worker_restarts": 1, "worker_oomkilled_pods": 1, + "evicted": 0, "worker_pods_lost": 0}, ) summary = json.loads(out_json.read_text()) row = json.loads(out_jsonl.read_text().splitlines()[0]) sleep.assert_called_once_with(65) # until end 105 + 70, from 110 self.assertEqual(summary["metadata"]["atelet_lag_s"], 70) - # Packing and PSI are not shifted; active actors are, like snapshots. - windows = {("sampled_actors" in c.args[1], c.args[2:4]) + # Packing, PSI and cAdvisor are not shifted; atelet stats are, like snapshots. + windows = {("ate_actor_stats" in c.args[1], c.args[2:4]) for c in mock_range.call_args_list} self.assertEqual(windows, {(False, (100, 105)), (True, (135, 140))}) self.assertEqual( set(summary), {"metadata", "cluster_packing", "node_psi", "pod_psi", "snapshots", - "active_actors"}, + "active_actors", "working_set", "oom_events"}, ) packing = summary["cluster_packing"] self.assertEqual(packing["summary"]["p50"], 0.8) # 3 + 1 busy / 5 workers @@ -303,7 +320,7 @@ def test_packing_and_checkpoint_math(self, mock_instant, mock_range): self.assertEqual(row["metric"], "server_summary") m = row["measurements"] # Every source answered, so a key wired to a wrong name would read None. - self.assertEqual(len(m), 49) + self.assertEqual(len(m), 80) self.assertEqual([k for k, v in m.items() if v is None], []) # Every value a string, so one row's types match every other row's. self.assertTrue(all(isinstance(v, str) for v in m.values())) @@ -315,6 +332,11 @@ def test_packing_and_checkpoint_math(self, mock_instant, mock_range): self.assertEqual(m["restore_mean_s"], "0.5") self.assertEqual(m["checkpoint_mean_s"], "2.0") self.assertEqual(m["active_actors_p99"], "3.0") + # Read at 135, moved back to 100; 6 GiB over the 3 actors on atelet-a. + actor = summary["working_set"]["actor"] + self.assertEqual(actor["timeseries"][0]["timestamp"], 100) + self.assertEqual(m["working_set_actor_p50_gb"], "6.0") + self.assertEqual(m["working_set_per_actor_p50_gb"], "2.0") @mock.patch("server_telemetry.query_prometheus_range") @mock.patch("server_telemetry.query_prometheus_instant") @@ -464,6 +486,105 @@ def test_steady_window_bounds_packing_and_psi(self, mock_instant, mock_range): self.assertEqual(out["node_psi"]["cpu_stall_pct"]["max"], 0.5) self.assertEqual(out["pod_psi"]["cpu_stall_pct"]["max"], 0.5) + @mock.patch("server_telemetry.query_prometheus_instant") + def test_oom_pod_needs_both_reads(self, mock_instant): + leaf, worker = 'pod=""', 'namespace="benchmark-workloads"' + mock_instant.side_effect = snapshot_prom( + oom={'container="node"': "2", leaf: "3", worker: "1"}) + out = server_telemetry._count_oom_events("http://p", 100, 105) + self.assertEqual(out, {"node": 2, "pod": 4, + "window": {"start_ts": 100, "end_ts": 105}}) + # Half a best-effort read is no read. + mock_instant.side_effect = snapshot_prom( + oom={'container="node"': "2", leaf: None, worker: "1"}) + out = server_telemetry._count_oom_events("http://p", 100, 105) + self.assertEqual((out["node"], out["pod"]), (2, None)) + + @mock.patch("server_telemetry.query_prometheus_range") + def test_working_set_gib_and_steady_window(self, mock_range): + gib = 2**30 + mock_range.side_effect = ranges([], ws=[ + {"metric": {"instance": "node-a"}, + "values": [[90, str(8 * gib)], [100, str(gib)], [110, str(3 * gib)]]}, + {"metric": {"instance": "node-b"}, "values": [[110, str(gib)]]}, + ]) + out = server_telemetry._measure_working_set("http://p", 90, 110, 100, 110) + node = out["node"] + # The 8 GiB sample at 90 is ramp-up: in the timeseries, not the summary. + self.assertEqual((node["summary"]["p50"], node["summary"]["max"]), (1.0, 3.0)) + self.assertEqual(node["count"], 2) + self.assertEqual(node["timeseries"], [ + {"timestamp": 90, "total_gb": 8.0, "max_gb": 8.0}, + {"timestamp": 100, "total_gb": 1.0, "max_gb": 1.0}, + {"timestamp": 110, "total_gb": 4.0, "max_gb": 3.0}, + ]) + self.assertEqual(set(out), {"node", "worker", "atelet", "actor", "per_actor"}) + + @mock.patch("server_telemetry.query_prometheus_range") + def test_working_set_null_not_zero(self, mock_range): + mock_range.side_effect = ranges([]) + out = server_telemetry._measure_working_set("http://p", 90, 110, 100, 110) + for block in out.values(): + self.assertEqual(block, {"summary": NO_PERCENTILES, "count": None, + "timeseries": []}) + + @mock.patch("server_telemetry.query_prometheus_range") + def test_working_set_actor_keys_like_active_actors(self, mock_range): + # One atelet through 2 meter pods, a stale 4 GiB and a fresh 6 GiB. + gib = 2**30 + meter = [{"exported_instance": "a", "instance": f"meter-{i}"} for i in "01"] + mock_range.side_effect = ranges( + [], + ws=[{"metric": {"pod": p}, "values": [[100, str(gib)]]} for p in "ab"], + actor_ws=[{"metric": m, "values": [[100, str(v * gib)]]} + for m, v in zip(meter, (4, 6))], + active=[{"metric": m, "values": [[100, "2"]]} for m in meter]) + out = server_telemetry._measure_working_set("http://p", 90, 110, 100, 110) + self.assertEqual(out["actor"]["count"], 1) + self.assertEqual(out["per_actor"]["summary"]["min"], 3.0) # max 6 / 2 + # cAdvisor keeps the label key: 2 worker pods are still 2. + self.assertEqual(out["worker"]["count"], 2) + + def test_working_set_queries_keep_live_workers(self): + q = server_telemetry.WORKING_SET_QUERIES + live = "rate(container_cpu_usage_seconds_total" + for block in q.values(): + self.assertTrue(block.startswith("max by (")) # a restart has 2 series + self.assertIn(live, block) + self.assertIn('container="node"', q["node"]) + self.assertIn("and on (pod)", q["worker"]) + + def test_oom_query_shape(self): + q = server_telemetry._oom_growth_query('container="node"', 100, 600) + s = 'container_oom_events_total{container="node"}' + # Clamped per series inside the sum, so one reset can't hide a kill. + self.assertTrue(q.startswith(f"sum(clamp_min(max_over_time({s}[600s]) - ")) + self.assertIn(f"({s} @ 100 or max_over_time({s}[600s]) * 0), 0))", q) + self.assertTrue(q.endswith(" or vector(0)")) + # One backslash pair in PromQL, so the regex reads a literal dot. + self.assertIn(r"\\.scope/", server_telemetry.OOM_POD[0]) + + @mock.patch("server_telemetry.query_prometheus_instant") + def test_oom_metric_missing_is_null(self, mock_instant): + mock_instant.side_effect = snapshot_prom(oom=None) + self.assertEqual(server_telemetry._count_oom_events("http://p", 100, 105), { + "node": None, "pod": None, "window": {"start_ts": 100, "end_ts": 105}}) + + @mock.patch("server_telemetry.harvest_server_telemetry", + return_value={"oom_events": {"node": 0, "pod": 0}}) + def test_worker_pod_fields_null_when_not_read(self, _harvest): + with tempfile.TemporaryDirectory() as td: + out_json = Path(td) / "server_summary.json" + with contextlib.redirect_stdout(io.StringIO()): + server_telemetry.extract_and_record_server_telemetry( + "http://p", 100, 105, Path(td) / "missing.csv", out_json, + Path(td) / "stats.jsonl", "2026-01-01", "unit", "unit-run", + lag_s=0) + oom = json.loads(out_json.read_text())["oom_events"] + self.assertEqual(oom, {"node": 0, "pod": 0, "worker_restarts": None, + "worker_oomkilled_pods": None, "evicted": None, + "worker_pods_lost": None}) + @mock.patch("server_telemetry.query_prometheus_instant") def test_failed_delta_read_is_unknown(self, mock_instant): # A failed delta query is not taken as a series born in the window. diff --git a/benchmarking/monitoring.yaml b/benchmarking/monitoring.yaml index 66c13fdf22..772f85050e 100644 --- a/benchmarking/monitoring.yaml +++ b/benchmarking/monitoring.yaml @@ -188,10 +188,10 @@ data: # cAdvisor sends a very large number of metrics. Keep only the # metrics that the benchmarks read: kernel pressure stall (PSI), # container network bandwidth, disk I/O, memory RSS/working-set, - # CPU utilization, and CFS throttling. + # CPU utilization, CFS throttling, and OOM kill events. - source_labels: [__name__] action: keep - regex: container_((.*pressure.*)|network_(receive|transmit)_bytes_total|fs_(reads|writes)_bytes_total|memory_rss|memory_working_set_bytes|cpu_usage_seconds_total|cpu_cfs_.*|spec_memory_limit_bytes) + regex: container_((.*pressure.*)|network_(receive|transmit)_bytes_total|fs_(reads|writes)_bytes_total|memory_rss|memory_working_set_bytes|cpu_usage_seconds_total|cpu_cfs_.*|spec_memory_limit_bytes|oom_events_total) # Drop the series that have an empty container label only for cumulative # memory working set and CPU usage. cAdvisor gives a rollup for the whole # pod cgroup next to the series for each container; keeping both counts @@ -438,32 +438,32 @@ data: } }, { - "title": "PauseActor QPS", + "title": "SuspendActorAsync QPS", "type": "timeseries", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 48 }, "targets": [ - { "expr": "sum(rate(locust_requests_total_total{name=\"PauseActor\"}[1m])) or sum(rate(locust_requests_total{name=\"PauseActor\"}[1m]))", "refId": "A", "legendFormat": "Total" }, - { "expr": "sum(rate(locust_requests_total_total{name=\"PauseActor\"}[1m])) by (status) or sum(rate(locust_requests_total{name=\"PauseActor\"}[1m])) by (status)", "refId": "B", "legendFormat": "{{status}}" } + { "expr": "sum(rate(locust_requests_total_total{name=\"SuspendActorAsync\"}[1m])) or sum(rate(locust_requests_total{name=\"SuspendActorAsync\"}[1m]))", "refId": "A", "legendFormat": "Total" }, + { "expr": "sum(rate(locust_requests_total_total{name=\"SuspendActorAsync\"}[1m])) by (status) or sum(rate(locust_requests_total{name=\"SuspendActorAsync\"}[1m])) by (status)", "refId": "B", "legendFormat": "{{status}}" } ], "fieldConfig": { "defaults": { "unit": "req/sec" } } }, { - "title": "PauseActor Latency 99th Pct", + "title": "SuspendActorAsync Latency 99th Pct", "type": "timeseries", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 48 }, "targets": [ - { "expr": "histogram_quantile(0.99, sum(rate(locust_request_duration_milliseconds_bucket{name=\"PauseActor\"}[1m])) by (le))", "refId": "A", "legendFormat": "Total" }, - { "expr": "histogram_quantile(0.99, sum(rate(locust_request_duration_milliseconds_bucket{name=\"PauseActor\"}[1m])) by (le, status))", "refId": "B", "legendFormat": "{{status}}" } + { "expr": "histogram_quantile(0.99, sum(rate(locust_request_duration_milliseconds_bucket{name=\"SuspendActorAsync\"}[1m])) by (le))", "refId": "A", "legendFormat": "Total" }, + { "expr": "histogram_quantile(0.99, sum(rate(locust_request_duration_milliseconds_bucket{name=\"SuspendActorAsync\"}[1m])) by (le, status))", "refId": "B", "legendFormat": "{{status}}" } ], "fieldConfig": { "defaults": { "unit": "ms" } } }, { - "title": "PauseActor Latency Heatmap", + "title": "SuspendActorAsync Latency Heatmap", "type": "heatmap", "gridPos": { "h": 8, "w": 24, "x": 0, "y": 56 }, "targets": [ { - "expr": "sum(rate(locust_request_duration_milliseconds_bucket{name=\"PauseActor\"}[1m])) by (le)", + "expr": "sum(rate(locust_request_duration_milliseconds_bucket{name=\"SuspendActorAsync\"}[1m])) by (le)", "refId": "A" } ], @@ -483,13 +483,13 @@ data: } }, { - "title": "Hibernate Latency (Pause vs Suspend)", + "title": "Hibernate Latency (Async vs Sync Suspend)", "type": "timeseries", "gridPos": { "h": 8, "w": 24, "x": 0, "y": 64 }, "targets": [ - { "expr": "histogram_quantile(0.99, sum(rate(locust_request_duration_milliseconds_bucket{name=\"PauseActor\"}[1m])) by (le))", "refId": "A", "legendFormat": "PauseActor p99" }, + { "expr": "histogram_quantile(0.99, sum(rate(locust_request_duration_milliseconds_bucket{name=\"SuspendActorAsync\"}[1m])) by (le))", "refId": "A", "legendFormat": "SuspendActorAsync p99" }, { "expr": "histogram_quantile(0.99, sum(rate(locust_request_duration_milliseconds_bucket{name=\"SuspendActor\"}[1m])) by (le))", "refId": "B", "legendFormat": "SuspendActor p99" }, - { "expr": "histogram_quantile(0.50, sum(rate(locust_request_duration_milliseconds_bucket{name=\"PauseActor\"}[1m])) by (le))", "refId": "C", "legendFormat": "PauseActor p50" }, + { "expr": "histogram_quantile(0.50, sum(rate(locust_request_duration_milliseconds_bucket{name=\"SuspendActorAsync\"}[1m])) by (le))", "refId": "C", "legendFormat": "SuspendActorAsync p50" }, { "expr": "histogram_quantile(0.50, sum(rate(locust_request_duration_milliseconds_bucket{name=\"SuspendActor\"}[1m])) by (le))", "refId": "D", "legendFormat": "SuspendActor p50" } ], "fieldConfig": { "defaults": { "unit": "ms" } } diff --git a/benchmarking/nighthawk-egress/traffic-driver-template.yaml.tmpl b/benchmarking/nighthawk-egress/traffic-driver-template.yaml.tmpl index 7dc71a7ea5..2395016880 100644 --- a/benchmarking/nighthawk-egress/traffic-driver-template.yaml.tmpl +++ b/benchmarking/nighthawk-egress/traffic-driver-template.yaml.tmpl @@ -65,9 +65,6 @@ resources: quantity: "2" - name: memory quantity: 1Gi -sandboxConfig: - sandboxClass: gvisor - configName: gvisor-default snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/nighthawk/traffic-driver/ diff --git a/benchmarking/workloads/deploy.sh b/benchmarking/workloads/deploy.sh index 59c4d15b21..e994a68fd4 100755 --- a/benchmarking/workloads/deploy.sh +++ b/benchmarking/workloads/deploy.sh @@ -151,16 +151,8 @@ run_kubectl_ate() { } substitute() { - # SandboxConfig names are pinned per class in the ActorTemplates (rather - # than defaulted) so a stale config from a dirty teardown fails loudly - # instead of silently binding these workloads. gvisor-default is applied by - # hack/install-ate.sh; microvm is applied by hack/install-microvm-deps.sh. local manifest="$1" - local sandbox_config_name worker_resources="" - case "${SANDBOX_CLASS}" in - gvisor) sandbox_config_name="gvisor-default" ;; - microvm) sandbox_config_name="microvm" ;; - esac + local worker_resources="" # One flow-style line, so an unset WORKER_MEMORY leaves only a blank line. if [[ -n "${WORKER_MEMORY}" ]]; then worker_resources="resources: {requests: {memory: \"${WORKER_MEMORY}\"}, limits: {memory: \"${WORKER_MEMORY}\"}}" @@ -168,7 +160,6 @@ substitute() { sed -e "s|\${BUCKET_NAME}|${BUCKET_NAME}|g" \ -e "s|\${WORKER_COUNT}|${WORKER_COUNT}|g" \ -e "s|\${SANDBOX_CLASS}|${SANDBOX_CLASS}|g" \ - -e "s|\${SANDBOX_CONFIG_NAME}|${sandbox_config_name}|g" \ -e "s|\${OTLP_ENDPOINT}|${OTLP_ENDPOINT}|g" \ -e "s|\${ACTOR_MEMORY}|${ACTOR_MEMORY}|g" \ -e "s|\${ACTOR_CPU}|${ACTOR_CPU}|g" \ diff --git a/benchmarking/workloads/manifests/glutton-durdir-data-template.yaml.tmpl b/benchmarking/workloads/manifests/glutton-durdir-data-template.yaml.tmpl index 02ad6c98d7..75bb218164 100644 --- a/benchmarking/workloads/manifests/glutton-durdir-data-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/glutton-durdir-data-template.yaml.tmpl @@ -53,6 +53,3 @@ resources: snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_VOLUMES storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/glutton-durdir-data/ -sandboxConfig: - sandboxClass: ${SANDBOX_CLASS} - configName: ${SANDBOX_CONFIG_NAME} diff --git a/benchmarking/workloads/manifests/glutton-durdir-full-template.yaml.tmpl b/benchmarking/workloads/manifests/glutton-durdir-full-template.yaml.tmpl index 6e738624ab..4f40c33618 100644 --- a/benchmarking/workloads/manifests/glutton-durdir-full-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/glutton-durdir-full-template.yaml.tmpl @@ -50,6 +50,3 @@ resources: snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/glutton-durdir-full/ -sandboxConfig: - sandboxClass: ${SANDBOX_CLASS} - configName: ${SANDBOX_CONFIG_NAME} diff --git a/benchmarking/workloads/manifests/glutton-storage-template.yaml.tmpl b/benchmarking/workloads/manifests/glutton-storage-template.yaml.tmpl index c64e36e26a..24237aa064 100644 --- a/benchmarking/workloads/manifests/glutton-storage-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/glutton-storage-template.yaml.tmpl @@ -59,6 +59,3 @@ resources: snapshotConfig: onCommit: SNAPSHOT_CONTENT_SCOPE_FULL storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/glutton-storage/ -sandboxConfig: - sandboxClass: ${SANDBOX_CLASS} - configName: ${SANDBOX_CONFIG_NAME} diff --git a/benchmarking/workloads/manifests/glutton-template.yaml.tmpl b/benchmarking/workloads/manifests/glutton-template.yaml.tmpl index 8d26c6c643..4749253b32 100644 --- a/benchmarking/workloads/manifests/glutton-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/glutton-template.yaml.tmpl @@ -14,8 +14,6 @@ # ActorTemplate (protojson-shaped ateapipb.ActorTemplate) for the glutton # benchmark workload, created through the ate API by workloads/deploy.sh. -# The sandbox class must match the WorkerPool's so snapshots stay within a -# class. metadata: atespace: benchmark-workloads name: glutton @@ -64,6 +62,3 @@ resources: snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/glutton/ -sandboxConfig: - sandboxClass: ${SANDBOX_CLASS} - configName: ${SANDBOX_CONFIG_NAME} diff --git a/benchmarking/workloads/manifests/kernelmem-template.yaml.tmpl b/benchmarking/workloads/manifests/kernelmem-template.yaml.tmpl index b39d7bb26a..2849c9a494 100644 --- a/benchmarking/workloads/manifests/kernelmem-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/kernelmem-template.yaml.tmpl @@ -18,8 +18,7 @@ # part of deploy.sh's default template set; deploy with # WORKLOAD_TEMPLATES="kernelmem ...". Its memory limit is fixed rather than # --actor-memory: the 50000 sleeping processes need far more than the 256Mi -# default. The sandbox class must match the WorkerPool's so snapshots stay -# within a class. +# default. metadata: atespace: benchmark-workloads name: kernelmem @@ -41,6 +40,3 @@ resources: snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/kernelmem/ -sandboxConfig: - sandboxClass: ${SANDBOX_CLASS} - configName: ${SANDBOX_CONFIG_NAME} diff --git a/benchmarking/workloads/manifests/sleep-template.yaml.tmpl b/benchmarking/workloads/manifests/sleep-template.yaml.tmpl index 0a30855682..a4692ff5d3 100644 --- a/benchmarking/workloads/manifests/sleep-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/sleep-template.yaml.tmpl @@ -14,8 +14,6 @@ # ActorTemplate (protojson-shaped ateapipb.ActorTemplate) for the sleep # benchmark workload, created through the ate API by workloads/deploy.sh. -# The sandbox class must match the WorkerPool's so snapshots stay within a -# class. metadata: atespace: benchmark-workloads name: sleep @@ -39,6 +37,3 @@ resources: snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/sleep/ -sandboxConfig: - sandboxClass: ${SANDBOX_CLASS} - configName: ${SANDBOX_CONFIG_NAME} diff --git a/benchmarking/workloads/manifests/sweperf/generated/sweperf-astropy-7336-template.yaml.tmpl b/benchmarking/workloads/manifests/sweperf/generated/sweperf-astropy-7336-template.yaml.tmpl index 050f1d9676..55ee16942d 100644 --- a/benchmarking/workloads/manifests/sweperf/generated/sweperf-astropy-7336-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/sweperf/generated/sweperf-astropy-7336-template.yaml.tmpl @@ -47,9 +47,6 @@ resources: quantity: "2" - name: memory quantity: 4Gi -sandboxConfig: - sandboxClass: ${SANDBOX_CLASS} - configName: ${SANDBOX_CONFIG_NAME} snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/sweperf-astropy-7336/ diff --git a/benchmarking/workloads/manifests/sweperf/generated/sweperf-django-11099-template.yaml.tmpl b/benchmarking/workloads/manifests/sweperf/generated/sweperf-django-11099-template.yaml.tmpl index 43ce70e1ff..da8965ea68 100644 --- a/benchmarking/workloads/manifests/sweperf/generated/sweperf-django-11099-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/sweperf/generated/sweperf-django-11099-template.yaml.tmpl @@ -47,9 +47,6 @@ resources: quantity: "2" - name: memory quantity: 4Gi -sandboxConfig: - sandboxClass: ${SANDBOX_CLASS} - configName: ${SANDBOX_CONFIG_NAME} snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/sweperf-django-11099/ diff --git a/benchmarking/workloads/manifests/sweperf/generated/sweperf-matplotlib-20826-template.yaml.tmpl b/benchmarking/workloads/manifests/sweperf/generated/sweperf-matplotlib-20826-template.yaml.tmpl index 315add0315..16d29b9ce7 100644 --- a/benchmarking/workloads/manifests/sweperf/generated/sweperf-matplotlib-20826-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/sweperf/generated/sweperf-matplotlib-20826-template.yaml.tmpl @@ -47,9 +47,6 @@ resources: quantity: "2" - name: memory quantity: 4Gi -sandboxConfig: - sandboxClass: ${SANDBOX_CLASS} - configName: ${SANDBOX_CONFIG_NAME} snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/sweperf-matplotlib-20826/ diff --git a/benchmarking/workloads/manifests/sweperf/generated/sweperf-sphinx-8638-template.yaml.tmpl b/benchmarking/workloads/manifests/sweperf/generated/sweperf-sphinx-8638-template.yaml.tmpl index df415cc62e..555db42def 100644 --- a/benchmarking/workloads/manifests/sweperf/generated/sweperf-sphinx-8638-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/sweperf/generated/sweperf-sphinx-8638-template.yaml.tmpl @@ -47,9 +47,6 @@ resources: quantity: "2" - name: memory quantity: 4Gi -sandboxConfig: - sandboxClass: ${SANDBOX_CLASS} - configName: ${SANDBOX_CONFIG_NAME} snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/sweperf-sphinx-8638/ diff --git a/benchmarking/workloads/manifests/sweperf/generated/sweperf-sympy-13647-template.yaml.tmpl b/benchmarking/workloads/manifests/sweperf/generated/sweperf-sympy-13647-template.yaml.tmpl index 12310ad3fe..77459b205c 100644 --- a/benchmarking/workloads/manifests/sweperf/generated/sweperf-sympy-13647-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/sweperf/generated/sweperf-sympy-13647-template.yaml.tmpl @@ -47,9 +47,6 @@ resources: quantity: "2" - name: memory quantity: 4Gi -sandboxConfig: - sandboxClass: ${SANDBOX_CLASS} - configName: ${SANDBOX_CONFIG_NAME} snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/sweperf-sympy-13647/ diff --git a/benchmarking/workloads/manifests/sweperf/sweperf-template.yaml.j2 b/benchmarking/workloads/manifests/sweperf/sweperf-template.yaml.j2 index 47a259b7ee..9fb8514a40 100644 --- a/benchmarking/workloads/manifests/sweperf/sweperf-template.yaml.j2 +++ b/benchmarking/workloads/manifests/sweperf/sweperf-template.yaml.j2 @@ -47,9 +47,6 @@ resources: quantity: "2" - name: memory quantity: 4Gi -sandboxConfig: - sandboxClass: ${SANDBOX_CLASS} - configName: ${SANDBOX_CONFIG_NAME} snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/{{ name }}/ diff --git a/benchmarking/workloads/manifests/usermem-template.yaml.tmpl b/benchmarking/workloads/manifests/usermem-template.yaml.tmpl index 75553f0120..e717ec174a 100644 --- a/benchmarking/workloads/manifests/usermem-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/usermem-template.yaml.tmpl @@ -17,8 +17,7 @@ # memory to size memory snapshots. Not part of deploy.sh's default template # set; deploy with WORKLOAD_TEMPLATES="usermem ...". Its memory limit is # fixed rather than --actor-memory: the workload allocates 1 GiB, well above -# the 256Mi default. The sandbox class must match the WorkerPool's so -# snapshots stay within a class. +# the 256Mi default. metadata: atespace: benchmark-workloads name: usermem @@ -39,6 +38,3 @@ resources: snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/usermem/ -sandboxConfig: - sandboxClass: ${SANDBOX_CLASS} - configName: ${SANDBOX_CONFIG_NAME} diff --git a/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml b/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml index 716da1ee32..56e30286c5 100644 --- a/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml +++ b/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml @@ -44,8 +44,9 @@ spec: openAPIV3Schema: description: |- SandboxConfig is cluster-scoped configuration describing the versions of the - sandbox binaries for a sandbox runtime family. It is referenced by an - ActorTemplate's sandbox_config.config_name (required) and decouples sandbox + sandbox binaries for a sandbox runtime family. It is referenced by a + WorkerPool's spec.sandboxClasses[].configRef (or resolved via the + sandboxconfig.ate.dev/is-class-default annotation) and decouples sandbox binary selection from the workload definition. properties: apiVersion: @@ -78,9 +79,9 @@ spec: sandboxClass: default: gvisor description: |- - sandboxClass is the sandbox runtime family this config applies to. An - ActorTemplate only uses SandboxConfigs whose SandboxClass matches its - sandbox_config.sandbox_class. + sandboxClass is the sandbox runtime family this config applies to. A + WorkerPool only uses SandboxConfigs whose sandboxClass matches the entry + in its spec.sandboxClasses. enum: - gvisor - microvm @@ -128,11 +129,10 @@ spec: architecture (GOARCH, e.g. "amd64", "arm64") and then by asset name. The asset names are interpreted by the sandbox backend: gVisor expects a "gvisor" asset (the release's gvisor.tar.zstd, which atelet extracts so - the gvisor-bin/ helpers sit next to runsc; a legacy bare-binary "runsc" - asset is still accepted); a micro-VM backend expects several (e.g. - "cloud-hypervisor", "kata-kernel", "kata-image"). The schema is - intentionally generic; per-class requirements are enforced by a - ValidatingAdmissionPolicy. + the gvisor-bin/ helpers sit next to runsc); a micro-VM backend expects + several (e.g. "cloud-hypervisor", "kata-kernel", "kata-image"). The + schema is intentionally generic; per-class requirements are enforced by + a ValidatingAdmissionPolicy. type: object name: description: |- diff --git a/charts/substrate-crds/templates/ate.dev_workerpools.yaml b/charts/substrate-crds/templates/ate.dev_workerpools.yaml index 31a1574ce8..bc4e72e16f 100644 --- a/charts/substrate-crds/templates/ate.dev_workerpools.yaml +++ b/charts/substrate-crds/templates/ate.dev_workerpools.yaml @@ -107,9 +107,7 @@ spec: name selects the sandbox runtime family, which drives the worker pod shape (KVM/vhost device mounts and node placement). The concrete binary is still selected by WorkerImage. The sandbox binaries themselves come - from the SandboxConfig each ActorTemplate names (required). - - See Also: TODOs in ActorTemplate SandboxClass + from the SandboxConfig referenced by ConfigRef (or the class default). enum: - gvisor - microvm diff --git a/charts/substrate/README.md b/charts/substrate/README.md index 6a7d7e9ff8..4f04e61153 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -66,7 +66,7 @@ See `values.yaml` for the full set; the important keys: | `atelet.imageCredentialProviderConfigYAML` | `""` | Credential provider config as text, rendered into a ConfigMap, for nodes where that file cannot be authored or must differ from the kubelet's; mutually exclusive with the host path | | `atelet.imageCredentialProviderBinDir` | `""` | Host path to the kubelet credential provider binaries; both paths are mounted read-only | | `credentialProvider.namespacePolicies` | `[]` | Default-deny atespace-to-namespace grants; the chart includes get-only Secret RBAC for the provider | -| `ateApi.actorJWTIssuer` | `https://idp..svc` | Issuer placed in actor JWTs; set a stable URL reachable by relying parties | +| `ateApi.actorJWTIssuer` | `https://..svc` | Issuer placed in actor JWTs; set a stable URL reachable by relying parties | | `objectStorePlugin.extraEnv` | `[]` | Object storage credentials/settings shared by both object-store plugins | | `objectStorePlugin.control.resources` / `objectStorePlugin.node.resources` | see `values.yaml` | Resources for the API and node object-store sidecars | | `ateApi.logLevel` | `info` | API server log level in its mounted config; one of `debug`, `info`, `warn`, or `error` | @@ -92,8 +92,8 @@ memory values. Worker images are now named `ateworker-gvisor` and Upgrade `substrate-crds` before the application chart. Custom `SandboxConfig` resources now put assets and `pauseImage` under `spec.versions[]`, with a named -`spec.defaultVersion`. WorkerPools use `spec.sandboxClasses[]`, and actor -templates continue to select the config with `sandboxConfig.configName`. +`spec.defaultVersion`. WorkerPools select the runtime through +`spec.sandboxClasses[]`; remove `sandboxConfig` from actor templates. WorkerPools now resolve `spec.sandboxClasses[].configRef` or the class default before creating their worker Deployment. Ensure each pool names an existing @@ -121,9 +121,10 @@ verifying and publishing them. `ATE_STORAGE_BACKEND` must not be set on the API server or atelet itself; select it through `atelet.storageBackend`. RustFS is wired automatically. Private S3 asset URLs must use `s3://`. -The default gVisor config now uses the 2026-10-10 nightly (`v3`) for split -filesystem checkpoints. Upgrade the sandbox config together with ateworker-gvisor; -custom configs must pin a release with the same checkpoint support. +The default gVisor config now uses the 2026-10-17 nightly (`v4`) for separate +container-rootfs and durable-directory checkpoint bundles. Upgrade the sandbox config together with ateworker-gvisor; +custom configs must pin a release with the same checkpoint support and supply +the `gvisor` release tarball asset. Bare `runsc` assets are no longer accepted. The chart now denies ingress to the control-plane namespace by default and allows the component, router, egress, storage, and monitoring traffic it needs. @@ -195,5 +196,13 @@ atelet, and workers together for the checkpoint validation and per-actor sandbox asset paths. The `ate-idp-server` binary serves actor JWT discovery and public signing keys -over HTTPS. It is not deployed by this chart yet; upgrading the chart does not -expose a discovery endpoint. +over HTTPS. The chart deploys it behind the `idp` Service (prefixed with the +release name for non-default releases) and permits HTTPS discovery traffic. +The default actor JWT issuer points at this Service; external consumers need +an externally reachable issuer URL and routing to it. + +Atelet now listens on TCP port 21998; update external firewall rules that +allowed port 8085. Replace `PauseActor` calls with +`SuspendActor` and `options.mode: DURABILITY_MODE_ASYNCHRONOUS`. +`SuspendActor` defaults to asynchronous durability; callers that require an +external snapshot before returning must request `DURABILITY_MODE_SYNCHRONOUS`. diff --git a/charts/substrate/templates/_helpers.tpl b/charts/substrate/templates/_helpers.tpl index ee4501e086..d8a497a4ef 100644 --- a/charts/substrate/templates/_helpers.tpl +++ b/charts/substrate/templates/_helpers.tpl @@ -44,6 +44,11 @@ get the same prefixed name that this subchart's own templates render. {{- end -}} {{- end -}} +{{/* The API signs actor JWTs with the issuer served by the discovery endpoint. */}} +{{- define "substrate.actorJWTIssuer" -}} +{{- .Values.ateApi.actorJWTIssuer | default (printf "https://%s.%s.svc" (include "substrate.fullname" (list "idp" .)) .Release.Namespace) -}} +{{- end -}} + {{/* ServiceAccount name of ate-api-server, as this chart creates it. Parent charts that need to bind additional Roles to this SA (e.g. env-source diff --git a/charts/substrate/templates/ate-api-server-envvars.yaml b/charts/substrate/templates/ate-api-server-envvars.yaml index 26f8f0edc8..8b04a2015e 100644 --- a/charts/substrate/templates/ate-api-server-envvars.yaml +++ b/charts/substrate/templates/ate-api-server-envvars.yaml @@ -27,4 +27,4 @@ metadata: namespace: {{ .Release.Namespace }} data: ATE_API_POSTGRES_SCHEMA: {{ .Values.postgres.schema | quote }} - ATE_API_ACTOR_JWT_ISSUER: {{ .Values.ateApi.actorJWTIssuer | default (printf "https://idp.%s.svc" .Release.Namespace) | quote }} + ATE_API_ACTOR_JWT_ISSUER: {{ include "substrate.actorJWTIssuer" . | quote }} diff --git a/charts/substrate/templates/ate-idp-server.yaml b/charts/substrate/templates/ate-idp-server.yaml new file mode 100644 index 0000000000..67f7c16741 --- /dev/null +++ b/charts/substrate/templates/ate-idp-server.yaml @@ -0,0 +1,121 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ate-idp-server serves the OpenID discovery document and key set relying +# parties use to verify actor JWTs. It needs no Kubernetes API access. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "ate-idp-server" .) }} + namespace: {{ .Release.Namespace }} +automountServiceAccountToken: false +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "ate-idp-server" .) }} + namespace: {{ .Release.Namespace }} +spec: + replicas: 2 + strategy: + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + selector: + matchLabels: + app: ate-idp-server + template: + metadata: + labels: + app: ate-idp-server + spec: + {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} + serviceAccountName: {{ include "substrate.fullname" (list "ate-idp-server" .) }} + containers: + - name: ate-idp-server + image: {{ include "substrate.componentImage" (list "ate-idp-server" .) }} + resources: +{{ toYaml .Values.ateIdpServer.resources | indent 10 }} + args: + - {{ printf "--issuer=%s" (include "substrate.actorJWTIssuer" .) | quote }} + - --actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json + - --server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem + ports: + - name: https + containerPort: 8443 + readinessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + periodSeconds: 5 + livenessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + periodSeconds: 10 + securityContext: + capabilities: + drop: + - ALL + volumeMounts: + # The serving certificate, with a DNS SAN for each Service that + # selects the pod. + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + - name: actor-id-jwt-pool + mountPath: /run/actor-id-jwt-pool + volumes: + - name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + # The pool ate-api-server signs with. + - name: actor-id-jwt-pool + projected: + sources: + - secret: + name: actor-id-jwt-pool + items: + - key: pool + path: pool.json +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "substrate.fullname" (list "ate-idp-server" .) }} + namespace: {{ .Release.Namespace }} +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ate-idp-server +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "idp" .) }} + namespace: {{ .Release.Namespace }} +spec: + selector: + app: ate-idp-server + ports: + - name: https + port: 443 + targetPort: https + appProtocol: https diff --git a/charts/substrate/templates/ate-system-networkpolicies.yaml b/charts/substrate/templates/ate-system-networkpolicies.yaml index 6c8517918f..72430b546c 100644 --- a/charts/substrate/templates/ate-system-networkpolicies.yaml +++ b/charts/substrate/templates/ate-system-networkpolicies.yaml @@ -217,3 +217,23 @@ spec: port: stats - protocol: TCP port: readiness +--- +# ate-idp-server serves the actor JWT discovery document and key set. Relying +# parties fetch them from pods in any namespace and, through an Ingress or a +# load balancer, from outside the cluster. Both documents are public, so the +# port admits any source. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "substrate.fullname" (list "ate-idp-server" .) }} + namespace: {{ .Release.Namespace }} +spec: + podSelector: + matchLabels: + app: ate-idp-server + policyTypes: + - Ingress + ingress: + - ports: + - protocol: TCP + port: https diff --git a/charts/substrate/templates/atelet.yaml b/charts/substrate/templates/atelet.yaml index 66c0f86a01..278150d72f 100644 --- a/charts/substrate/templates/atelet.yaml +++ b/charts/substrate/templates/atelet.yaml @@ -57,9 +57,7 @@ rules: - apiGroups: ["certificates.k8s.io"] resources: ["clustertrustbundles"] verbs: ["get", "watch", "list"] -# SandboxConfigs are watched to pre-download sandbox assets into the node's -# cache before the first actor needs them (see cmd/atelet/sandbox_prewarm.go). -# WorkerPools identify the SandboxConfig for each local worker. +# WorkerPools and SandboxConfigs resolve and prewarm each local worker's runtimes. - apiGroups: ["ate.dev"] resources: ["sandboxconfigs", "workerpools"] verbs: ["get", "list", "watch"] @@ -135,6 +133,10 @@ spec: - key: ate.dev/sandboxClass operator: Exists effect: NoSchedule + # GKE reserves ComputeClass nodes with this taint. + - key: cloud.google.com/compute-class + operator: Exists + effect: NoSchedule {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} initContainers: - name: objectstore-plugin @@ -226,8 +228,8 @@ spec: {{- end }} ports: - name: grpc - containerPort: 8085 - hostPort: 8085 + containerPort: 21998 + hostPort: 21998 - name: prometheus containerPort: 9090 protocol: TCP diff --git a/charts/substrate/templates/sandboxconfig-gvisor.yaml b/charts/substrate/templates/sandboxconfig-gvisor.yaml index 055016f758..1f41aab78d 100644 --- a/charts/substrate/templates/sandboxconfig-gvisor.yaml +++ b/charts/substrate/templates/sandboxconfig-gvisor.yaml @@ -15,12 +15,12 @@ limitations under the License. */}} # Cluster-wide SandboxConfig for the gVisor (runsc) sandbox class, shipped with -# the platform so gVisor ActorTemplates have a config to name via -# sandboxConfig.configName. atelet fetches the gVisor release tarball +# the platform as the class default so gVisor WorkerPools resolve it unless +# they pin a specific configRef. atelet fetches the gVisor release tarball # (gvisor.tar.zstd: runsc plus the gvisor-bin/ helpers runsc requires next to # it) matching the worker node's architecture and extracts it locally. To pin a # different release, add a version below and point defaultVersion at it, or -# create another SandboxConfig and name it from the ActorTemplate. +# create another SandboxConfig and reference it from the WorkerPool. apiVersion: ate.dev/v1alpha1 kind: SandboxConfig metadata: @@ -29,12 +29,13 @@ metadata: sandboxconfig.ate.dev/is-class-default: "true" spec: sandboxClass: gvisor - defaultVersion: v3 + defaultVersion: v4 versions: - # gVisor nightly 2026-10-10, for the split filesystem checkpoint - # (runsc fscheckpoint, -fs-checkpoint-paths and -fs-restore-image-path) that - # ateworker-gvisor uses to save and restore durable-dir volumes. - - name: v3 + # gVisor nightly 2026-10-17, the first to include google/gvisor@b79256080dec + # (save and restore multiple filesystem checkpoint bundles). ateworker-gvisor + # uses it to save container rootfs overlays (rootfs/) and durable-dir volumes + # (fs/) as separate bundles. + - name: v4 state: Enabled # The root sandbox container's image. On GCP, prefer the in-project mirror # gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da. @@ -42,9 +43,9 @@ spec: assets: amd64: gvisor: - url: "gs://gvisor/releases/nightly/2026-10-10/x86_64/gvisor.tar.zstd" - sha256: "4ae7fab988c3647d2a835d6c3c70c6a99021c562db97827592562b6c51c3e88c" + url: "gs://gvisor/releases/nightly/2026-10-17/x86_64/gvisor.tar.zstd" + sha256: "fa72f0d45756c69a30beec21222c78241b0d330c36f4f2185648617cc23415c8" arm64: gvisor: - url: "gs://gvisor/releases/nightly/2026-10-10/aarch64/gvisor.tar.zstd" - sha256: "aaf414f06eea7960238e43165c691f0a36cefc0593fb141e011e96211c7b6f57" + url: "gs://gvisor/releases/nightly/2026-10-17/aarch64/gvisor.tar.zstd" + sha256: "238634978d99f0dd3e631c4f88eae7b19fb36937065428467f0b6d6b10697689" diff --git a/charts/substrate/templates/sandboxconfig-validation.yaml b/charts/substrate/templates/sandboxconfig-validation.yaml index 3ddbbbe2a2..947b84f173 100644 --- a/charts/substrate/templates/sandboxconfig-validation.yaml +++ b/charts/substrate/templates/sandboxconfig-validation.yaml @@ -33,15 +33,13 @@ spec: resources: ["sandboxconfigs"] validations: # gVisor needs a "gvisor" release-tarball asset (gvisor.tar.zstd: runsc plus - # its gvisor-bin/ helpers) for every architecture it advertises. A legacy - # bare-binary "runsc" asset is still accepted for releases that predate the - # tarball. + # its gvisor-bin/ helpers) for every architecture it advertises. - expression: >- object.spec.sandboxClass != 'gvisor' || object.spec.versions.all(v, has(v.assets) && size(v.assets) > 0 && - v.assets.all(arch, 'gvisor' in v.assets[arch] || 'runsc' in v.assets[arch])) - message: "a gvisor SandboxConfig must define a 'gvisor' (release tarball) or legacy 'runsc' asset for every architecture of every version under spec.versions[].assets" + v.assets.all(arch, 'gvisor' in v.assets[arch])) + message: "a gvisor SandboxConfig must define a 'gvisor' (release tarball) asset for every architecture of every version under spec.versions[].assets" # The micro-VM (cloud-hypervisor) runtime needs its asset set for every # architecture it advertises. - expression: >- diff --git a/charts/substrate/tests/control_plane_test.yaml b/charts/substrate/tests/control_plane_test.yaml index 798ec2cd8b..593aad8908 100644 --- a/charts/substrate/tests/control_plane_test.yaml +++ b/charts/substrate/tests/control_plane_test.yaml @@ -72,6 +72,12 @@ tests: path: kind value: DaemonSet asserts: + - contains: + path: spec.template.spec.containers[0].ports + content: + name: grpc + containerPort: 21998 + hostPort: 21998 - contains: path: spec.template.spec.containers[0].ports content: @@ -90,6 +96,12 @@ tests: - equal: path: spec.template.spec.containers[0].livenessProbe.httpGet.port value: health + - contains: + path: spec.template.spec.tolerations + content: + key: cloud.google.com/compute-class + operator: Exists + effect: NoSchedule - it: probes the credential provider on its health listener template: k8s-credential-provider.yaml documentSelector: diff --git a/charts/substrate/tests/idp_test.yaml b/charts/substrate/tests/idp_test.yaml new file mode 100644 index 0000000000..c8d3e50c11 --- /dev/null +++ b/charts/substrate/tests/idp_test.yaml @@ -0,0 +1,140 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +suite: Actor JWT discovery +templates: +- ate-idp-server.yaml +- ate-api-server-envvars.yaml +- ate-system-networkpolicies.yaml +release: + name: custom + namespace: custom-system +tests: +- it: serves discovery with the API signing pool and a service certificate + template: ate-idp-server.yaml + documentSelector: + path: kind + value: Deployment + set: + image.registry: mirror.example + image.tag: test + imagePullSecrets: + - name: registry-auth + ateIdpServer.resources.requests.memory: 64Mi + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --issuer=https://custom-idp.custom-system.svc + - equal: + path: spec.template.spec.serviceAccountName + value: custom-ate-idp-server + - equal: + path: spec.template.spec.containers[0].image + value: mirror.example/kagent-dev/substrate/ate-idp-server:test + - equal: + path: spec.template.spec.containers[0].resources.requests.memory + value: 64Mi + - contains: + path: spec.template.spec.imagePullSecrets + content: + name: registry-auth + - contains: + path: spec.template.spec.volumes + content: + name: actor-id-jwt-pool + projected: + sources: + - secret: + name: actor-id-jwt-pool + items: + - key: pool + path: pool.json + - contains: + path: spec.template.spec.volumes + content: + name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem +- it: does not grant the discovery server a Kubernetes API token + template: ate-idp-server.yaml + documentSelector: + path: kind + value: ServiceAccount + asserts: + - equal: + path: automountServiceAccountToken + value: false +- it: exposes the release discovery service on HTTPS + template: ate-idp-server.yaml + documentSelector: + path: kind + value: Service + asserts: + - equal: + path: metadata.name + value: custom-idp + - equal: + path: metadata.namespace + value: custom-system + - equal: + path: spec.ports + value: + - name: https + port: 443 + targetPort: https + appProtocol: https +- it: defaults the actor issuer to the release discovery service + template: ate-api-server-envvars.yaml + asserts: + - equal: + path: data.ATE_API_ACTOR_JWT_ISSUER + value: https://custom-idp.custom-system.svc +- it: preserves an externally configured actor issuer + template: ate-api-server-envvars.yaml + set: + ateApi.actorJWTIssuer: https://actors.example + asserts: + - equal: + path: data.ATE_API_ACTOR_JWT_ISSUER + value: https://actors.example +- it: serves discovery for the externally configured issuer + template: ate-idp-server.yaml + set: + ateApi.actorJWTIssuer: https://actors.example + documentSelector: + path: kind + value: Deployment + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --issuer=https://actors.example +- it: admits public HTTPS discovery through the default ingress deny + template: ate-system-networkpolicies.yaml + documentSelector: + path: metadata.name + value: custom-ate-idp-server + asserts: + - equal: + path: spec.podSelector.matchLabels.app + value: ate-idp-server + - equal: + path: spec.ingress + value: + - ports: + - protocol: TCP + port: https diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index 8bc5bc47f2..9dcd7f6473 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -124,6 +124,9 @@ ateApi: cpu: "2" memory: 1Gi +ateIdpServer: + resources: {} + ateController: resources: requests: diff --git a/cmd/ate-idp-server/main.go b/cmd/ate-idp-server/main.go index 8ab03fe758..aa4dee95a9 100644 --- a/cmd/ate-idp-server/main.go +++ b/cmd/ate-idp-server/main.go @@ -35,6 +35,7 @@ import ( "go.ate.dev/substrate/cmd/ate-idp-server/internal/server" "go.ate.dev/substrate/internal/credbundle" "go.ate.dev/substrate/internal/localjwtauthority" + "go.ate.dev/substrate/internal/preview" "go.ate.dev/substrate/internal/serverboot" "go.ate.dev/substrate/internal/version" ) @@ -46,6 +47,7 @@ var ( serverBundle = pflag.String("server-cred-bundle", "", "Credential bundle (PEM key and chain) presented for serving TLS (required).") logLevel = pflag.String("log-level", "info", "One of debug, info, warn, error.") drainGrace = pflag.Duration("drain-grace", 5*time.Second, "How long to wait for in-flight requests on shutdown.") + previewFlags = pflag.StringSlice("preview", nil, "Preview gates to enable.") ) func main() { @@ -56,6 +58,9 @@ func main() { if err := serverboot.SetLogLevel(*logLevel); err != nil { serverboot.Fatal(ctx, "invalid --log-level", err) } + if err := preview.Init(*previewFlags...); err != nil { + serverboot.Fatal(ctx, "invalid --preview", err) + } slog.InfoContext(ctx, "starting ate-idp-server", slog.String("version", version.String())) if err := run(ctx); err != nil { diff --git a/cmd/ate-setup/internal/cmd/deploy.go b/cmd/ate-setup/internal/cmd/deploy.go index 20cddbe891..bbee1390db 100644 --- a/cmd/ate-setup/internal/cmd/deploy.go +++ b/cmd/ate-setup/internal/cmd/deploy.go @@ -52,8 +52,8 @@ var deployAteSystemCmd = &cobra.Command{ Long: `Deploy the whole Agent Substrate control plane. This installs the CRDs and RBAC, the podcertificate controller and the secrets -it signs, PostgreSQL, ate-api-server, ate-controller, the atenet dataplane, and -the atelet DaemonSet, then waits for each to roll out. +it signs, PostgreSQL, ate-api-server, ate-idp-server, ate-controller, the atenet +dataplane, and the atelet DaemonSet, then waits for each to roll out. The bundled PostgreSQL StatefulSet is skipped when ATE_API_POSTGRES_READ_WRITE_CONNECTION_STRING or the ATE_API_POSTGRES_CLOUDSQL_* variables diff --git a/cmd/ate-setup/internal/config/config_test.go b/cmd/ate-setup/internal/config/config_test.go index e9d8c79dd4..d229737dbf 100644 --- a/cmd/ate-setup/internal/config/config_test.go +++ b/cmd/ate-setup/internal/config/config_test.go @@ -714,8 +714,9 @@ func TestLoadActorJWTIssuer(t *testing.T) { }{ {name: "unset", want: "https://idp.ate-system.svc"}, {name: "unset in a relocated install", namespace: "team-a", want: "https://idp.team-a.svc"}, - {name: "set is used as given", env: "https://idp.example.com/prod/", want: "https://idp.example.com/prod/"}, + {name: "set is used as given", env: "https://idp.example.com", want: "https://idp.example.com"}, {name: "not https", env: "http://idp.example.com", wantErr: true}, + {name: "path", env: "https://idp.example.com/prod", wantErr: true}, } { t.Run(tt.name, func(t *testing.T) { loadEnv(t) diff --git a/cmd/ate-setup/internal/config/load.go b/cmd/ate-setup/internal/config/load.go index 13b2ad9f29..da6c4d3c8a 100644 --- a/cmd/ate-setup/internal/config/load.go +++ b/cmd/ate-setup/internal/config/load.go @@ -24,8 +24,8 @@ import ( "github.com/spf13/pflag" "go.ate.dev/substrate/cmd/ate-setup/internal/images" + "go.ate.dev/substrate/internal/actoridjwt" "go.ate.dev/substrate/internal/installdefaults" - "go.ate.dev/substrate/internal/oidcdiscovery" "go.ate.dev/substrate/internal/preview" ) @@ -429,7 +429,7 @@ func validateResolved(cfg *Config, r *Resolved) error { alg, _ := r.Value("actorJWT.algorithm") return &InvalidError{Value: alg, Want: "ES256 or RS256"} } - if err := oidcdiscovery.ValidateIssuer(cfg.ActorJWTIssuer); err != nil { + if err := actoridjwt.ValidateIssuer(cfg.ActorJWTIssuer); err != nil { issuer, _ := r.Value("actorJWT.issuer") return fmt.Errorf("%s: %w (from %s)", issuer.Setting.Key, err, issuer.From.Describe(issuer.Setting)) } diff --git a/cmd/ate-setup/internal/config/setting.go b/cmd/ate-setup/internal/config/setting.go index f6d829bc73..216425f2c6 100644 --- a/cmd/ate-setup/internal/config/setting.go +++ b/cmd/ate-setup/internal/config/setting.go @@ -125,8 +125,8 @@ var Registry = []Setting{ { // The default depends on the namespace, so buildConfig applies it. Key: "actorJWT.issuer", Env: "ACTOR_JWT_ISSUER", Flag: "actor-jwt-issuer", Kind: KindString, - Usage: "iss claim of actor JWTs, verbatim. Relying parties fetch /.well-known/openid-configuration " + - "(default https://idp..svc)", + Usage: "iss claim of actor JWTs, as https:// with no path. Relying parties fetch " + + "/.well-known/openid-configuration (default https://idp..svc)", }, { // Whitespace-separated, as the environment variable is. The registry diff --git a/cmd/ate-setup/internal/images/images.go b/cmd/ate-setup/internal/images/images.go index 6cc862dd7c..79ae3fd69b 100644 --- a/cmd/ate-setup/internal/images/images.go +++ b/cmd/ate-setup/internal/images/images.go @@ -40,6 +40,7 @@ const ModulePath = "go.ate.dev/substrate" // under the same repository and tag as the rest before a pre-built install can // use it. var Components = []string{ + "cmd/ate-idp-server", "cmd/ateapi", "cmd/atecontroller", "cmd/atelet", diff --git a/cmd/ate-setup/internal/steps/deploy.go b/cmd/ate-setup/internal/steps/deploy.go index 02ab4737a4..c8defae1af 100644 --- a/cmd/ate-setup/internal/steps/deploy.go +++ b/cmd/ate-setup/internal/steps/deploy.go @@ -167,6 +167,7 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { var waits []rollout waits = append(waits, rollout{kube.KindDeployment, "ate-api-server"}, + rollout{kube.KindDeployment, "ate-idp-server"}, rollout{kube.KindDeployment, "ate-controller"}, rollout{kube.KindDeployment, "atenet-router"}, rollout{kube.KindDeployment, "atenet-egress"}, @@ -568,10 +569,8 @@ func (e *Env) DeploySandboxConfig(ctx context.Context) error { return err } - // Install the cluster-wide sandbox config. Sandbox binaries live on - // cluster-scoped SandboxConfigs each ActorTemplate names via - // sandboxConfig.configName; gVisor templates name this one unless they - // create their own SandboxConfig. + // Install the cluster-wide default gVisor SandboxConfig that gVisor + // WorkerPools resolve by default unless they pin their own configRef. return e.Kube.ApplyPath(ctx, e.Cfg.Manifest("sandboxconfig-gvisor.yaml")) } diff --git a/cmd/ate-setup/internal/steps/idpserver.go b/cmd/ate-setup/internal/steps/idpserver.go new file mode 100644 index 0000000000..ba74c4d21b --- /dev/null +++ b/cmd/ate-setup/internal/steps/idpserver.go @@ -0,0 +1,68 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package steps + +import ( + "bytes" + "errors" + "fmt" + "strings" + + "sigs.k8s.io/kustomize/kyaml/kio" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +const idpIssuerFlag = "--issuer" + +// setIDPServerIssuer sets --issuer on the ate-idp-server container in manifest, +// replacing any --issuer it already carries. +func setIDPServerIssuer(manifest []byte, issuer string) ([]byte, error) { + nodes, err := (&kio.ByteReader{ + Reader: bytes.NewReader(manifest), + OmitReaderAnnotations: true, + }).Read() + if err != nil { + return nil, fmt.Errorf("while parsing manifest: %w", err) + } + for _, node := range nodes { + if node.GetKind() != "Deployment" || node.GetName() != "ate-idp-server" { + continue + } + container, err := node.Pipe(yaml.Lookup("spec", "template", "spec", "containers", "[name=ate-idp-server]")) + if err != nil { + return nil, err + } + if container == nil { + return nil, errors.New("deployment/ate-idp-server has no ate-idp-server container") + } + args, err := container.Pipe(yaml.LookupCreate(yaml.SequenceNode, "args")) + if err != nil { + return nil, err + } + kept := args.YNode().Content[:0] + for _, item := range args.YNode().Content { + if !strings.HasPrefix(item.Value, idpIssuerFlag+"=") { + kept = append(kept, item) + } + } + args.YNode().Content = append(kept, yaml.NewStringRNode(idpIssuerFlag+"="+issuer).YNode()) + var out bytes.Buffer + if err := (kio.ByteWriter{Writer: &out}).Write(nodes); err != nil { + return nil, fmt.Errorf("while writing manifest: %w", err) + } + return out.Bytes(), nil + } + return nil, errors.New("the manifest has no deployment/ate-idp-server") +} diff --git a/cmd/ate-setup/internal/steps/idpserver_test.go b/cmd/ate-setup/internal/steps/idpserver_test.go new file mode 100644 index 0000000000..fadaaee2b7 --- /dev/null +++ b/cmd/ate-setup/internal/steps/idpserver_test.go @@ -0,0 +1,104 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package steps + +import ( + "slices" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" +) + +// ate-idp-server has to publish under the issuer ate-api-server signs with, +// and the keys of the pool it signs from, or no relying party can verify a +// token. +func TestIDPServerGetsTheIssuerAndSharesTheAPIServerPool(t *testing.T) { + const issuer = "https://idp.example.com" + root := repoRoot(t) + for _, kind := range []bool{false, true} { + cfg := config.Config{Root: root, Router: config.RouterEnvoy, Kind: kind} + e := &Env{Cfg: &cfg} + rendered, err := e.render(e.Cfg.Path(SystemOverlay(e.Cfg))) + if err != nil { + t.Fatalf("kind=%t: render: %v", kind, err) + } + // Twice, so the second call has an --issuer to replace. + for range 2 { + if rendered, err = setIDPServerIssuer(rendered, issuer); err != nil { + t.Fatalf("kind=%t: setIDPServerIssuer: %v", kind, err) + } + } + objs, err := kube.DecodeManifestBytes(rendered) + if err != nil { + t.Fatalf("kind=%t: DecodeManifestBytes: %v", kind, err) + } + pods := map[string]corev1.PodSpec{} + for _, obj := range objs { + if obj.GetKind() != "Deployment" { + continue + } + var dep appsv1.Deployment + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &dep); err != nil { + t.Fatalf("kind=%t: deployment/%s: %v", kind, obj.GetName(), err) + } + pods[dep.Name] = dep.Spec.Template.Spec + } + api, ok := pods["ate-api-server"] + if !ok { + t.Fatalf("kind=%t: bundle has no deployment/ate-api-server", kind) + } + idp, ok := pods["ate-idp-server"] + if !ok { + t.Fatalf("kind=%t: bundle has no deployment/ate-idp-server", kind) + } + + args := idp.Containers[0].Args + issuerArgs := slices.DeleteFunc(slices.Clone(args), func(arg string) bool { return !strings.HasPrefix(arg, "--issuer=") }) + if diff := cmp.Diff([]string{"--issuer=" + issuer}, issuerArgs); diff != "" { + t.Errorf("kind=%t: ate-idp-server --issuer args (-want +got):\n%s", kind, diff) + } + if len(idp.Containers[0].Env) != 0 { + t.Errorf("kind=%t: ate-idp-server sets env %v, want its configuration in flags", kind, idp.Containers[0].Env) + } + + volume := func(spec corev1.PodSpec) *corev1.Volume { + i := slices.IndexFunc(spec.Volumes, func(v corev1.Volume) bool { return v.Name == "actor-id-jwt-pool" }) + if i < 0 { + return nil + } + return &spec.Volumes[i] + } + if volume(api) == nil { + t.Fatalf("kind=%t: ate-api-server has no actor-id-jwt-pool volume", kind) + } + if diff := cmp.Diff(volume(api), volume(idp)); diff != "" { + t.Errorf("kind=%t: actor-id-jwt-pool volume (-ate-api-server +ate-idp-server):\n%s", kind, diff) + } + } +} + +func TestSetIDPServerIssuerWithoutTheDeployment(t *testing.T) { + manifest := []byte("apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: ate-api-server\n") + if _, err := setIDPServerIssuer(manifest, "https://idp.example.com"); err == nil { + t.Error("setIDPServerIssuer succeeded on a manifest without deployment/ate-idp-server") + } +} diff --git a/cmd/ate-setup/internal/steps/overlay.go b/cmd/ate-setup/internal/steps/overlay.go index 46edf55c70..597d280d15 100644 --- a/cmd/ate-setup/internal/steps/overlay.go +++ b/cmd/ate-setup/internal/steps/overlay.go @@ -147,6 +147,9 @@ func (e *Env) renderSystemManifests(ctx context.Context) ([]byte, error) { if err != nil { return nil, err } + if manifest, err = setIDPServerIssuer(manifest, e.Cfg.ActorJWTIssuer); err != nil { + return nil, err + } if e.Cfg.Router != config.RouterAgentgateway { imageReference, err := e.dockerfileImage(ctx, envoyDataplaneImage, envoyDataplaneDockefile) if err != nil { diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index 98bb29d613..9f4b270918 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -421,19 +421,19 @@ func TestRenderCordonControlPlane(t *testing.T) { name: "base bundle", cfg: config.Config{Router: config.RouterEnvoy}, path: func(e *Env) string { return e.Cfg.Path(SystemOverlay(e.Cfg)) }, - want: []string{"ate-api-server", "ate-controller", "atenet-router"}, + want: []string{"ate-api-server", "ate-idp-server", "ate-controller", "atenet-router"}, }, { name: "kind bundle", cfg: config.Config{Router: config.RouterEnvoy, Kind: true}, path: func(e *Env) string { return e.Cfg.Path(SystemOverlay(e.Cfg)) }, - want: []string{"ate-api-server", "ate-controller", "atenet-router"}, + want: []string{"ate-api-server", "ate-idp-server", "ate-controller", "atenet-router"}, }, { name: "agentgateway bundle", cfg: config.Config{Router: config.RouterAgentgateway}, path: func(e *Env) string { return e.Cfg.Path(SystemOverlay(e.Cfg)) }, - want: []string{"ate-api-server", "ate-controller", "atenet-router"}, + want: []string{"ate-api-server", "ate-idp-server", "ate-controller", "atenet-router"}, }, { name: "api server file", diff --git a/cmd/ate-setup/internal/steps/substrate_test.go b/cmd/ate-setup/internal/steps/substrate_test.go index 13bc86c561..9ff7498ef9 100644 --- a/cmd/ate-setup/internal/steps/substrate_test.go +++ b/cmd/ate-setup/internal/steps/substrate_test.go @@ -43,11 +43,9 @@ func TestActorTemplateFromManifestDemos(t *testing.T) { // The values benchmarking/workloads/deploy.sh substitutes, beyond the // ${BUCKET_NAME} every installer provides. benchmarkValues := map[string]string{ - "${ACTOR_CPU}": "1", - "${ACTOR_MEMORY}": "256Mi", - "${OTLP_ENDPOINT}": "http://otel-collector.otel-system:4317", - "${SANDBOX_CLASS}": "gvisor", - "${SANDBOX_CONFIG_NAME}": "gvisor-default", + "${ACTOR_CPU}": "1", + "${ACTOR_MEMORY}": "256Mi", + "${OTLP_ENDPOINT}": "http://otel-collector.otel-system:4317", } tests := []struct { diff --git a/cmd/ateapi/helm_test.go b/cmd/ateapi/helm_test.go index 106750c52f..64bc7d96bf 100644 --- a/cmd/ateapi/helm_test.go +++ b/cmd/ateapi/helm_test.go @@ -239,7 +239,7 @@ func TestHelmObjectStorePlugins(t *testing.T) { for _, external := range []bool{false, true} { t.Run(fmt.Sprintf("external=%t", external), func(t *testing.T) { args := []string{"template", "test", "../../charts/substrate", "-n", "custom"} - issuer := "https://idp.custom.svc" + issuer := "https://test-idp.custom.svc" if external { issuer = "https://actors.example.com" args = append(args, "--set", "rustfs.enabled=false", "--set", "atelet.storageBackend=gcs", diff --git a/cmd/ateapi/internal/apivalidation/actor.go b/cmd/ateapi/internal/apivalidation/actor.go index 92c9f0c7b4..564d10840d 100644 --- a/cmd/ateapi/internal/apivalidation/actor.go +++ b/cmd/ateapi/internal/apivalidation/actor.go @@ -58,12 +58,6 @@ func ValidateDeleteActorRequest(ctx context.Context, req *ateapipb.DeleteActorRe return Validate_DeleteActorRequest(ctx, op, nil, req, nil) } -func ValidatePauseActorRequest(ctx context.Context, req *ateapipb.PauseActorRequest) field.ErrorList { - // Call the generated validation. - op := MakeCreateOp(ctx) - return Validate_PauseActorRequest(ctx, op, nil, req, nil) -} - func ValidateResumeActorRequest(ctx context.Context, req *ateapipb.ResumeActorRequest) field.ErrorList { // Call the generated validation. op := MakeCreateOp(ctx) diff --git a/cmd/ateapi/internal/apivalidation/actor_template.go b/cmd/ateapi/internal/apivalidation/actor_template.go index 7a8cced518..2b558e74b2 100644 --- a/cmd/ateapi/internal/apivalidation/actor_template.go +++ b/cmd/ateapi/internal/apivalidation/actor_template.go @@ -19,7 +19,6 @@ import ( "fmt" "go.ate.dev/substrate/internal/resources" - atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" "go.ate.dev/substrate/pkg/proto/ateapipb" "k8s.io/apimachinery/pkg/api/operation" "k8s.io/apimachinery/pkg/api/resource" @@ -59,18 +58,9 @@ func ValidateActorTemplateUpdate(ctx context.Context, fldPath *field.Path, newVa // ValidateCustom_CreateActorTemplateRequest_ActorTemplate holds the rules // that span more than one field of a template: container volume mounts must -// reference volumes the template declares, and a ROOTFS preferred fidelity -// needs a sandbox class that can capture rootfs changes without memory, which -// today is the micro-VM class alone. +// reference volumes the template declares. func ValidateCustom_CreateActorTemplateRequest_ActorTemplate(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *ateapipb.ActorTemplate) field.ErrorList { var errs field.ErrorList - if value.GetSnapshotConfig().GetPreferredFidelity() == ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS && - value.GetSandboxConfig().GetSandboxClass() != string(atev1alpha1.SandboxClassMicroVM) { - errs = append(errs, field.Invalid( - fldPath.Child("snapshot_config", "preferred_fidelity"), - value.GetSnapshotConfig().GetPreferredFidelity().String(), - "ROOTFS fidelity requires sandbox_config.sandbox_class microvm")) - } declared := make(map[string]bool, len(value.GetVolumes())) for _, vol := range value.GetVolumes() { declared[vol.GetName()] = true @@ -183,10 +173,6 @@ func ValidateCustom_ExternalVolumeTemplate_Capacity(_ context.Context, _ operati return nil } -func ValidateCustom_SandboxConfig_SandboxClass(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { - return resources.ValidateSandboxClass(fldPath, *value) -} - // ValidateCustom_SnapshotConfig_StorageLocation ensures an // ActorTemplate's snapshotConfig.location is a well-formed // URI with a bucket, so a bad location fails fast. diff --git a/cmd/ateapi/internal/apivalidation/actor_template_test.go b/cmd/ateapi/internal/apivalidation/actor_template_test.go index a0e1c883b6..7b4d963cee 100644 --- a/cmd/ateapi/internal/apivalidation/actor_template_test.go +++ b/cmd/ateapi/internal/apivalidation/actor_template_test.go @@ -70,19 +70,10 @@ func TestValidateCreateActorTemplateRequest(t *testing.T) { })}, nil, }, { - // Only the micro-VM runtime captures rootfs changes without memory; - // the fixture is a gVisor template. - "rootfs fidelity on gvisor rejected", + "valid rootfs fidelity", &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) { tmpl.SnapshotConfig.PreferredFidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS })}, - field.ErrorList{field.Invalid(field.NewPath("actor_template", "snapshot_config", "preferred_fidelity"), "SNAPSHOT_FIDELITY_ROOTFS", "")}, - }, { - "rootfs fidelity on microvm accepted", - &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) { - tmpl.SnapshotConfig.PreferredFidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS - tmpl.SandboxConfig = &ateapipb.SandboxConfig{SandboxClass: "microvm", ConfigName: "microvm-default"} - })}, nil, }, { "invalid worker_selector label key", @@ -161,24 +152,6 @@ func TestValidateCreateActorTemplateRequest(t *testing.T) { field.ErrorList{ field.Required(field.NewPath("actor_template", "snapshot_config", "preferred_fidelity"), ""), }, - }, { - "missing sandbox_config", - &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) { - tmpl.SandboxConfig = nil - })}, - field.ErrorList{field.Required(field.NewPath("actor_template", "sandbox_config"), "")}, - }, { - "missing sandbox_config.sandbox_class", - &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) { - tmpl.SandboxConfig.SandboxClass = "" - })}, - field.ErrorList{field.Required(field.NewPath("actor_template", "sandbox_config", "sandbox_class"), "")}, - }, { - "missing sandbox_config.config_name", - &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) { - tmpl.SandboxConfig.ConfigName = "" - })}, - field.ErrorList{field.Required(field.NewPath("actor_template", "sandbox_config", "config_name"), "")}, }, { "missing resources", &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) { @@ -385,40 +358,6 @@ func TestValidateActorTemplate(t *testing.T) { tmpl.WorkerSelector = &ateapipb.Selector{MatchLabels: map[string]string{"tier": "Not Valid"}} }, want: field.ErrorList{field.Invalid(field.NewPath("worker_selector", "match_labels").Key("tier"), nil, "").WithOrigin("format=k8s-label-value")}, - }, { - name: "missing sandbox_config", - mutate: func(tmpl *ateapipb.ActorTemplate) { tmpl.SandboxConfig = nil }, - want: field.ErrorList{field.Required(field.NewPath("sandbox_config"), "")}, - }, { - name: "missing sandbox_class", - mutate: func(tmpl *ateapipb.ActorTemplate) { tmpl.SandboxConfig.SandboxClass = "" }, - want: field.ErrorList{field.Required(field.NewPath("sandbox_config", "sandbox_class"), "")}, - }, { - name: "microvm sandbox_class", - mutate: func(tmpl *ateapipb.ActorTemplate) { tmpl.SandboxConfig.SandboxClass = "microvm" }, - }, { - name: "unsupported sandbox_class", - mutate: func(tmpl *ateapipb.ActorTemplate) { tmpl.SandboxConfig.SandboxClass = "kata" }, - want: field.ErrorList{field.NotSupported[string](field.NewPath("sandbox_config", "sandbox_class"), nil, nil)}, - }, { - name: "sandbox_class is case-sensitive", - mutate: func(tmpl *ateapipb.ActorTemplate) { tmpl.SandboxConfig.SandboxClass = "GVISOR" }, - want: field.ErrorList{field.NotSupported[string](field.NewPath("sandbox_config", "sandbox_class"), nil, nil)}, - }, { - name: "sandbox_class too long", - mutate: func(tmpl *ateapipb.ActorTemplate) { tmpl.SandboxConfig.SandboxClass = strings.Repeat("g", 64) }, - want: field.ErrorList{ - field.TooLong(field.NewPath("sandbox_config", "sandbox_class"), nil, 63).WithOrigin("maxLength"), - field.NotSupported[string](field.NewPath("sandbox_config", "sandbox_class"), nil, nil), - }, - }, { - name: "missing config_name", - mutate: func(tmpl *ateapipb.ActorTemplate) { tmpl.SandboxConfig.ConfigName = "" }, - want: field.ErrorList{field.Required(field.NewPath("sandbox_config", "config_name"), "")}, - }, { - name: "invalid config_name", - mutate: func(tmpl *ateapipb.ActorTemplate) { tmpl.SandboxConfig.ConfigName = "NOT_A_NAME" }, - want: field.ErrorList{field.Invalid(field.NewPath("sandbox_config", "config_name"), nil, "").WithOrigin("format=k8s-long-name")}, }, { name: "missing snapshot_config", mutate: func(tmpl *ateapipb.ActorTemplate) { tmpl.SnapshotConfig = nil }, @@ -965,8 +904,7 @@ func validActorTemplate(mutations ...func(*ateapipb.ActorTemplate)) *ateapipb.Ac StorageLocation: "gs://my-bucket/snapshots", PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, - SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor", ConfigName: "gvisor-default"}, - Resources: resources.CPUMemory(1000, 512<<20), + Resources: resources.CPUMemory(1000, 512<<20), } for _, m := range mutations { m(template) diff --git a/cmd/ateapi/internal/apivalidation/actor_test.go b/cmd/ateapi/internal/apivalidation/actor_test.go index f8540e86b5..0cc84108b4 100644 --- a/cmd/ateapi/internal/apivalidation/actor_test.go +++ b/cmd/ateapi/internal/apivalidation/actor_test.go @@ -895,43 +895,6 @@ func TestValidateDeleteActorRequest(t *testing.T) { } } -func TestValidatePauseActorRequest(t *testing.T) { - tests := []struct { - name string - req *ateapipb.PauseActorRequest - want field.ErrorList - }{{ - "valid", - &ateapipb.PauseActorRequest{Actor: &ateapipb.ObjectRef{Atespace: "ns1", Name: "id1"}}, - nil, - }, { - "missing actor", - &ateapipb.PauseActorRequest{}, - field.ErrorList{field.Required(field.NewPath("actor"), "")}, - }, { - "missing actor.atespace", - &ateapipb.PauseActorRequest{Actor: &ateapipb.ObjectRef{Name: "id1"}}, - field.ErrorList{field.Required(field.NewPath("actor", "atespace"), "")}, - }, { - "invalid actor.atespace", - &ateapipb.PauseActorRequest{Actor: &ateapipb.ObjectRef{Atespace: "NS1", Name: "id1"}}, - field.ErrorList{field.Invalid(field.NewPath("actor", "atespace"), "NS1", "").WithOrigin("format=k8s-short-name")}, - }, { - "missing actor.name", - &ateapipb.PauseActorRequest{Actor: &ateapipb.ObjectRef{Atespace: "ns1"}}, - field.ErrorList{field.Required(field.NewPath("actor", "name"), "")}, - }, { - "invalid actor.name", - &ateapipb.PauseActorRequest{Actor: &ateapipb.ObjectRef{Atespace: "ns1", Name: "ID1"}}, - field.ErrorList{field.Invalid(field.NewPath("actor", "name"), "ID1", "").WithOrigin("format=k8s-short-name")}, - }} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assertValidateErr(t, ValidatePauseActorRequest(context.Background(), tt.req), tt.want) - }) - } -} - func TestValidateResumeActorRequest(t *testing.T) { tests := []struct { name string @@ -998,6 +961,41 @@ func TestValidateSuspendActorRequest(t *testing.T) { "invalid actor.name", &ateapipb.SuspendActorRequest{Actor: &ateapipb.ObjectRef{Atespace: "ns1", Name: "ID1"}}, field.ErrorList{field.Invalid(field.NewPath("actor", "name"), "ID1", "").WithOrigin("format=k8s-short-name")}, + }, { + "synchronous mode", + &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: "ns1", Name: "id1"}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, + }, + nil, + }, { + "asynchronous mode", + &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: "ns1", Name: "id1"}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, + }, + nil, + }, { + "mode unspecified", + &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: "ns1", Name: "id1"}, + Options: &ateapipb.SuspendOptions{}, + }, + nil, + }, { + "mode negative", + &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: "ns1", Name: "id1"}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode(-1)}, + }, + field.ErrorList{field.Invalid(field.NewPath("options", "mode"), nil, "").WithOrigin("minimum")}, + }, { + "mode unknown", + &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: "ns1", Name: "id1"}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode(3)}, + }, + field.ErrorList{field.Invalid(field.NewPath("options", "mode"), nil, "").WithOrigin("maximum")}, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/cmd/ateapi/internal/apivalidation/validation_test.go b/cmd/ateapi/internal/apivalidation/validation_test.go index 6e1451850e..0b1d5e252c 100644 --- a/cmd/ateapi/internal/apivalidation/validation_test.go +++ b/cmd/ateapi/internal/apivalidation/validation_test.go @@ -1712,7 +1712,7 @@ func TestValidateTagRequestPayloads(t *testing.T) { func validGoldenSnapshotStatus(mutate ...func(*ateapipb.GoldenSnapshotStatus)) *ateapipb.GoldenSnapshotStatus { s := &ateapipb.GoldenSnapshotStatus{ - GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "01234567-89ab-cdef-0123-456789abcdef"}, + GoldenTag: &ateapipb.ObjectRef{Atespace: "team-a", Name: "01234567-89ab-cdef-0123-456789abcdef"}, SnapshotTime: ×tamppb.Timestamp{Seconds: 867}, } for _, m := range mutate { diff --git a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go index dd490a1a36..18e09da61c 100644 --- a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go +++ b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go @@ -1238,37 +1238,6 @@ func Validate_ActorTemplate( errs = append(errs, fn(fldPath.Child("snapshot_config"), obj.SnapshotConfig, oldVal, oldObj != nil)...) } - { // field ateapipb.ActorTemplate.SandboxConfig - fn := func( - fldPath *field.Path, - obj, oldObj *ateapipb.SandboxConfig, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if ateDeepEqual(obj, oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - // call the type's validation function - errs = append(errs, Validate_SandboxConfig(ctx, op, fldPath, obj, oldObj)...) - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ActorTemplate) *ateapipb.SandboxConfig { - return oldObj.SandboxConfig - }) - errs = append(errs, fn(fldPath.Child("sandbox_config"), obj.SandboxConfig, oldVal, oldObj != nil)...) - } - { // field ateapipb.ActorTemplate.Resources fn := func( fldPath *field.Path, @@ -6348,61 +6317,6 @@ func Validate_ObjectSnapshot( return errs } -// Validate_PauseActorRequest validates an instance of PauseActorRequest according -// to declarative validation rules in the API schema. -func Validate_PauseActorRequest( - ctx context.Context, op operation.Operation, fldPath *field.Path, - obj, oldObj *ateapipb.PauseActorRequest) (errs field.ErrorList) { - - { // field ateapipb.PauseActorRequest.Actor - fn := func( - fldPath *field.Path, - obj, oldObj *ateapipb.ObjectRef, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if ateDeepEqual(obj, oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - func() { // cohort = "atespace" - earlyReturn := false - if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", - func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.RequiredValue).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", - func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.OptionalValue).MarkShortCircuit(); len(e) != 0 { - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - }() - // call the type's validation function - errs = append(errs, Validate_ObjectRef(ctx, op, fldPath, obj, oldObj)...) - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.PauseActorRequest) *ateapipb.ObjectRef { - return oldObj.Actor - }) - errs = append(errs, fn(fldPath.Child("actor"), obj.Actor, oldVal, oldObj != nil)...) - } - - return errs -} - var unionMembershipFor_go_ate_dev_substrate_pkg_proto_ateapipb_Ports_ = validate.NewUnionMembership(validate.NewUnionMember("all"), validate.NewUnionMember("numbers")) // Validate_Ports validates an instance of Ports according @@ -7310,83 +7224,6 @@ func Validate_RevertActorRequest( return errs } -// Validate_SandboxConfig validates an instance of SandboxConfig according -// to declarative validation rules in the API schema. -func Validate_SandboxConfig( - ctx context.Context, op operation.Operation, fldPath *field.Path, - obj, oldObj *ateapipb.SandboxConfig) (errs field.ErrorList) { - - { // field ateapipb.SandboxConfig.SandboxClass - fn := func( - fldPath *field.Path, - obj, oldObj *string, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - // custom validation - if e := ValidateCustom_SandboxConfig_SandboxClass(ctx, op, fldPath, obj, oldObj); len(e) != 0 { - errs = append(errs, e...) - } - if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 63); len(e) != 0 { - errs = append(errs, e...) - } - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.SandboxConfig) *string { - return &oldObj.SandboxClass - }) - errs = append(errs, fn(fldPath.Child("sandbox_class"), &obj.SandboxClass, oldVal, oldObj != nil)...) - } - - { // field ateapipb.SandboxConfig.ConfigName - fn := func( - fldPath *field.Path, - obj, oldObj *string, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - if e := validate.LongName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { - errs = append(errs, e...) - } - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.SandboxConfig) *string { - return &oldObj.ConfigName - }) - errs = append(errs, fn(fldPath.Child("config_name"), &obj.ConfigName, oldVal, oldObj != nil)...) - } - - return errs -} - // Validate_SandboxRuntime validates an instance of SandboxRuntime according // to declarative validation rules in the API schema. func Validate_SandboxRuntime( @@ -8261,6 +8098,79 @@ func Validate_SuspendActorRequest( errs = append(errs, fn(fldPath.Child("actor"), obj.Actor, oldVal, oldObj != nil)...) } + { // field ateapipb.SuspendActorRequest.Options + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.SuspendOptions, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_SuspendOptions(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.SuspendActorRequest) *ateapipb.SuspendOptions { + return oldObj.Options + }) + errs = append(errs, fn(fldPath.Child("options"), obj.Options, oldVal, oldObj != nil)...) + } + + return errs +} + +// Validate_SuspendOptions validates an instance of SuspendOptions according +// to declarative validation rules in the API schema. +func Validate_SuspendOptions( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.SuspendOptions) (errs field.ErrorList) { + + { // field ateapipb.SuspendOptions.Mode + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.DurabilityMode, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 2); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.SuspendOptions) *ateapipb.DurabilityMode { + return &oldObj.Mode + }) + errs = append(errs, fn(fldPath.Child("mode"), &obj.Mode, oldVal, oldObj != nil)...) + } + return errs } diff --git a/cmd/ateapi/internal/authz/authz_util.go b/cmd/ateapi/internal/authz/authz_util.go index e4c86cc91e..a85729f4d7 100644 --- a/cmd/ateapi/internal/authz/authz_util.go +++ b/cmd/ateapi/internal/authz/authz_util.go @@ -44,10 +44,11 @@ const ( RoleEditor = "editor" RoleViewer = "viewer" - RoleController = "controller" - RoleEgressGateway = "egress_gateway" - RoleIngressRouter = "ingress_router" - RoleAtelet = "atelet" + RoleController = "controller" + RoleEgressGateway = "egress_gateway" + RoleIngressRouter = "ingress_router" + RoleAtelet = "atelet" + RoleAPIServerSidecar = "api_server_sidecar" RelationCanCreateAtespace = "can_create_atespace" RelationCanListAtespaces = "can_list_atespaces" @@ -61,7 +62,6 @@ const ( RelationCanUseTemplate = "can_use_template" RelationCanUpdate = "can_update" RelationCanDelete = "can_delete" - RelationCanPause = "can_pause" RelationCanResume = "can_resume" RelationCanSuspend = "can_suspend" RelationCanRevert = "can_revert" @@ -232,10 +232,11 @@ func EnsureStoreAndModel(ctx context.Context, pool *pgxpool.Pool, fgaServer *ser // The Authorizer grants each one its system component role on global:root on // every check, independent of any stored AccessPolicy. type SystemPrincipals struct { - ControllerSPIFFEID string - EgressGatewaySPIFFEID string - IngressRouterSPIFFEID string - AteletSPIFFEID string + ControllerSPIFFEID string + EgressGatewaySPIFFEID string + IngressRouterSPIFFEID string + AteletSPIFFEID string + APIServerSidecarSPIFFEID string } // New provisions the default OpenFGA store and authorization model via @@ -296,6 +297,7 @@ func parseSystemPrincipals(system SystemPrincipals) (map[string][]string, error) {role: RoleEgressGateway, spiffeID: system.EgressGatewaySPIFFEID}, {role: RoleIngressRouter, spiffeID: system.IngressRouterSPIFFEID}, {role: RoleAtelet, spiffeID: system.AteletSPIFFEID}, + {role: RoleAPIServerSidecar, spiffeID: system.APIServerSidecarSPIFFEID}, } out := make(map[string][]string, len(bindings)) for _, b := range bindings { diff --git a/cmd/ateapi/internal/authz/authz_util_test.go b/cmd/ateapi/internal/authz/authz_util_test.go index 7007d23675..77a13c26cf 100644 --- a/cmd/ateapi/internal/authz/authz_util_test.go +++ b/cmd/ateapi/internal/authz/authz_util_test.go @@ -25,10 +25,11 @@ import ( func testSystemPrincipals() SystemPrincipals { return SystemPrincipals{ - ControllerSPIFFEID: installdefaults.ControllerSPIFFEID(installdefaults.SystemNamespace), - EgressGatewaySPIFFEID: installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace), - IngressRouterSPIFFEID: installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), - AteletSPIFFEID: installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), + ControllerSPIFFEID: installdefaults.ControllerSPIFFEID(installdefaults.SystemNamespace), + EgressGatewaySPIFFEID: installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace), + IngressRouterSPIFFEID: installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), + AteletSPIFFEID: installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), + APIServerSidecarSPIFFEID: installdefaults.APIServerSPIFFEID(installdefaults.SystemNamespace), } } @@ -94,10 +95,11 @@ func TestParseBootstrapOwners(t *testing.T) { func TestParseSystemPrincipals(t *testing.T) { const router = "spiffe://cluster.local/ns/ate-system/sa/atenet-router" got, err := parseSystemPrincipals(SystemPrincipals{ - ControllerSPIFFEID: "spiffe://cluster.local/ns/ate-system/sa/ate-controller", - EgressGatewaySPIFFEID: router, - IngressRouterSPIFFEID: router, - AteletSPIFFEID: "spiffe://cluster.local/ns/ate-system/sa/atelet", + ControllerSPIFFEID: "spiffe://cluster.local/ns/ate-system/sa/ate-controller", + EgressGatewaySPIFFEID: router, + IngressRouterSPIFFEID: router, + AteletSPIFFEID: "spiffe://cluster.local/ns/ate-system/sa/atelet", + APIServerSidecarSPIFFEID: "spiffe://cluster.local/ns/ate-system/sa/ate-api-server", }) if err != nil { t.Fatalf("parseSystemPrincipals failed: %v", err) @@ -106,6 +108,7 @@ func TestParseSystemPrincipals(t *testing.T) { "user:spiffe%3A//cluster.local/ns/ate-system/sa/ate-controller": {RoleController}, "user:spiffe%3A//cluster.local/ns/ate-system/sa/atenet-router": {RoleEgressGateway, RoleIngressRouter}, "user:spiffe%3A//cluster.local/ns/ate-system/sa/atelet": {RoleAtelet}, + "user:spiffe%3A//cluster.local/ns/ate-system/sa/ate-api-server": {RoleAPIServerSidecar}, } if diff := cmp.Diff(want, got); diff != "" { t.Errorf("parseSystemPrincipals (-want +got):\n%s", diff) @@ -123,6 +126,7 @@ func TestNewRejectsInvalidSystemPrincipals(t *testing.T) { {"empty egress gateway SPIFFE ID", func(p *SystemPrincipals) { p.EgressGatewaySPIFFEID = "" }, "invalid egress_gateway SPIFFE ID"}, {"empty ingress router SPIFFE ID", func(p *SystemPrincipals) { p.IngressRouterSPIFFEID = "" }, "invalid ingress_router SPIFFE ID"}, {"empty atelet SPIFFE ID", func(p *SystemPrincipals) { p.AteletSPIFFEID = "" }, "invalid atelet SPIFFE ID"}, + {"empty api server sidecar SPIFFE ID", func(p *SystemPrincipals) { p.APIServerSidecarSPIFFEID = "" }, "invalid api_server_sidecar SPIFFE ID"}, {"bare ServiceAccount name", func(p *SystemPrincipals) { p.ControllerSPIFFEID = "ate-controller" }, "invalid controller SPIFFE ID"}, {"non-spiffe URI", func(p *SystemPrincipals) { p.EgressGatewaySPIFFEID = "https://cluster.local/ns/ate-system/sa/atenet-egress" diff --git a/cmd/ateapi/internal/authz/interceptor_test.go b/cmd/ateapi/internal/authz/interceptor_test.go index 4bab436a23..52df3035c7 100644 --- a/cmd/ateapi/internal/authz/interceptor_test.go +++ b/cmd/ateapi/internal/authz/interceptor_test.go @@ -511,7 +511,6 @@ func TestUnaryServerInterceptor_LifecycleEgressWorkerAndTagChecks(t *testing.T) } actor := &ateapipb.ObjectRef{Atespace: "team-a", Name: "runner"} worker := &ateapipb.ObjectRef{Name: "w-1"} - pause := rpcCall{ateapipb.Control_PauseActor_FullMethodName, &ateapipb.PauseActorRequest{Actor: actor}} resume := rpcCall{ateapipb.Control_ResumeActor_FullMethodName, &ateapipb.ResumeActorRequest{Actor: actor}} suspend := rpcCall{ateapipb.Control_SuspendActor_FullMethodName, &ateapipb.SuspendActorRequest{Actor: actor}} revert := rpcCall{ateapipb.Control_RevertActor_FullMethodName, &ateapipb.RevertActorRequest{Actor: actor}} @@ -560,7 +559,6 @@ func TestUnaryServerInterceptor_LifecycleEgressWorkerAndTagChecks(t *testing.T) } cloneInTeamA := cloneTag("team-a", tag) cloneInTeamB := cloneTag("team-b", tag) - cloneGolden := cloneTag("team-a", &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden-v1"}) createWithoutTag := cloneTag("team-a", nil) tests := []struct { @@ -570,17 +568,15 @@ func TestUnaryServerInterceptor_LifecycleEgressWorkerAndTagChecks(t *testing.T) wantCode codes.Code }{ // Lifecycle operations need editor on the actor's atespace. - {"editor pauses actor", "editor", pause, codes.OK}, {"editor resumes actor", "editor", resume, codes.OK}, {"editor suspends actor", "editor", suspend, codes.OK}, {"editor reverts actor", "editor", revert, codes.OK}, - {"viewer cannot pause actor", "viewer", pause, codes.PermissionDenied}, {"viewer cannot resume actor", "viewer", resume, codes.PermissionDenied}, {"viewer cannot suspend actor", "viewer", suspend, codes.PermissionDenied}, {"viewer cannot revert actor", "viewer", revert, codes.PermissionDenied}, {"editor of another atespace cannot resume actor", "outsider", resume, codes.PermissionDenied}, {"global viewer cannot suspend actor", "global-viewer", suspend, codes.PermissionDenied}, - {"global owner pauses actor", "global-owner", pause, codes.OK}, + {"global owner suspends actor", "global-owner", suspend, codes.OK}, {"bootstrap owner resumes actor", "bootstrap-owner", resume, codes.OK}, // Egress policies: viewers read, editors write. @@ -641,14 +637,13 @@ func TestUnaryServerInterceptor_LifecycleEgressWorkerAndTagChecks(t *testing.T) {"bootstrap owner deletes tag", "bootstrap-owner", deleteTag, codes.OK}, // Cloning from a tag needs editor on the tag's atespace, whatever its - // scope; golden tags and actors created without a tag are unaffected. + // scope. An actor created without an explicit tag only needs its template. {"editor clones tag", "editor", cloneInTeamA, codes.OK}, {"viewer cannot clone tag", "viewer", cloneInTeamA, codes.PermissionDenied}, {"editor of another atespace cannot clone tag", "outsider", cloneInTeamB, codes.PermissionDenied}, {"viewer of tag's atespace cannot clone tag into another atespace", "outsider-with-viewer", cloneInTeamB, codes.PermissionDenied}, {"global owner clones tag into another atespace", "global-owner", cloneInTeamB, codes.OK}, {"bootstrap owner clones tag", "bootstrap-owner", cloneInTeamA, codes.OK}, - {"editor clones golden tag", "editor", cloneGolden, codes.OK}, {"editor creates actor without a tag", "editor", createWithoutTag, codes.OK}, // Only the egress gateway mints actor credentials; not even owners. @@ -689,13 +684,15 @@ func TestUnaryServerInterceptor_SystemPrincipals(t *testing.T) { egressID = "spiffe://cluster.local/ns/ate-system/sa/atenet-egress" routerID = "spiffe://cluster.local/ns/ate-system/sa/atenet-router" ateletID = "spiffe://cluster.local/ns/ate-system/sa/atelet" + pluginID = "spiffe://cluster.local/ns/ate-system/sa/ate-api-server" otherID = "spiffe://cluster.local/ns/ate-system/sa/other" ) authorizer, _, err := New(ctx, pool, fgaServer, nil, SystemPrincipals{ - ControllerSPIFFEID: controllerID, - EgressGatewaySPIFFEID: egressID, - IngressRouterSPIFFEID: routerID, - AteletSPIFFEID: ateletID, + ControllerSPIFFEID: controllerID, + EgressGatewaySPIFFEID: egressID, + IngressRouterSPIFFEID: routerID, + AteletSPIFFEID: ateletID, + APIServerSidecarSPIFFEID: pluginID, }) if err != nil { t.Fatalf("New failed: %v", err) @@ -711,7 +708,6 @@ func TestUnaryServerInterceptor_SystemPrincipals(t *testing.T) { deleteActor := rpcCall{ateapipb.Control_DeleteActor_FullMethodName, &ateapipb.DeleteActorRequest{Actor: actor}} listAllActors := rpcCall{ateapipb.Control_ListActors_FullMethodName, &ateapipb.ListActorsRequest{}} listAtespaces := rpcCall{ateapipb.Control_ListAtespaces_FullMethodName, &ateapipb.ListAtespacesRequest{}} - pause := rpcCall{ateapipb.Control_PauseActor_FullMethodName, &ateapipb.PauseActorRequest{Actor: actor}} resume := rpcCall{ateapipb.Control_ResumeActor_FullMethodName, &ateapipb.ResumeActorRequest{Actor: actor}} suspend := rpcCall{ateapipb.Control_SuspendActor_FullMethodName, &ateapipb.SuspendActorRequest{Actor: actor}} getEgress := rpcCall{ateapipb.Control_GetActorEgressPolicy_FullMethodName, &ateapipb.GetActorEgressPolicyRequest{Actor: actor}} @@ -773,7 +769,6 @@ func TestUnaryServerInterceptor_SystemPrincipals(t *testing.T) { {"ingress router resumes actor", mtls(routerID), resume, codes.OK}, {"ingress router lists actors", mtls(routerID), listAllActors, codes.OK}, {"ingress router cannot get actor", mtls(routerID), getActor, codes.PermissionDenied}, - {"ingress router cannot pause actor", mtls(routerID), pause, codes.PermissionDenied}, {"ingress router cannot suspend actor", mtls(routerID), suspend, codes.PermissionDenied}, {"ingress router cannot delete actor", mtls(routerID), deleteActor, codes.PermissionDenied}, {"ingress router cannot get egress policy", mtls(routerID), getEgress, codes.PermissionDenied}, @@ -790,6 +785,15 @@ func TestUnaryServerInterceptor_SystemPrincipals(t *testing.T) { {"atelet cannot resume actor", mtls(ateletID), resume, codes.PermissionDenied}, {"atelet cannot mint actor JWT", mtls(ateletID), mintJWT, codes.PermissionDenied}, {"atelet cannot get worker", mtls(ateletID), getWorker, codes.PermissionDenied}, + // The objectstore control plugin reads actors to scope their storage + // credentials, and nothing else. + {"api server sidecar gets actor", mtls(pluginID), getActor, codes.OK}, + {"api server sidecar cannot delete actor", mtls(pluginID), deleteActor, codes.PermissionDenied}, + {"api server sidecar cannot list actors", mtls(pluginID), listAllActors, codes.PermissionDenied}, + {"api server sidecar cannot get egress policy", mtls(pluginID), getEgress, codes.PermissionDenied}, + {"api server sidecar cannot resume actor", mtls(pluginID), resume, codes.PermissionDenied}, + {"api server sidecar cannot mint actor JWT", mtls(pluginID), mintJWT, codes.PermissionDenied}, + {"api server sidecar cannot get worker", mtls(pluginID), getWorker, codes.PermissionDenied}, // A bearer token whose subject is a granted SPIFFE ID gets nothing. {"jwt with controller ID cannot create worker", jwt(controllerID), createWorker, codes.PermissionDenied}, @@ -797,6 +801,7 @@ func TestUnaryServerInterceptor_SystemPrincipals(t *testing.T) { {"jwt with egress ID cannot mint actor JWT", jwt(egressID), mintJWT, codes.PermissionDenied}, {"jwt with router ID cannot resume actor", jwt(routerID), resume, codes.PermissionDenied}, {"jwt with atelet ID cannot get egress policy", jwt(ateletID), getEgress, codes.PermissionDenied}, + {"jwt with api server sidecar ID cannot get actor", jwt(pluginID), getActor, codes.PermissionDenied}, // Other SPIFFE IDs get nothing. {"other mTLS identity cannot resume actor", mtls(otherID), resume, codes.PermissionDenied}, diff --git a/cmd/ateapi/internal/authz/model.fga b/cmd/ateapi/internal/authz/model.fga index ea41388938..1d42a82e3e 100644 --- a/cmd/ateapi/internal/authz/model.fga +++ b/cmd/ateapi/internal/authz/model.fga @@ -51,6 +51,7 @@ type global define egress_gateway: [user] define ingress_router: [user] define atelet: [user] + define api_server_sidecar: [user] define can_create_atespace: owner define can_list_atespaces: viewer @@ -83,6 +84,7 @@ type atespace define egress_gateway: egress_gateway from parent_global define ingress_router: ingress_router from parent_global define atelet: atelet from parent_global + define api_server_sidecar: api_server_sidecar from parent_global # --- 2. Policy Governance --- define can_create_access_policy: owner @@ -139,8 +141,11 @@ type actor define host_node: host_node from scheduled_worker # --- User-Facing Operations (Inherited from Atespace) --- - # atenet-egress reads the actor to resolve its egress policy. - define can_get: viewer from parent_atespace or egress_gateway from parent_atespace + # atenet-egress reads the actor to resolve its egress policy. The + # ate-api-server pod's sidecars, today the objectstore control plugin, + # read it to derive the snapshot prefixes a per-actor storage credential + # may cover, and nothing else. + define can_get: viewer from parent_atespace or egress_gateway from parent_atespace or api_server_sidecar from parent_atespace define can_update: editor from parent_atespace define can_delete: editor from parent_atespace @@ -170,6 +175,5 @@ type actor define can_resume: [user] or editor from parent_atespace or ingress_router from parent_atespace # Lifecycle Operations - define can_pause: [user] or editor from parent_atespace define can_suspend: [user] or editor from parent_atespace define can_revert: [user] or editor from parent_atespace diff --git a/cmd/ateapi/internal/authz/model_test.fga.yaml b/cmd/ateapi/internal/authz/model_test.fga.yaml index e2f945c27a..02741ca1db 100644 --- a/cmd/ateapi/internal/authz/model_test.fga.yaml +++ b/cmd/ateapi/internal/authz/model_test.fga.yaml @@ -90,6 +90,9 @@ tuples: - user: user:sys-atelet relation: atelet object: global:root + - user: user:sys-api-server-sidecar + relation: api_server_sidecar + object: global:root tests: - name: 1. Global Superadmin & Viewer Propagation @@ -266,7 +269,6 @@ tests: can_get: true can_update: true can_delete: true - can_pause: true can_resume: true can_suspend: true can_revert: true @@ -280,7 +282,6 @@ tests: can_get: true can_update: true can_delete: true - can_pause: true can_resume: true can_suspend: true can_revert: true @@ -294,7 +295,6 @@ tests: can_get: true can_update: false can_delete: false - can_pause: false can_resume: false can_suspend: false can_revert: false @@ -308,7 +308,6 @@ tests: can_get: true can_update: false can_delete: false - can_pause: false can_resume: false can_suspend: false can_revert: false @@ -320,14 +319,12 @@ tests: object: actor:runner-1 assertions: can_get: false - can_pause: false can_get_egress_policy: false can_update_egress_policy: false - user: user:sa-router # Ingress router proxy can only resume object: actor:runner-1 assertions: can_resume: true - can_pause: false can_get_egress_policy: false - user: user:sa-runner-1 # Workload identity can suspend and revert object: actor:runner-1 @@ -411,7 +408,6 @@ tests: can_update_egress_policy: false can_delete_egress_policy: false can_resume: false - can_pause: false - user: user:sys-egress # lists actors as its ateapi health check object: global:root assertions: @@ -450,12 +446,35 @@ tests: can_list_actors: false can_list_workers: false can_create_worker: false + - user: user:sys-api-server-sidecar # the control plugin reads actors to scope their storage credentials + object: actor:runner-1 + assertions: + can_get: true + can_update: false + can_delete: false + can_get_egress_policy: false + can_resume: false + can_suspend: false + can_mint_actor_jwt: false + can_mint_actor_certificate: false + - user: user:sys-api-server-sidecar + object: atespace:space-1 + assertions: + viewer: false + can_get: false + can_list_actors: false + can_create_actor: false + - user: user:sys-api-server-sidecar + object: global:root + assertions: + can_list_actors: false + can_list_workers: false + can_list_atespaces: false - user: user:sys-router # atenet-router resumes actors and lists them object: actor:runner-1 assertions: can_resume: true can_get: false - can_pause: false can_suspend: false can_revert: false can_delete: false @@ -476,6 +495,7 @@ tests: egress_gateway: false ingress_router: false atelet: false + api_server_sidecar: false - user: user:carol # Global owner does not hold system roles object: global:root assertions: @@ -483,6 +503,7 @@ tests: egress_gateway: false ingress_router: false atelet: false + api_server_sidecar: false - name: 8. Tag Operations (Inherited from Atespace) check: diff --git a/cmd/ateapi/internal/authz/registry.go b/cmd/ateapi/internal/authz/registry.go index c22d81a810..4acacf82cf 100644 --- a/cmd/ateapi/internal/authz/registry.go +++ b/cmd/ateapi/internal/authz/registry.go @@ -124,10 +124,9 @@ func onTag(relation string, ref resourceRef, fldPath *field.Path) ([]check, fiel } // onSourceTag checks can_use_tag on the tag an actor is created from, if the -// request names one. Golden tags are exempt: the handler only accepts one -// built from the actor's own template, which can_use_template already covers. +// request names one. func onSourceTag(ref *ateapipb.ObjectRef, fldPath *field.Path) ([]check, field.ErrorList) { - if ref == nil || ref.GetAtespace() == resources.GoldenActorAtespace { + if ref == nil { return nil, nil } return onTag(RelationCanUseTag, ref, fldPath) @@ -250,9 +249,6 @@ var defaultRPCPermissions = map[string]rpcRule{ })), // Actor lifecycle. - ateapipb.Control_PauseActor_FullMethodName: rule(checksOf(func(r *ateapipb.PauseActorRequest) ([]check, field.ErrorList) { - return onActor(RelationCanPause, r.GetActor(), field.NewPath("actor")) - })), ateapipb.Control_ResumeActor_FullMethodName: rule(checksOf(func(r *ateapipb.ResumeActorRequest) ([]check, field.ErrorList) { return onActor(RelationCanResume, r.GetActor(), field.NewPath("actor")) })), diff --git a/cmd/ateapi/internal/authz/registry_test.go b/cmd/ateapi/internal/authz/registry_test.go index b042904319..05d4e26386 100644 --- a/cmd/ateapi/internal/authz/registry_test.go +++ b/cmd/ateapi/internal/authz/registry_test.go @@ -123,16 +123,17 @@ func TestDefaultRPCPermissions(t *testing.T) { }, }, { - name: "CreateActor from a golden tag does not check the tag", + name: "CreateActor from a template tag checks the tag", fullMethod: ateapipb.Control_CreateActor_FullMethodName, req: &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ Metadata: actorMeta, ActorTemplate: sharedTemplate, - SourceTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "0b5c7d4e-golden"}, + SourceTag: &ateapipb.ObjectRef{Atespace: "shared", Name: "0b5c7d4e-golden"}, }}, want: []check{ {RelationCanCreateActor, "atespace:team-a"}, {RelationCanUseTemplate, "actor_template:shared/tmpl"}, + {RelationCanUseTag, "tag:shared/0b5c7d4e-golden"}, }, }, { @@ -170,12 +171,6 @@ func TestDefaultRPCPermissions(t *testing.T) { }, // Actor lifecycle. - { - name: "PauseActor", - fullMethod: ateapipb.Control_PauseActor_FullMethodName, - req: &ateapipb.PauseActorRequest{Actor: actorRef}, - want: []check{{RelationCanPause, "actor:team-a/runner"}}, - }, { name: "ResumeActor", fullMethod: ateapipb.Control_ResumeActor_FullMethodName, @@ -513,9 +508,9 @@ func TestDefaultRPCPermissions_MalformedIdentifierFailsClosed(t *testing.T) { wantField: "actor.atespace", }, { - name: "PauseActor without an actor", - fullMethod: ateapipb.Control_PauseActor_FullMethodName, - req: &ateapipb.PauseActorRequest{}, + name: "SuspendActor without an actor", + fullMethod: ateapipb.Control_SuspendActor_FullMethodName, + req: &ateapipb.SuspendActorRequest{}, wantField: "actor.atespace", }, { diff --git a/cmd/ateapi/internal/controlapi/access_policy.go b/cmd/ateapi/internal/controlapi/access_policy.go index f83c5a3940..226bbd7578 100644 --- a/cmd/ateapi/internal/controlapi/access_policy.go +++ b/cmd/ateapi/internal/controlapi/access_policy.go @@ -81,10 +81,8 @@ func (s *ServiceImpl) UpdateGlobalAccessPolicy(ctx context.Context, precondition if err := mutate(toUpdate); err != nil { return err } - if errs := apivalidation.ValidateGlobalAccessPolicyUpdate(ctx, field.NewPath("access_policy"), toUpdate, oldVal); len(errs) > 0 { - return resources.ToAPIError(errs) - } - return nil + errs := apivalidation.ValidateGlobalAccessPolicyUpdate(ctx, field.NewPath("access_policy"), toUpdate, oldVal) + return resources.ToAPIError(errs) }) return mapAccessPolicyWrite(updated, err) } @@ -141,10 +139,8 @@ func (s *ServiceImpl) UpdateAtespaceAccessPolicy(ctx context.Context, name strin if err := mutate(toUpdate); err != nil { return err } - if errs := apivalidation.ValidateAtespaceAccessPolicyUpdate(ctx, field.NewPath("access_policy"), toUpdate, oldVal); len(errs) > 0 { - return resources.ToAPIError(errs) - } - return nil + errs := apivalidation.ValidateAtespaceAccessPolicyUpdate(ctx, field.NewPath("access_policy"), toUpdate, oldVal) + return resources.ToAPIError(errs) }) return mapAccessPolicyWrite(updated, err) } diff --git a/cmd/ateapi/internal/controlapi/access_policy_test.go b/cmd/ateapi/internal/controlapi/access_policy_test.go index ec3fcc1348..b0421c4fae 100644 --- a/cmd/ateapi/internal/controlapi/access_policy_test.go +++ b/cmd/ateapi/internal/controlapi/access_policy_test.go @@ -48,7 +48,7 @@ func TestAccessPolicy_GlobalAndAtespaceGovernance(t *testing.T) { } persistence.SetPolicyManager(policyManager) - svc := NewRPCService(persistence, nil, nil, nil, nil, nil, nil, "", 0, nil, nil, "", nil, nil) + svc := NewRPCService(persistence, nil, nil, nil, nil, nil, nil, nil, "", 0, nil, nil, "", nil, nil) interceptor := authz.UnaryServerInterceptor(authorizer, true) userCtx := func(id string) context.Context { diff --git a/cmd/ateapi/internal/controlapi/actor.go b/cmd/ateapi/internal/controlapi/actor.go index 9b0ad3f727..42ed43295e 100644 --- a/cmd/ateapi/internal/controlapi/actor.go +++ b/cmd/ateapi/internal/controlapi/actor.go @@ -108,11 +108,6 @@ func (s *ServiceImpl) CreateActor(ctx context.Context, inActor *ateapipb.Actor) if err != nil { return nil, err } - if inActor.GetSourceTag() == nil { - if err := validateGoldenSnapshotFidelity(sourceTag.GetStatus().GetSnapshot()); err != nil { - return nil, err - } - } } atespace := inActor.GetMetadata().GetAtespace() @@ -302,19 +297,20 @@ func (s *ServiceImpl) UpdateActor(ctx context.Context, actorRef resources.ActorR // Update actor template is only allowed while the actor is suspended. // The repointed ref must also resolve, mirroring CreateActor's // check (same non-atomicity caveat; resume re-resolves and fails - // cleanly), and the replacement's sandbox config, volumes, and - // volume mounts must match the old template's. It must also store - // snapshots under the location the actor's own already live in. + // cleanly), and the replacement's volumes and volume mounts must match + // the old template's. It must also store snapshots under the location + // the actor's own already live in. if !proto.Equal(oldVal.GetActorTemplate(), newVal.GetActorTemplate()) { if state := oldVal.GetStatus().GetState(); state != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { return apierror.FailedPrecondition( "actor must be %s to change its actor template (got: %s)", ateapipb.ActorState_ACTOR_STATE_SUSPENDED, state) } - // A paused actor's latest snapshot is node-local and was captured - // from the old template's sandbox; suspend or revert it first. - if latestSnapshotIsLocalOnly(oldVal.GetStatus()) { + // A paused actor resumes from its node-local snapshot, uploaded or + // not, which was captured from the old template's sandbox. The + // snapshot stays local until atelet detaches it from the node. + if latestSnapshotIsLocal(oldVal.GetStatus()) { return apierror.FailedPrecondition( - "actor must hold no node-local snapshot to change its actor template: suspend or revert it first") + "actor must hold no node-local snapshot to change its actor template: wait for its snapshot to be uploaded and detached from the node, or revert it") } newTemplate, err := resolveActorTemplate(ctx, s.store, newVal) if err != nil { @@ -326,21 +322,12 @@ func (s *ServiceImpl) UpdateActor(ctx context.Context, actorRef resources.ActorR oldTemplate, err := resolveActorTemplate(ctx, s.store, oldVal) switch { case err == nil: - // Snapshots are not portable across sandbox runtime - // families, so the replacement template must name the same - // SandboxConfig. - if !proto.Equal(oldTemplate.GetSandboxConfig(), newTemplate.GetSandboxConfig()) { - oldSC, newSC := oldTemplate.GetSandboxConfig(), newTemplate.GetSandboxConfig() - return apierror.FailedPrecondition( - "the current actor template names SandboxConfig %q (class %s) but the new one names %q (class %s); the sandbox config must be identical to repoint an actor", - oldSC.GetConfigName(), oldSC.GetSandboxClass(), newSC.GetConfigName(), newSC.GetSandboxClass()) - } if err := validateTemplateVolumesUnchanged(oldTemplate, newTemplate); err != nil { return err } case errors.Is(err, errActorTemplateNotFound): // The old template is gone, so there is nothing left to - // compare the sandbox config or volume layout against. + // compare the volume layout against. default: return err } @@ -473,28 +460,6 @@ func (s *ServiceImpl) DeleteActor(ctx context.Context, actorRef resources.ActorR return s.store.DeleteActor(ctx, actorRef, precondition) } -func (s *RPCService) PauseActor(ctx context.Context, req *ateapipb.PauseActorRequest) (*ateapipb.PauseActorResponse, error) { - if errs := apivalidation.ValidatePauseActorRequest(ctx, req); len(errs) > 0 { - return nil, resources.ToAPIError(errs) - } - actorRef := resources.ActorRefFromObjectRef(req.GetActor()) - setSpanActorRefAttributes(ctx, actorRef) - - actor, err := s.actorWorkflow.PauseActor(ctx, actorRef) - if err != nil { - if errors.Is(err, store.ErrVersionConflict) { - return nil, apierror.Aborted("concurrent update conflict, please retry") - } - if errors.Is(err, store.ErrNotFound) { - return nil, apierror.NotFound("Actor %s not found", actorRef) - } - return nil, err - } - - setSpanActorAttributes(ctx, actor) - return &ateapipb.PauseActorResponse{Actor: actor}, nil -} - func (s *RPCService) ResumeActor(ctx context.Context, req *ateapipb.ResumeActorRequest) (*ateapipb.ResumeActorResponse, error) { if errs := apivalidation.ValidateResumeActorRequest(ctx, req); len(errs) > 0 { return nil, resources.ToAPIError(errs) @@ -524,7 +489,7 @@ func (s *RPCService) SuspendActor(ctx context.Context, req *ateapipb.SuspendActo actorRef := resources.ActorRefFromObjectRef(req.GetActor()) setSpanActorRefAttributes(ctx, actorRef) - actor, err := s.actorWorkflow.SuspendActor(ctx, actorRef) + actor, err := s.actorWorkflow.SuspendActor(ctx, actorRef, suspendModeFromOptions(req.GetOptions())) if err != nil { if errors.Is(err, store.ErrVersionConflict) { return nil, apierror.Aborted("concurrent update conflict, please retry") @@ -561,7 +526,7 @@ func (s *RPCService) RevertActor(ctx context.Context, req *ateapipb.RevertActorR func (s *RPCService) MintActorJwt(ctx context.Context, req *ateapipb.MintActorJwtRequest) (*ateapipb.MintActorJwtResponse, error) { if errs := apivalidation.ValidateMintActorJWTRequest(ctx, req); len(errs) > 0 { - return nil, apierror.InvalidArgument("%v", errs.ToAggregate()) + return nil, resources.ToAPIError(errs) } // With authz enforced, the interceptor requires can_mint_actor_jwt on the @@ -615,7 +580,7 @@ func (s *RPCService) MintActorJwt(ctx context.Context, req *ateapipb.MintActorJw func (s *RPCService) MintActorCertificate(ctx context.Context, req *ateapipb.MintActorCertificateRequest) (*ateapipb.MintActorCertificateResponse, error) { if errs := apivalidation.ValidateMintActorCertificateRequest(ctx, req); len(errs) > 0 { - return nil, apierror.InvalidArgument("%v", errs.ToAggregate()) + return nil, resources.ToAPIError(errs) } // With authz enforced, the interceptor requires can_mint_actor_certificate diff --git a/cmd/ateapi/internal/controlapi/actor_template.go b/cmd/ateapi/internal/controlapi/actor_template.go index d362038038..b9556f92f4 100644 --- a/cmd/ateapi/internal/controlapi/actor_template.go +++ b/cmd/ateapi/internal/controlapi/actor_template.go @@ -44,12 +44,6 @@ func (s *RPCService) CreateActorTemplate(ctx context.Context, req *ateapipb.Crea return nil, resources.ToAPIError(errs) } - // config_name is required; the declarative validation has already - // rejected an empty one. - if _, err := resolveTemplateSandboxConfig(s.sandboxConfigLister, in.GetSandboxConfig()); err != nil { - return nil, err - } - templateRef := resources.ActorTemplateRefFromActorTemplate(in) stored, err := s.impl.CreateActorTemplate(ctx, in) diff --git a/cmd/ateapi/internal/controlapi/actor_template_test.go b/cmd/ateapi/internal/controlapi/actor_template_test.go index b6028de4d3..b848388504 100644 --- a/cmd/ateapi/internal/controlapi/actor_template_test.go +++ b/cmd/ateapi/internal/controlapi/actor_template_test.go @@ -17,7 +17,6 @@ package controlapi import ( "context" "errors" - "fmt" "strings" "testing" @@ -27,12 +26,9 @@ import ( "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/objectstoreplugin/objectstoreplugintest" "go.ate.dev/substrate/internal/resources" - atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" - listersv1alpha1 "go.ate.dev/substrate/pkg/client/listers/api/v1alpha1" "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/protobuf/testing/protocmp" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // validActorTemplate returns the smallest template that passes create @@ -47,8 +43,7 @@ func validActorTemplate(mutations ...func(*ateapipb.ActorTemplate)) *ateapipb.Ac StorageLocation: "gs://my-bucket/snapshots", PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, - SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor", ConfigName: "gvisor-default"}, - Resources: resources.CPUMemory(1000, 512<<20), + Resources: resources.CPUMemory(1000, 512<<20), } for _, m := range mutations { m(template) @@ -56,76 +51,11 @@ func validActorTemplate(mutations ...func(*ateapipb.ActorTemplate)) *ateapipb.Ac return template } -// gvisorDefaultLister returns a SandboxConfig lister seeded with the -// "gvisor-default" config that validActorTemplate names. -func gvisorDefaultLister(t *testing.T) listersv1alpha1.SandboxConfigLister { - t.Helper() - return sandboxConfigListerFor(t, []*atev1alpha1.SandboxConfig{{ - ObjectMeta: metav1.ObjectMeta{Name: "gvisor-default"}, - Spec: atev1alpha1.SandboxConfigSpec{ - SandboxClass: atev1alpha1.SandboxClassGvisor, - DefaultVersion: "v1", - Versions: []atev1alpha1.SandboxVersionConfig{{ - Name: "v1", - PauseImage: "registry.k8s.io/pause@sha256:x", - Assets: testAssets(), - }}, - }, - }}) -} - -// TestCreateActorTemplate_SandboxConfigChecks pins the create-time checks on -// the template's named SandboxConfig: it must exist and match the template's -// class, both FailedPrecondition — they depend on cluster state, and the -// lister may briefly lag a just-created config, so the error is retryable. -func TestCreateActorTemplate_SandboxConfigChecks(t *testing.T) { - persistence := newTestPersistence(t) - s := &RPCService{impl: newServiceImpl(persistence, nil), sandboxConfigLister: gvisorDefaultLister(t)} - ctx := context.Background() - if _, err := persistence.CreateAtespace(ctx, &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: "ns1"}}); err != nil { - t.Fatalf("CreateAtespace failed: %v", err) - } - - tests := []struct { - name string - sandbox *ateapipb.SandboxConfig - wantCode codes.Code - }{{ - name: "named config exists and matches", - sandbox: &ateapipb.SandboxConfig{SandboxClass: "gvisor", ConfigName: "gvisor-default"}, - wantCode: codes.OK, - }, { - name: "empty config_name is rejected", - sandbox: &ateapipb.SandboxConfig{SandboxClass: "microvm"}, - wantCode: codes.InvalidArgument, - }, { - name: "named config missing", - sandbox: &ateapipb.SandboxConfig{SandboxClass: "gvisor", ConfigName: "does-not-exist"}, - wantCode: codes.FailedPrecondition, - }, { - name: "named config class mismatch", - sandbox: &ateapipb.SandboxConfig{SandboxClass: "microvm", ConfigName: "gvisor-default"}, - wantCode: codes.FailedPrecondition, - }} - for i, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - req := &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) { - tmpl.Metadata = &ateapipb.ResourceMetadata{Atespace: "ns1", Name: fmt.Sprintf("tmpl-%d", i)} - tmpl.SandboxConfig = tt.sandbox - })} - _, err := s.CreateActorTemplate(ctx, req) - if apierror.Code(err) != tt.wantCode { - t.Errorf("CreateActorTemplate error = %v, want code %v", err, tt.wantCode) - } - }) - } -} - // TestCreateActorTemplate covers the atespace precondition: creation fails // while the atespace is missing, and succeeds once the atespace exists. func TestCreateActorTemplate(t *testing.T) { persistence := newTestPersistence(t) - s := &RPCService{impl: newServiceImpl(persistence, nil), sandboxConfigLister: gvisorDefaultLister(t)} + s := &RPCService{impl: newServiceImpl(persistence, nil)} ctx := context.Background() req := func(atespace, name string) *ateapipb.CreateActorTemplateRequest { return &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) { @@ -155,7 +85,7 @@ func TestCreateActorTemplate(t *testing.T) { // the only guard. func TestCreateActorTemplateIgnoresServerOwnedFields(t *testing.T) { persistence := newTestPersistence(t) - s := &RPCService{impl: newServiceImpl(persistence, nil), sandboxConfigLister: gvisorDefaultLister(t)} + s := &RPCService{impl: newServiceImpl(persistence, nil)} ctx := context.Background() if _, err := persistence.CreateAtespace(ctx, &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: "ns1"}}); err != nil { @@ -171,7 +101,7 @@ func TestCreateActorTemplateIgnoresServerOwnedFields(t *testing.T) { // Server-owned status a client must not be able to set. tmpl.Status = &ateapipb.ActorTemplateStatus{ GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ - GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden-tag"}, + GoldenTag: &ateapipb.ObjectRef{Atespace: "ns1", Name: "golden-tag"}, }, } }) @@ -222,7 +152,7 @@ func TestDeleteActorTemplate(t *testing.T) { persistence := newTestPersistence(t) tmpl := seedSubstrateTemplate(t, ctx, persistence, "tmpl") templateRef := resources.ActorTemplateRefFromActorTemplate(tmpl) - goldenRef := resources.ActorRef{Atespace: resources.GoldenActorAtespace, Name: tmpl.GetMetadata().GetUid()} + goldenRef := resources.ActorRef{Atespace: "team-a", Name: tmpl.GetMetadata().GetUid()} actor := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: goldenRef.Atespace, Name: goldenRef.Name}, ActorTemplate: templateRef.ToObjectRef(), @@ -343,10 +273,6 @@ func seedSubstrateTemplate(t *testing.T, ctx context.Context, persistence store. SnapshotConfig: &ateapipb.SnapshotConfig{ StorageLocation: "gs://ate-snapshots/team-a/", }, - SandboxConfig: &ateapipb.SandboxConfig{ - SandboxClass: "gvisor", - ConfigName: "gvisor", - }, }) if err != nil { t.Fatalf("CreateActorTemplate: %v", err) @@ -440,7 +366,7 @@ func TestUpdateActorTemplateMetadata(t *testing.T) { // A server-owned status write passes validation and bumps the version. updated, err := persistence.UpdateActorTemplate(ctx, ref, store.PreconditionFrom(created), func(tmpl *ateapipb.ActorTemplate) error { tmpl.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ - GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden-tag"}, + GoldenTag: &ateapipb.ObjectRef{Atespace: "ns1", Name: "golden-tag"}, }} return nil }) diff --git a/cmd/ateapi/internal/controlapi/actor_test.go b/cmd/ateapi/internal/controlapi/actor_test.go index 3ba9b7079b..328f6430a4 100644 --- a/cmd/ateapi/internal/controlapi/actor_test.go +++ b/cmd/ateapi/internal/controlapi/actor_test.go @@ -251,22 +251,17 @@ func TestUpdateActor_RepointTemplate(t *testing.T) { storetest.MustCreateAtespace(t, ctx, persistence, testAtespace) // tmpl-a and tmpl-b are volume-compatible; tmpl-c mounts the data volume - // elsewhere, tmpl-d declares an extra volume, and tmpl-e runs on a - // different sandbox class. + // elsewhere, and tmpl-d declares an extra volume. dataVolume := &ateapipb.Volume{Name: "data", DurableDir: &ateapipb.DurableDirVolumeSource{}} scratchVolume := &ateapipb.Volume{Name: "scratch", DurableDir: &ateapipb.DurableDirVolumeSource{}} - gvisorConfig := &ateapipb.SandboxConfig{SandboxClass: "gvisor", ConfigName: "gvisor-default"} - microvmConfig := &ateapipb.SandboxConfig{SandboxClass: "microvm", ConfigName: "microvm"} templates := map[string]struct { - mountPath string - volumes []*ateapipb.Volume - sandboxConfig *ateapipb.SandboxConfig + mountPath string + volumes []*ateapipb.Volume }{ - "tmpl-a": {"/data", []*ateapipb.Volume{dataVolume}, gvisorConfig}, - "tmpl-b": {"/data", []*ateapipb.Volume{dataVolume}, gvisorConfig}, - "tmpl-c": {"/mnt/data", []*ateapipb.Volume{dataVolume}, gvisorConfig}, - "tmpl-d": {"/data", []*ateapipb.Volume{dataVolume, scratchVolume}, gvisorConfig}, - "tmpl-e": {"/data", []*ateapipb.Volume{dataVolume}, microvmConfig}, + "tmpl-a": {"/data", []*ateapipb.Volume{dataVolume}}, + "tmpl-b": {"/data", []*ateapipb.Volume{dataVolume}}, + "tmpl-c": {"/mnt/data", []*ateapipb.Volume{dataVolume}}, + "tmpl-d": {"/data", []*ateapipb.Volume{dataVolume, scratchVolume}}, } for name, tmpl := range templates { if _, err := persistence.CreateActorTemplate(ctx, &ateapipb.ActorTemplate{ @@ -278,7 +273,6 @@ func TestUpdateActor_RepointTemplate(t *testing.T) { }}, Volumes: tmpl.volumes, SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: "gs://my-bucket/snapshots"}, - SandboxConfig: tmpl.sandboxConfig, Resources: resources.CPUMemory(1000, 512<<20), }); err != nil { t.Fatalf("creating template %s: %v", name, err) @@ -319,15 +313,6 @@ func TestUpdateActor_RepointTemplate(t *testing.T) { t.Fatalf("UpdateActor to a template with different volumes = %v, want FailedPrecondition (err: %v)", got, err) } - // Repointing at a template naming a different SandboxConfig is rejected. - _, err = svc.UpdateActor(ctx, &ateapipb.UpdateActorRequest{Actor: &ateapipb.Actor{ - Metadata: created.GetMetadata(), - ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl-e"}, - }}) - if got := apierror.Code(err); got != codes.FailedPrecondition { - t.Fatalf("UpdateActor to a template with a different sandbox config = %v, want FailedPrecondition (err: %v)", got, err) - } - // Repointing at an existing template with identical volumes and mounts // succeeds. updated, err := svc.UpdateActor(ctx, &ateapipb.UpdateActorRequest{Actor: &ateapipb.Actor{ @@ -342,8 +327,8 @@ func TestUpdateActor_RepointTemplate(t *testing.T) { } // When the old template no longer exists there is nothing left to - // compare the sandbox config or volume layout against, so the repoint - // only requires the new ref to resolve. + // compare the volume layout against, so the repoint only requires the new + // ref to resolve. orphan := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "orphan-actor"}, ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl-gone"}, @@ -351,12 +336,12 @@ func TestUpdateActor_RepointTemplate(t *testing.T) { }) repointed, err := svc.UpdateActor(ctx, &ateapipb.UpdateActorRequest{Actor: &ateapipb.Actor{ Metadata: orphan.GetMetadata(), - ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl-e"}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl-d"}, }}) if err != nil { t.Fatalf("UpdateActor from a deleted template failed: %v", err) } - if got, want := repointed.GetActorTemplate().GetName(), "tmpl-e"; got != want { + if got, want := repointed.GetActorTemplate().GetName(), "tmpl-d"; got != want { t.Errorf("updated actor_template.name = %q, want %q", got, want) } @@ -375,6 +360,37 @@ func TestUpdateActor_RepointTemplate(t *testing.T) { t.Fatalf("UpdateActor repointing a running actor = %v, want FailedPrecondition (err: %v)", got, err) } + // Repointing a paused actor is rejected, even once atelet has uploaded its + // snapshot: it resumes from the node-local copy, captured under the old + // template. + for _, name := range []string{"paused-actor", "uploaded-paused-actor"} { + paused := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl-a"}, + Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}, + }) + uploaded := mustActorSnapshotURI(t, &ateapipb.ActorTemplate{SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: "gs://my-bucket/snapshots"}}, paused, "pause-snap") + paused = mustUpdateActorStatus(t, ctx, persistence, paused, func(s *ateapipb.ActorStatus) { + snap := newPausedSnapshot(t, 1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "pause-snap", "node-1") + if name == "uploaded-paused-actor" { + snap.DurableSnapshot = &ateapipb.SnapshotStorage{ + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: uploaded.String()}, + } + } + s.AssignedNode = "node-1" + s.LastAssignedGeneration = 1 + s.Snapshots = []*ateapipb.Snapshot{snap} + }) + _, err = svc.UpdateActor(ctx, &ateapipb.UpdateActorRequest{Actor: &ateapipb.Actor{ + Metadata: paused.GetMetadata(), + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl-b"}, + }}) + if got := apierror.Code(err); got != codes.FailedPrecondition { + t.Fatalf("UpdateActor repointing %s = %v, want FailedPrecondition (err: %v)", name, got, err) + } + } + // An update that keeps the template ref is still allowed while running: // the suspended-state gate applies only to repoints. kept, err := svc.UpdateActor(ctx, &ateapipb.UpdateActorRequest{Actor: &ateapipb.Actor{ @@ -418,7 +434,6 @@ func TestUpdateActor_RepointTemplateStorageLocation(t *testing.T) { if _, err := persistence.CreateActorTemplate(ctx, &ateapipb.ActorTemplate{ Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name}, SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: location}, - SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor", ConfigName: "gvisor-default"}, }); err != nil { t.Fatalf("creating template %s: %v", name, err) } @@ -810,16 +825,15 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { ctx := t.Context() persistence := newTestPersistence(t) storetest.MustCreateAtespace(t, ctx, persistence, "team-a") - storetest.MustCreateAtespace(t, ctx, persistence, resources.GoldenActorAtespace) tmpl := seedSubstrateTemplate(t, ctx, persistence, "tmpl") - ref := &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: "golden"} + ref := &ateapipb.ObjectRef{Atespace: "team-a", Name: tmpl.GetMetadata().GetUid()} tag := &ateapipb.Tag{ Metadata: &ateapipb.ResourceMetadata{Atespace: ref.Atespace, Name: ref.Name}, SourceActor: ref, Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapipb.TagStatus{ ActorTemplateUid: tmpl.GetMetadata().GetUid(), - Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), someActorUID, "gs://bucket/atespaces/ate-golden/tags/"+someActorUID, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), someActorUID, "gs://bucket/atespaces/team-a/tags/"+someActorUID, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), }, } wantCode := codes.OK @@ -834,7 +848,6 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { wantCode = codes.FailedPrecondition case "data scope": tag.Status.Snapshot.Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES - wantCode = codes.FailedPrecondition } if scenario != "missing" { if _, err := persistence.CreateTag(ctx, tag); err != nil { @@ -850,7 +863,7 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { actor := &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor"}, ActorTemplate: resources.ActorTemplateRefFromActorTemplate(tmpl).ToObjectRef()} if scenario == "explicit tag" { tag.Metadata.Name = "explicit" - tag.Status.Snapshot = newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), "explicit", "gs://bucket/atespaces/ate-golden/tags/explicit", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + tag.Status.Snapshot = newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), "explicit", "gs://bucket/atespaces/team-a/tags/explicit", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) if _, err := persistence.CreateTag(ctx, tag); err != nil { t.Fatal(err) } diff --git a/cmd/ateapi/internal/controlapi/dialer_test.go b/cmd/ateapi/internal/controlapi/dialer_test.go index 728c5f0030..a13fe86b0e 100644 --- a/cmd/ateapi/internal/controlapi/dialer_test.go +++ b/cmd/ateapi/internal/controlapi/dialer_test.go @@ -24,12 +24,14 @@ import ( "errors" "math/big" "net/url" + "strconv" "sync" "testing" "time" "github.com/spiffe/go-spiffe/v2/bundle/x509bundle" "github.com/spiffe/go-spiffe/v2/spiffeid" + "go.ate.dev/substrate/internal/atelet" "go.ate.dev/substrate/internal/installdefaults" "go.ate.dev/substrate/internal/substratex509" "google.golang.org/grpc" @@ -44,6 +46,9 @@ import ( const testAteletSPIFFEID = "spiffe://cluster.local/ns/ate-system/sa/atelet" +// ateletPort is the port every atelet dial target should carry. +var ateletPort = strconv.Itoa(atelet.DefaultPort) + func TestAteletDialerInsecureRequiresOptIn(t *testing.T) { secure := NewAteletDialer(nil, testAteletSPIFFEID, "", "") if _, err := secure.dialCredentials("pod-uid"); err == nil { @@ -172,17 +177,17 @@ func TestDialForAteletOnNodeTarget(t *testing.T) { { name: "IPv4 atelet", ateletIP: "10.244.1.7", - wantTarget: "10.244.1.7:8085", + wantTarget: "10.244.1.7:" + ateletPort, }, { name: "IPv6 atelet is bracketed", ateletIP: "fd00:10:244::7", - wantTarget: "[fd00:10:244::7]:8085", + wantTarget: "[fd00:10:244::7]:" + ateletPort, }, { name: "IPv6 loopback is bracketed", ateletIP: "::1", - wantTarget: "[::1]:8085", + wantTarget: "[::1]:" + ateletPort, }, } @@ -403,7 +408,7 @@ func TestDialForAteletOnNode(t *testing.T) { if fresh == old { t.Fatal("DialForAteletOnNode returned the stale connection after the IP changed") } - if got, want := fresh.Target(), "10.0.0.9:8085"; got != want { + if got, want := fresh.Target(), "10.0.0.9:"+ateletPort; got != want { t.Errorf("dial target = %q, want %q", got, want) } if got := old.GetState(); got != connectivity.Shutdown { diff --git a/cmd/ateapi/internal/controlapi/export_test.go b/cmd/ateapi/internal/controlapi/export_test.go index df91876b22..3d0a435cac 100644 --- a/cmd/ateapi/internal/controlapi/export_test.go +++ b/cmd/ateapi/internal/controlapi/export_test.go @@ -52,9 +52,6 @@ type ReleaseWindowWorkflow struct { State ateapipb.ActorState // Snapshot, when set, is the actor's latest snapshot. Snapshot func(t *testing.T) *ateapipb.Snapshot - // WorkerClass is the worker's sandbox class. Templates ask for gVisor, - // so "microvm" makes the worker ineligible. - WorkerClass string // Finalize runs the workflow or step that releases the worker and then // records the actor's new state against st. Pause and suspend reach the // fake atelet newWireCaptureWorkflow serves for the worker @@ -73,21 +70,24 @@ type ReleaseWindowWorkflow struct { // the worker assignment. Revert and delete release theirs in REVERTING and // DELETING, which HostingNode never resolves to a node. func ReleaseWindowWorkflows() []ReleaseWindowWorkflow { - gvisorTemplate := &ateapipb.ActorTemplate{SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor"}} + // The workers these tests create carry no labels, so a template + // selecting a pool makes any of them ineligible. + ineligibleTemplate := &ateapipb.ActorTemplate{ + WorkerSelector: &ateapipb.Selector{MatchLabels: map[string]string{"pool": "other"}}, + } return []ReleaseWindowWorkflow{ { Name: "pause", State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, - Snapshot: func(*testing.T) *ateapipb.Snapshot { - // A pause in progress: node-local and not yet captured. - return newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-snap-1", "") + Snapshot: func(t *testing.T) *ateapipb.Snapshot { + // A pause in progress: a PENDING durable copy, not yet captured. + return newInProgressSnapshot(t, 1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "local-snap-1") }, - WorkerClass: "gvisor", // The actor is seeded SUSPENDING, so the pause skips its mark step // and its first actor update is the finalize commit. Finalize: func(t *testing.T, ctx context.Context, st store.Interface, actorRef resources.ActorRef, _ *ateapipb.Actor) error { w, _ := newWireCaptureWorkflow(t, st) - _, err := w.PauseActor(ctx, actorRef) + _, err := w.SuspendActor(ctx, actorRef, suspendAsync) return err }, WantRetry: codes.OK, @@ -99,20 +99,18 @@ func ReleaseWindowWorkflows() []ReleaseWindowWorkflow { Snapshot: func(t *testing.T) *ateapipb.Snapshot { return newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snapshot-1", someActorSnapshotURI(t, testStorageLocation, "team-a", "snapshot-1"), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) }, - WorkerClass: "gvisor", // As for the pause, the finalize commit is the first actor update. Finalize: func(t *testing.T, ctx context.Context, st store.Interface, actorRef resources.ActorRef, _ *ateapipb.Actor) error { w, _ := newWireCaptureWorkflow(t, st) - _, err := w.SuspendActor(ctx, actorRef) + _, err := w.SuspendActor(ctx, actorRef, suspendSync) return err }, WantRetry: codes.OK, WantFinalState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, }, { - Name: "crash", - State: ateapipb.ActorState_ACTOR_STATE_RUNNING, - WorkerClass: "gvisor", + Name: "crash", + State: ateapipb.ActorState_ACTOR_STATE_RUNNING, Finalize: func(_ *testing.T, ctx context.Context, st store.Interface, actorRef resources.ActorRef, _ *ateapipb.Actor) error { return crashActor(ctx, st, actorRef, ateattr.OperationResume, "test crash") }, @@ -123,11 +121,10 @@ func ReleaseWindowWorkflows() []ReleaseWindowWorkflow { // A resume that finds its worker ineligible releases it, then // crashes the actor. The retry finds the worker no longer hosts // the actor and crashes it. - Name: "resume onto an ineligible worker", - State: ateapipb.ActorState_ACTOR_STATE_RESUMING, - WorkerClass: "microvm", + Name: "resume onto an ineligible worker", + State: ateapipb.ActorState_ACTOR_STATE_RESUMING, Finalize: func(_ *testing.T, ctx context.Context, st store.Interface, actorRef resources.ActorRef, actor *ateapipb.Actor) error { - _, err := (&ActorWorkflow{store: st, scheduler: scheduling.New(nil)}).validateAssignedWorker(ctx, actorRef, actor, gvisorTemplate) + _, err := (&ActorWorkflow{store: st, scheduler: scheduling.New(nil)}).validateAssignedWorker(ctx, actorRef, actor, ineligibleTemplate) return err }, WantRetry: codes.Aborted, diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go index 27df83d3f0..68a27b5dba 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go @@ -38,22 +38,17 @@ func TestActorTemplateCRUD(t *testing.T) { tc := setupTest(t, ns) defer tc.cleanup() ctx := context.Background() - ensureDefaultGvisorSandboxConfig(t, tc) created, err := tc.client.CreateActorTemplate(ctx, &ateapipb.CreateActorTemplateRequest{ ActorTemplate: &ateapipb.ActorTemplate{ Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "tmpl-a"}, Containers: []*ateapipb.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: "gs://my-bucket/snapshots"}, - SandboxConfig: &ateapipb.SandboxConfig{ - SandboxClass: "gvisor", - ConfigName: "gvisor-default", - }, - Resources: &ateapipb.Resources{Limits: []*ateapipb.Limits{{Name: "cpu", Quantity: "1"}, {Name: "memory", Quantity: "1Gi"}}}, + Resources: &ateapipb.Resources{Limits: []*ateapipb.Limits{{Name: "cpu", Quantity: "1"}, {Name: "memory", Quantity: "1Gi"}}}, // Server-owned status on the request is ignored. Status: &ateapipb.ActorTemplateStatus{ GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ - GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden-tag"}, + GoldenTag: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "golden-tag"}, }, }, }, @@ -69,10 +64,6 @@ func TestActorTemplateCRUD(t *testing.T) { PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, GoldenSnapshotConfig: &ateapipb.GoldenSnapshotConfig{Mode: ateapipb.GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_ENABLED}, }, - SandboxConfig: &ateapipb.SandboxConfig{ - SandboxClass: "gvisor", - ConfigName: "gvisor-default", - }, Resources: &ateapipb.Resources{Limits: []*ateapipb.Limits{{Name: "cpu", Quantity: "1"}, {Name: "memory", Quantity: "1Gi"}}}, Status: &ateapipb.ActorTemplateStatus{}, } @@ -85,7 +76,6 @@ func TestActorTemplateCRUD(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "tmpl-a"}, Containers: []*ateapipb.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: "gs://my-bucket/snapshots"}, - SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor", ConfigName: "gvisor-default"}, Resources: resources.CPUMemory(1000, 512<<20), }, }) @@ -135,18 +125,6 @@ func TestActorTemplateCRUD(t *testing.T) { assertGrpcError(t, err, codes.NotFound, "ActorTemplate "+testAtespace+"/tmpl-a not found") _, err = tc.client.DeleteActorTemplate(ctx, &ateapipb.DeleteActorTemplateRequest{ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl-a"}}) assertGrpcError(t, err, codes.NotFound, "ActorTemplate "+testAtespace+"/tmpl-a not found") - - // config_name is required: a template must name its SandboxConfig. - _, err = tc.client.CreateActorTemplate(ctx, &ateapipb.CreateActorTemplateRequest{ - ActorTemplate: &ateapipb.ActorTemplate{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "tmpl-unnamed-config"}, - Containers: []*ateapipb.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, - SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: "gs://my-bucket/snapshots"}, - SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor"}, - Resources: resources.CPUMemory(1000, 512<<20), - }, - }) - assertGrpcErrorRegex(t, err, codes.InvalidArgument, `sandbox_config\.config_name`) } func TestGoldenTagLifecycle(t *testing.T) { @@ -185,7 +163,7 @@ func TestGoldenTagLifecycle(t *testing.T) { if err != nil { t.Fatal(err) } - if !parsed.OwnedBy(resources.TagSnapshotOwner(resources.GoldenActorAtespace, golden.GetMetadata().GetUid())) { + if !parsed.OwnedBy(resources.TagSnapshotOwner(testAtespace, golden.GetMetadata().GetUid())) { t.Fatalf("golden snapshot %q is not under the golden tag's own prefix", uri) } assertSnapshotPresent(t, tc, uri) @@ -222,7 +200,10 @@ func TestGoldenTagLifecycle(t *testing.T) { if !tc.fakeAtelet.RestoreCalled { t.Fatal("actor did not restore golden tag") } - if _, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: resources.ActorRefFromActor(late).ToObjectRef()}); err != nil { + if _, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: resources.ActorRefFromActor(late).ToObjectRef(), + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, + }); err != nil { t.Fatal(err) } assertSnapshotPresent(t, tc, uri) @@ -247,7 +228,6 @@ func TestActorLifecycleWithoutGoldenSnapshot(t *testing.T) { defer tc.cleanup() ctx, cancel := context.WithCancel(t.Context()) defer cancel() - ensureDefaultGvisorSandboxConfig(t, tc) createWorkerPool(t, tc, ns, "pool1", map[string]string{poolLabelKey: ns}) tmpl, err := tc.client.CreateActorTemplate(ctx, &ateapipb.CreateActorTemplateRequest{ActorTemplate: &ateapipb.ActorTemplate{ Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "no-golden"}, @@ -255,7 +235,6 @@ func TestActorLifecycleWithoutGoldenSnapshot(t *testing.T) { StorageLocation: testStorageLocation, GoldenSnapshotConfig: &ateapipb.GoldenSnapshotConfig{Mode: ateapipb.GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_DISABLED}, }, - SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor", ConfigName: "gvisor-default"}, Containers: []*ateapipb.Container{{Name: "main", Image: "main@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", Command: []string{"/main"}}}, WorkerSelector: &ateapipb.Selector{MatchLabels: map[string]string{poolLabelKey: ns}}, Resources: resources.CPUMemory(1000, 512<<20), @@ -268,7 +247,7 @@ func TestActorLifecycleWithoutGoldenSnapshot(t *testing.T) { } workerName := createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") controlTemplate := createTemplateWithSelector(t, tc, "golden-control", &ateapipb.Selector{MatchLabels: map[string]string{poolLabelKey: ns}}) - goldenRef := &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: tmpl.GetMetadata().GetUid()} + goldenRef := &ateapipb.ObjectRef{Atespace: testAtespace, Name: tmpl.GetMetadata().GetUid()} assertNoGolden := func(ctx context.Context) error { if _, err := tc.client.GetActor(ctx, &ateapipb.GetActorRequest{Actor: goldenRef}); status.Code(err) != codes.NotFound { return fmt.Errorf("disabled template's golden actor: got %v, want NotFound", err) @@ -316,7 +295,7 @@ func TestActorLifecycleWithoutGoldenSnapshot(t *testing.T) { if !ran || restored { t.Fatal("first resume did not cold boot") } - suspended, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: ref}) + suspended, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: ref, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}) if err != nil { t.Fatal(err) } @@ -401,7 +380,6 @@ func TestCreateActorTemplate_ExternalVolumesPreview(t *testing.T) { ns := namespaceForTest("ns-template-ext-vol-preview") tc := setupTest(t, ns) defer tc.cleanup() - ensureDefaultGvisorSandboxConfig(t, tc) for i, tt := range []struct { name string @@ -433,10 +411,6 @@ func TestCreateActorTemplate_ExternalVolumesPreview(t *testing.T) { }}, Resources: &ateapipb.Resources{Limits: []*ateapipb.Limits{{Name: "cpu", Quantity: "1"}, {Name: "memory", Quantity: "1Gi"}}}, SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: "gs://my-bucket/snapshots"}, - SandboxConfig: &ateapipb.SandboxConfig{ - SandboxClass: "gvisor", - ConfigName: "gvisor-default", - }, }, }) if status.Code(err) != tt.wantCode { diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index b061f3d5b3..4a6cefcf38 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -273,13 +273,11 @@ func TestCreateActor_SubstrateTemplateRef(t *testing.T) { defer tc.cleanup() ctx := context.Background() - ensureDefaultGvisorSandboxConfig(t, tc) if _, err := tc.client.CreateActorTemplate(ctx, &ateapipb.CreateActorTemplateRequest{ ActorTemplate: &ateapipb.ActorTemplate{ Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "sub-tmpl"}, Containers: []*ateapipb.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: "gs://my-bucket/snapshots"}, - SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor", ConfigName: "gvisor-default"}, Resources: resources.CPUMemory(1000, 512<<20), }, }); err != nil { @@ -399,15 +397,10 @@ func TestCreateActor_RejectsSnapshotWithExternalVolumes(t *testing.T) { ns := namespaceForTest("ns-snapshot-external-volume") tc := setupPreviewTest(t, ns) defer tc.cleanup() - ensureDefaultGvisorSandboxConfig(t, tc) template, err := tc.client.CreateActorTemplate(context.Background(), &ateapipb.CreateActorTemplateRequest{ ActorTemplate: &ateapipb.ActorTemplate{ Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "tmpl1"}, SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: "gs://snapshots"}, - SandboxConfig: &ateapipb.SandboxConfig{ - SandboxClass: "gvisor", - ConfigName: "gvisor-default", - }, Containers: []*ateapipb.Container{{ Name: "main", Image: "main@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", VolumeMounts: []*ateapipb.VolumeMount{{Name: "data", MountPath: "/data"}}, }}, @@ -803,8 +796,8 @@ func TestUpdateActor_Success(t *testing.T) { // TestUpdateActor_RepointTemplate verifies UpdateActor can point an actor at // a different substrate ActorTemplate (effective on the next ResumeActor), -// and that a ref to an absent template, or to one with a different sandbox -// config, volumes, or volume mounts, is rejected. +// and that a ref to an absent template, or to one with different volumes or +// volume mounts, is rejected. func TestUpdateActor_RepointTemplate(t *testing.T) { tests := []struct { name string @@ -814,7 +807,6 @@ func TestUpdateActor_RepointTemplate(t *testing.T) { {name: "absent-template", template: "absent", wantCode: codes.FailedPrecondition}, {name: "different-mounts", template: "tmpl-c", wantCode: codes.FailedPrecondition}, {name: "different-volumes", template: "tmpl-d", wantCode: codes.FailedPrecondition}, - {name: "different-sandbox-config", template: "tmpl-e", wantCode: codes.FailedPrecondition}, {name: "same-volumes", template: "tmpl-b", wantCode: codes.OK}, } for _, tt := range tests { @@ -824,23 +816,18 @@ func TestUpdateActor_RepointTemplate(t *testing.T) { defer tc.cleanup() ctx := context.Background() - ensureDefaultGvisorSandboxConfig(t, tc) - ensureGvisorSandboxConfig(t, tc, "gvisor-nightly") // tmpl-a and tmpl-b are volume-compatible; tmpl-c mounts the data - // volume elsewhere, tmpl-d declares an extra volume, and tmpl-e - // names a different SandboxConfig. + // volume elsewhere, and tmpl-d declares an extra volume. dataVolume := &ateapipb.Volume{Name: "data", DurableDir: &ateapipb.DurableDirVolumeSource{}} scratchVolume := &ateapipb.Volume{Name: "scratch", DurableDir: &ateapipb.DurableDirVolumeSource{}} templates := map[string]struct { - mountPath string - volumes []*ateapipb.Volume - configName string + mountPath string + volumes []*ateapipb.Volume }{ - "tmpl-a": {"/data", []*ateapipb.Volume{dataVolume}, "gvisor-default"}, - "tmpl-b": {"/data", []*ateapipb.Volume{dataVolume}, "gvisor-default"}, - "tmpl-c": {"/mnt/data", []*ateapipb.Volume{dataVolume}, "gvisor-default"}, - "tmpl-d": {"/data", []*ateapipb.Volume{dataVolume, scratchVolume}, "gvisor-default"}, - "tmpl-e": {"/data", []*ateapipb.Volume{dataVolume}, "gvisor-nightly"}, + "tmpl-a": {"/data", []*ateapipb.Volume{dataVolume}}, + "tmpl-b": {"/data", []*ateapipb.Volume{dataVolume}}, + "tmpl-c": {"/mnt/data", []*ateapipb.Volume{dataVolume}}, + "tmpl-d": {"/data", []*ateapipb.Volume{dataVolume, scratchVolume}}, } for name, tmpl := range templates { if _, err := tc.client.CreateActorTemplate(ctx, &ateapipb.CreateActorTemplateRequest{ @@ -853,7 +840,6 @@ func TestUpdateActor_RepointTemplate(t *testing.T) { }}, Volumes: tmpl.volumes, SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: "gs://my-bucket/snapshots"}, - SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor", ConfigName: tmpl.configName}, Resources: resources.CPUMemory(1000, 512<<20), }, }); err != nil { @@ -1424,8 +1410,8 @@ func TestDeleteActor_Paused(t *testing.T) { if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { t.Fatalf("ResumeActor failed: %v", err) } - if _, err := tc.client.PauseActor(context.Background(), &ateapipb.PauseActorRequest{Actor: actorRef}); err != nil { - t.Fatalf("PauseActor failed: %v", err) + if _, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: actorRef, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}}); err != nil { + t.Fatalf("SuspendActor failed: %v", err) } paused, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{Actor: actorRef}) if err != nil { @@ -1673,11 +1659,6 @@ func TestValidation_Actor(t *testing.T) { assertGrpcErrorRegex(t, err, codes.InvalidArgument, "actor: Required value") }) - t.Run("PauseActor", func(t *testing.T) { - _, err := tc.client.PauseActor(context.Background(), &ateapipb.PauseActorRequest{}) - assertGrpcErrorRegex(t, err, codes.InvalidArgument, "actor: Required value") - }) - t.Run("SuspendActor", func(t *testing.T) { _, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{}) assertGrpcErrorRegex(t, err, codes.InvalidArgument, "actor: Required value") @@ -1768,12 +1749,13 @@ func TestActorLifecycle_WithExternalVolumes(t *testing.T) { t.Fatalf("expected non-empty storageVolumeId after ResumeActor") } - // 3. PauseActor - pauseResp, err := tc.client.PauseActor(context.Background(), &ateapipb.PauseActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-vol-lc"}, + // 3. Async SuspendActor + pauseResp, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-vol-lc"}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, }) if err != nil { - t.Fatalf("PauseActor failed: %v", err) + t.Fatalf("SuspendActor failed: %v", err) } if pauseResp.GetActor().GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Fatalf("expected state ACTOR_STATE_SUSPENDED after pause, got %v", pauseResp.GetActor().GetStatus().GetState()) @@ -1795,9 +1777,10 @@ func TestActorLifecycle_WithExternalVolumes(t *testing.T) { t.Fatalf("expected state ACTOR_STATE_RUNNING after second resume, got %v", resumeResp2.GetActor().GetStatus().GetState()) } - // 5. SuspendActor + // 5. Sync SuspendActor suspendResp, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-vol-lc"}, + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-vol-lc"}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, }) if err != nil { t.Fatalf("SuspendActor failed: %v", err) @@ -2133,7 +2116,12 @@ func (a *attachFailVolumePlugin) AttachVolume(ctx context.Context, req volume.At return volume.AttachVolumeResponse{}, fmt.Errorf("simulated volume attach failure on attempt %d", a.attachAttempts) } a.attachedNodes = append(a.attachedNodes, req.Node) - return volume.AttachVolumeResponse{}, nil + return volume.AttachVolumeResponse{ + PublishContext: map[string]string{ + "mock.substrate.io/attached-node": req.Node, + "mock.substrate.io/device-path": "/dev/mock-" + req.VolumeID, + }, + }, nil } func (a *attachFailVolumePlugin) DetachVolume(ctx context.Context, volumeID, node string) error { @@ -2787,19 +2775,16 @@ func TestSuspendActor_VolumeDetachFailure_DeleteActorAnyState(t *testing.T) { } } -// TestPauseActor_VolumeLifecycle_DetachAndResumeAttach tests scenario D: -// external volume lifecycle across PauseActor and ResumeActor: -// 1. When an actor is resumed to RUNNING, external volumes are attached to the worker node. -// 2. When the actor is paused, volumes are detached from the worker node and the actor transitions to ACTOR_STATE_SUSPENDED with a local snapshot. -// 3. When the actor is resumed from pause, volumes are re-attached to the worker node and the actor transitions to ACTOR_STATE_RUNNING. -// -// Workflow: -// 1. Creates a template with an external volume and creates a mock worker pod on node1. -// 2. Creates and resumes an actor to ACTOR_STATE_RUNNING; verifies volume is attached to node1. -// 3. Calls PauseActor; verifies volume is detached from node1 and actor transitions to ACTOR_STATE_SUSPENDED with a local snapshot. -// 4. Calls ResumeActor from pause; verifies volume is re-attached to node1 and actor transitions to ACTOR_STATE_RUNNING. -// 5. Cleans up by suspending and deleting the actor. -func TestPauseActor_VolumeLifecycle_DetachAndResumeAttach(t *testing.T) { +// TestSuspendActorAsync_VolumeLifecycle_DetachResumeAndMigrate tests external volume +// attachment, publishContext delivery, detachment, cross-node migration, and +// deletion across async SuspendActor, ResumeActor, SuspendActor, and DeleteActor: +// 1. Creates a template with an external volume and a mock worker pod on node1. +// 2. Creates and resumes an actor to ACTOR_STATE_RUNNING; verifies volume is attached to node1 and restore request carries node1 publishContext. +// 3. Calls async SuspendActor; verifies volume is detached from node1 and actor transitions to ACTOR_STATE_SUSPENDED with a local snapshot. +// 4. Calls ResumeActor from pause; verifies volume is re-attached to node1 and restore request carries node1 publishContext. +// 5. Calls SuspendActor, replaces worker-1 (node1) with worker-2 (node2), and calls ResumeActor; verifies volume is attached to node2 and restore request carries node2 publishContext. +// 6. Calls SuspendActor and DeleteActor; verifies volume is detached from node2 and deleted. +func TestSuspendActorAsync_VolumeLifecycle_DetachResumeAndMigrate(t *testing.T) { preview.InitForTest(t, "ExternalVolumes") ns := namespaceForTest("ns-pause-vol-lifecycle") @@ -2826,7 +2811,7 @@ func TestPauseActor_VolumeLifecycle_DetachAndResumeAttach(t *testing.T) { workerName := createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") // 2. Create actor and resume to RUNNING; volume is attached to node1. - _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ + createResp, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ Actor: &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "pause-vol-actor"}, ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, @@ -2835,6 +2820,7 @@ func TestPauseActor_VolumeLifecycle_DetachAndResumeAttach(t *testing.T) { if err != nil { t.Fatalf("CreateActor failed: %v", err) } + actorUID := createResp.GetMetadata().GetUid() _, err = tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "pause-vol-actor"}, @@ -2847,13 +2833,17 @@ func TestPauseActor_VolumeLifecycle_DetachAndResumeAttach(t *testing.T) { t.Errorf("attached nodes after resume mismatch (-want +got):\n%s", diff) } plugin.mu.Unlock() + if got := tc.fakeAtelet.lastRestoreVolumePublishContext("vol1"); got["mock.substrate.io/attached-node"] != "node1" { + t.Errorf("atelet restore publishContext after first resume = %v, want attached-node=node1", got) + } - // 3. PauseActor: volume is detached from node1 and actor transitions to ACTOR_STATE_SUSPENDED with a local snapshot. - _, err = tc.client.PauseActor(context.Background(), &ateapipb.PauseActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "pause-vol-actor"}, + // 3. Async SuspendActor: volume is detached from node1 and actor transitions to ACTOR_STATE_SUSPENDED with a local snapshot. + _, err = tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "pause-vol-actor"}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, }) if err != nil { - t.Fatalf("PauseActor failed: %v", err) + t.Fatalf("SuspendActor failed: %v", err) } getResp, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ @@ -2876,6 +2866,7 @@ func TestPauseActor_VolumeLifecycle_DetachAndResumeAttach(t *testing.T) { waitForWorkerAvailable(t, tc, workerName) // 4. ResumeActor from pause: volume is re-attached to node1 and actor transitions to ACTOR_STATE_RUNNING. + tc.fakeAtelet.Reset() _, err = tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "pause-vol-actor"}, }) @@ -2898,36 +2889,92 @@ func TestPauseActor_VolumeLifecycle_DetachAndResumeAttach(t *testing.T) { t.Errorf("attached nodes after resume from pause mismatch (-want +got):\n%s", diff) } plugin.mu.Unlock() + if got := tc.fakeAtelet.lastRestoreVolumePublishContext("vol1"); got["mock.substrate.io/attached-node"] != "node1" { + t.Errorf("atelet restore publishContext after resume from pause = %v, want attached-node=node1", got) + } - // 5. Clean up by suspending and deleting the actor. + // 5. Suspend to detach from node1 and clear local snapshot, then migrate to node2. _, err = tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "pause-vol-actor"}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, + }) + if err != nil { + t.Fatalf("SuspendActor before migration failed: %v", err) + } + plugin.mu.Lock() + if diff := cmp.Diff([]string{"node1", "node1"}, plugin.detachedNodes); diff != "" { + t.Errorf("detached nodes after suspend before migration mismatch (-want +got):\n%s", diff) + } + plugin.mu.Unlock() + + setupAteletOnNode(t, tc, "atelet-node2", "node2") + createWorkerPod(t, tc, ns, "worker-2", "node2", "pool1") + deleteWorkerPod(t, tc, ns, "worker-1") + tc.fakeAtelet.Reset() + + _, err = tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "pause-vol-actor"}, }) if err != nil { - t.Fatalf("SuspendActor failed: %v", err) + t.Fatalf("ResumeActor on migration failed: %v", err) + } + plugin.mu.Lock() + if diff := cmp.Diff([]string{"node1", "node1", "node2"}, plugin.attachedNodes); diff != "" { + t.Errorf("attached nodes after migration mismatch (-want +got):\n%s", diff) + } + plugin.mu.Unlock() + if got := tc.fakeAtelet.lastRestoreVolumePublishContext("vol1"); got["mock.substrate.io/attached-node"] != "node2" { + t.Errorf("atelet restore publishContext after migration = %v, want attached-node=node2", got) + } + + // 6. Suspend and delete the actor on node2. + _, err = tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "pause-vol-actor"}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, + }) + if err != nil { + t.Fatalf("SuspendActor after migration failed: %v", err) + } + plugin.mu.Lock() + if diff := cmp.Diff([]string{"node1", "node1", "node2"}, plugin.detachedNodes); diff != "" { + t.Errorf("detached nodes after final suspend mismatch (-want +got):\n%s", diff) } + plugin.mu.Unlock() + _, err = tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "pause-vol-actor"}, }) if err != nil { t.Fatalf("DeleteActor failed: %v", err) } + expectedVolumeID := "storage-substrate-" + actorUID + "-vol1" + plugin.mu.Lock() + if diff := cmp.Diff([]string{"node1", "node1", "node2"}, plugin.detachedNodes); diff != "" { + t.Errorf("detached nodes after delete mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff([]string{expectedVolumeID}, plugin.deleted); diff != "" { + t.Errorf("deleted volume IDs mismatch (-want +got):\n%s", diff) + } + plugin.mu.Unlock() + if _, err := tc.persistence.GetActor(context.Background(), resources.ActorRef{Atespace: testAtespace, Name: "pause-vol-actor"}); !errors.Is(err, store.ErrNotFound) { + t.Errorf("GetActor from store after delete: err = %v, want %v", err, store.ErrNotFound) + } } -// TestPauseActor_VolumeDetachFailure_RetrySuccess tests scenario D: -// when volume detachment fails during PauseActor: -// 1. PauseActor fails, leaving the actor in ACTOR_STATE_SUSPENDING. -// 2. A subsequent retry of PauseActor re-attempts volume detachment and succeeds, transitioning the actor to ACTOR_STATE_SUSPENDED. +// TestSuspendActorAsync_VolumeDetachFailure_RetrySuccess tests scenario D: +// when volume detachment fails during async SuspendActor: +// 1. Async SuspendActor fails, leaving the actor in ACTOR_STATE_SUSPENDING. +// 2. A subsequent retry of async SuspendActor re-attempts volume detachment and succeeds, transitioning the actor to ACTOR_STATE_SUSPENDED. // // Workflow: // 1. Creates a template with an external volume and creates a mock worker pod on node1. // 2. Creates and resumes an actor to ACTOR_STATE_RUNNING. -// 3. Calls PauseActor; plugin is configured to fail DetachVolume on attempt 1. -// 4. Verifies PauseActor returns an error and actor state is left in ACTOR_STATE_SUSPENDING. -// 5. Calls PauseActor a second time (retry); detachment succeeds. +// 3. Calls async SuspendActor; plugin is configured to fail DetachVolume on attempt 1. +// 4. Verifies async SuspendActor returns an error and actor state is left in ACTOR_STATE_SUSPENDING. +// 5. Calls async SuspendActor a second time (retry); detachment succeeds. // 6. Verifies actor state transitions to ACTOR_STATE_SUSPENDED with a local snapshot. // 7. Cleans up by deleting the actor with AnyState=true. -func TestPauseActor_VolumeDetachFailure_RetrySuccess(t *testing.T) { +func TestSuspendActorAsync_VolumeDetachFailure_RetrySuccess(t *testing.T) { preview.InitForTest(t, "ExternalVolumes") ns := namespaceForTest("ns-pause-vol-detach-retry") @@ -2970,12 +3017,13 @@ func TestPauseActor_VolumeDetachFailure_RetrySuccess(t *testing.T) { t.Fatalf("ResumeActor failed: %v", err) } - // 3. Attempt 1: PauseActor fails on DetachVolume. - _, err = tc.client.PauseActor(context.Background(), &ateapipb.PauseActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "pause-detach-retry-actor"}, + // 3. Attempt 1: async SuspendActor fails on DetachVolume. + _, err = tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "pause-detach-retry-actor"}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, }) if err == nil { - t.Fatalf("expected first PauseActor to fail, got nil") + t.Fatalf("expected first async SuspendActor to fail, got nil") } if !strings.Contains(err.Error(), "simulated volume detach failure on attempt 1") { t.Errorf("expected error containing simulated detach failure, got: %v", err) @@ -2992,12 +3040,13 @@ func TestPauseActor_VolumeDetachFailure_RetrySuccess(t *testing.T) { t.Errorf("actor state = %v, want ACTOR_STATE_SUSPENDING", getResp.GetStatus().GetState()) } - // 5. Attempt 2 (retry): PauseActor retry succeeds, detaching volume. - _, err = tc.client.PauseActor(context.Background(), &ateapipb.PauseActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "pause-detach-retry-actor"}, + // 5. Attempt 2 (retry): async SuspendActor retry succeeds, detaching volume. + _, err = tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "pause-detach-retry-actor"}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, }) if err != nil { - t.Fatalf("expected retry PauseActor to succeed, got: %v", err) + t.Fatalf("expected retry async SuspendActor to succeed, got: %v", err) } // 6. Verify actor state transitions to ACTOR_STATE_SUSPENDED with a local snapshot. @@ -3194,7 +3243,6 @@ func TestResumeActorPassesLiteralEnv(t *testing.T) { // snapshot while the template also has a golden snapshot. func createDataFidelityTemplate(t *testing.T, tc *testContext, ns string) *ateapipb.ActorTemplate { t.Helper() - ensureDefaultGvisorSandboxConfig(t, tc) createWorkerPool(t, tc, ns, "pool1", map[string]string{poolLabelKey: ns}) created, err := tc.client.CreateActorTemplate(context.Background(), &ateapipb.CreateActorTemplateRequest{ @@ -3207,10 +3255,6 @@ func createDataFidelityTemplate(t *testing.T, tc *testContext, ns string) *ateap StorageLocation: testStorageLocation, PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, - SandboxConfig: &ateapipb.SandboxConfig{ - SandboxClass: "gvisor", - ConfigName: "gvisor-default", - }, Containers: []*ateapipb.Container{{ Name: "main", Image: "main@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", @@ -3228,13 +3272,12 @@ func createDataFidelityTemplate(t *testing.T, tc *testContext, ns string) *ateap // Publish the golden snapshot as a tag and point the template at it, as // createTemplateWithContainersAndVolumes does. - createAtespace(t, tc, resources.GoldenActorAtespace) tag, err := tc.persistence.CreateTag(context.Background(), &ateapipb.Tag{ - Metadata: &ateapipb.ResourceMetadata{Atespace: resources.GoldenActorAtespace, Name: created.GetMetadata().GetUid()}, - SourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: created.GetMetadata().GetUid()}, + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: created.GetMetadata().GetUid()}, + SourceActor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: created.GetMetadata().GetUid()}, Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapipb.TagStatus{ - Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, created.GetMetadata().GetUid(), goldenSnapshotName, goldenSnapshotURI(t)), + Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, created.GetMetadata().GetUid(), goldenSnapshotName, goldenSnapshotURI(t)), ActorTemplateUid: created.GetMetadata().GetUid(), }, }) @@ -3282,7 +3325,7 @@ func TestResumeActor_DataSnapshotIgnoresGolden(t *testing.T) { if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { t.Fatalf("ResumeActor (first) failed: %v", err) } - suspended, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: actorRef}) + suspended, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: actorRef, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}) if err != nil { t.Fatalf("SuspendActor failed: %v", err) } @@ -3335,7 +3378,7 @@ func TestSuspendActor_ReplacedSnapshotReleaseFailure(t *testing.T) { if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { t.Fatalf("ResumeActor (first) failed: %v", err) } - first, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: actorRef}) + first, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: actorRef, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}) if err != nil { t.Fatalf("SuspendActor (first) failed: %v", err) } @@ -3349,7 +3392,7 @@ func TestSuspendActor_ReplacedSnapshotReleaseFailure(t *testing.T) { tc.objectStore.OnDelete = func(string, string) error { return status.Error(codes.Unavailable, "object storage is unavailable") } - second, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: actorRef}) + second, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: actorRef, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}) tc.objectStore.OnDelete = nil if err != nil { t.Fatalf("SuspendActor (second) failed: %v", err) @@ -3375,7 +3418,7 @@ func TestSuspendActor_ReplacedSnapshotReleaseFailure(t *testing.T) { // A later suspend releases only the snapshot it replaces; the one the // failed release left behind stays until the actor is deleted, which // collects everything under the actor's prefix. - third, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: actorRef}) + third, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: actorRef, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}) if err != nil { t.Fatalf("SuspendActor (third) failed: %v", err) } @@ -3672,8 +3715,8 @@ func TestResumeActor_LocalRestoreFailureCrashesActor(t *testing.T) { if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: ref}); err != nil { t.Fatalf("ResumeActor failed: %v", err) } - if _, err := tc.client.PauseActor(context.Background(), &ateapipb.PauseActorRequest{Actor: ref}); err != nil { - t.Fatalf("PauseActor failed: %v", err) + if _, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: ref, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}}); err != nil { + t.Fatalf("SuspendActor failed: %v", err) } tc.fakeAtelet.Reset() @@ -3763,7 +3806,8 @@ func TestSuspendActor(t *testing.T) { // Suspend suspended, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, }) if err != nil { t.Fatalf("SuspendActor failed: %v", err) @@ -3886,7 +3930,7 @@ func TestSuspendActor(t *testing.T) { // The clone's first suspend writes a snapshot of its own and stops // borrowing, which is what makes the tag's snapshot collectable later. - cloneSuspended, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "clone"}}) + cloneSuspended, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "clone"}, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}) if err != nil { t.Fatalf("SuspendActor clone failed: %v", err) } @@ -3898,10 +3942,15 @@ func TestSuspendActor(t *testing.T) { // It stopped borrowing without releasing what it had borrowed. assertSnapshotPresent(t, tc, taggedSnapshotURI) assertSnapshotPresent(t, tc, cloneSnapshotURI) - // The untagged suspend created no tag. + // The untagged suspend created no tag: only the golden and explicit tag remain. listed, err = tc.client.ListTags(context.Background(), &ateapipb.ListTagsRequest{Atespace: testAtespace}) - if err != nil || len(listed.GetTags()) != 1 { - t.Fatalf("ListTags after clone suspension = (%v, %v), want one", listed, err) + if err != nil || len(listed.GetTags()) != 2 { + t.Fatalf("ListTags after clone suspension = (%v, %v), want golden and explicit tags", listed, err) + } + for _, tag := range listed.GetTags() { + if ref := resources.TagRefFromTag(tag).ToObjectRef(); !proto.Equal(ref, tagRef) && !proto.Equal(ref, tmpl.GetStatus().GetGoldenSnapshotStatus().GetGoldenTag()) { + t.Errorf("unexpected tag after clone suspension: %v", ref) + } } getResp, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ @@ -4014,7 +4063,7 @@ func TestResumeActor_RepointTemplateBeforeResume(t *testing.T) { if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { t.Fatalf("ResumeActor(%s) failed: %v", sourceActorName, err) } - if _, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef}); err != nil { + if _, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}); err != nil { t.Fatalf("SuspendActor(%s) failed: %v", sourceActorName, err) } waitForWorkerAvailable(t, tc, worker) @@ -4107,7 +4156,7 @@ func TestResumeActor_PausedAfterRepointUsesLocalProvenance(t *testing.T) { if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { t.Fatalf("ResumeActor(v1) failed: %v", err) } - suspended, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef}) + suspended, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}) if err != nil { t.Fatalf("SuspendActor(v1) failed: %v", err) } @@ -4129,8 +4178,8 @@ func TestResumeActor_PausedAfterRepointUsesLocalProvenance(t *testing.T) { if got := tc.fakeAtelet.lastRestoreRequest().GetSnapshot().GetFidelity(); got != ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES { t.Fatalf("first resume on v2 had scope = %v, want DATA", got) } - if _, err := tc.client.PauseActor(ctx, &ateapipb.PauseActorRequest{Actor: actorRef}); err != nil { - t.Fatalf("PauseActor failed: %v", err) + if _, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}}); err != nil { + t.Fatalf("SuspendActor failed: %v", err) } waitForWorkerAvailable(t, tc, worker) @@ -4148,86 +4197,102 @@ func TestResumeActor_PausedAfterRepointUsesLocalProvenance(t *testing.T) { } } -// TestPauseActor tests the full workflow of pausing a running actor. +// TestSuspendActorAsynchronous tests the full workflow of suspending a running +// actor asynchronously. // Workflow: -// 1. Creates a mock ActorTemplate. -// 2. Creates a mock Atelet Pod on 'node1'. -// 3. Creates a mock worker Pod on 'node1'. -// 4. Waits for the WorkerPoolSyncer to mirror the worker to the store. -// 5. Creates an actor. -// 6. Calls ResumeActor to transition it to RUNNING. -// 7. Calls PauseActor RPC. -// 8. Verifies that the fake Atelet received the Pause call. -func TestPauseActor(t *testing.T) { - ns := namespaceForTest("ns-pause") - tc := setupTest(t, ns) - defer tc.cleanup() +// 1. Creates a mock ActorTemplate. +// 2. Creates a mock Atelet Pod on 'node1'. +// 3. Creates a mock worker Pod on 'node1'. +// 4. Waits for the WorkerPoolSyncer to mirror the worker to the store. +// 5. Creates an actor. +// 6. Calls ResumeActor to transition it to RUNNING. +// 7. Suspends it asynchronously. +// 8. Verifies that the fake Atelet received the Checkpoint call and the +// actor holds a node-local snapshot. +func TestSuspendActorAsynchronous(t *testing.T) { + tests := []struct { + name string + suspend func(ctx context.Context, client ateapipb.ControlClient, ref *ateapipb.ObjectRef) error + }{ + { + name: "SuspendActor asynchronous", + suspend: func(ctx context.Context, client ateapipb.ControlClient, ref *ateapipb.ObjectRef) error { + _, err := client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: ref, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, + }) + return err + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ns := namespaceForTest("ns-pause") + tc := setupTest(t, ns) + defer tc.cleanup() - tmpl := createTemplate(t, tc, ns) + tmpl := createTemplate(t, tc, ns) - createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") + createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") - name := "id1" - _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name}, - ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, - }}) - if err != nil { - t.Fatalf("CreateActor failed: %v", err) - } + name := "id1" + _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }}) + if err != nil { + t.Fatalf("CreateActor failed: %v", err) + } - // Resume first to make it running - _, err = tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, - }) - if err != nil { - t.Fatalf("ResumeActor failed: %v", err) - } + // Resume first to make it running + _, err = tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + }) + if err != nil { + t.Fatalf("ResumeActor failed: %v", err) + } - // Pause - _, err = tc.client.PauseActor(context.Background(), &ateapipb.PauseActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, - }) - if err != nil { - t.Fatalf("PauseActor failed: %v", err) - } + if err := tt.suspend(context.Background(), tc.client, &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}); err != nil { + t.Fatalf("suspend failed: %v", err) + } - if !tc.fakeAtelet.CheckpointCalled { - t.Errorf("expected atelet Checkpoint to be called") - } + if !tc.fakeAtelet.CheckpointCalled { + t.Errorf("expected atelet Checkpoint to be called") + } - getResp, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, - }) - if err != nil { - t.Fatalf("GetActor failed: %v", err) - } - ls := localSnapshot(getResp.GetStatus()) - if ls.GetUuid() == "" { - t.Error("Snapshot.Uuid is empty, want the name the pause checkpointed under") - } - want := &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Name: name, Atespace: testAtespace}, - ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, - Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - AssignedNode: "node1", - LastAssignedGeneration: 2, - Snapshots: []*ateapipb.Snapshot{ - newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), goldenSnapshotName, goldenSnapshotURI(t)), - newLocalSnapshot(2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), "ignored", "node1"), - }, - }, - } + getResp, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + }) + if err != nil { + t.Fatalf("GetActor failed: %v", err) + } + ls := localSnapshot(getResp.GetStatus()) + if ls.GetUuid() == "" { + t.Error("Snapshot.Uuid is empty, want the name the pause checkpointed under") + } + want := &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Name: name, Atespace: testAtespace}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + Status: &ateapipb.ActorStatus{ + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, + AssignedNode: "node1", + LastAssignedGeneration: 2, + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), goldenSnapshotName, goldenSnapshotURI(t)), + newPausedSnapshot(t, 2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), getResp.GetMetadata().GetUid(), ls.GetUuid(), "node1"), + }, + }, + } - if diff := cmp.Diff(want, getResp, - protocmp.Transform(), - ignoreUID, - ignoreVersion, - ignoreTimestamps, - protocmp.IgnoreFields(&ateapipb.Snapshot{}, "uuid"), - ); diff != "" { - t.Errorf("GetActor response mismatch (-want +got):\n%s", diff) + if diff := cmp.Diff(want, getResp, + protocmp.Transform(), + ignoreUID, + ignoreVersion, + ignoreTimestamps, + ); diff != "" { + t.Errorf("GetActor response mismatch (-want +got):\n%s", diff) + } + }) } } @@ -4260,10 +4325,11 @@ func TestResumeActor_PausedLocalSnapshotMissing_Crashes(t *testing.T) { t.Fatalf("ResumeActor failed: %v", err) } - if _, err := tc.client.PauseActor(context.Background(), &ateapipb.PauseActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + if _, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, }); err != nil { - t.Fatalf("PauseActor failed: %v", err) + t.Fatalf("SuspendActor failed: %v", err) } getResp, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ @@ -4319,19 +4385,20 @@ func TestResumeActor_PausedLocalSnapshotMissing_Crashes(t *testing.T) { assertGrpcErrorRegex(t, err, codes.FailedPrecondition, "ACTOR_STATE_CRASHED") } -// Pause stamps the ref identity before resolving the Actor record, so a failed +// Suspend stamps the ref identity before resolving the Actor record, so a failed // lookup still carries who/where; it must not invent template/version, which are // known only once the record resolves (and stamped on success). -func TestPauseActor_FailedLookupStampsRefIdentityOnly(t *testing.T) { - ns := namespaceForTest("ns-span-pause-err") +func TestSuspendActor_FailedLookupStampsRefIdentityOnly(t *testing.T) { + ns := namespaceForTest("ns-span-suspend-err") tc := setupTest(t, ns) defer tc.cleanup() attrs := recordRootSpanAttrs(t, func(ctx context.Context) { - if _, err := tc.service.PauseActor(ctx, &ateapipb.PauseActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: testActorID}, + if _, err := tc.service.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: testActorID}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, }); apierror.Code(err) != codes.NotFound { - t.Fatalf("PauseActor(missing) error = %v, want code NotFound", err) + t.Fatalf("SuspendActor(missing) error = %v, want code NotFound", err) } }) @@ -4339,7 +4406,7 @@ func TestPauseActor_FailedLookupStampsRefIdentityOnly(t *testing.T) { assertSpanStr(t, attrs, ateattr.ActorNameKey, testActorID) for _, k := range []attribute.Key{ateattr.ActorUIDKey, ateattr.TemplateNameKey, ateattr.TemplateAtespaceKey, ateattr.ActorVersionKey} { if _, ok := attrs[k]; ok { - t.Errorf("unexpected %s on failed-pause span", k) + t.Errorf("unexpected %s on failed-suspend span", k) } } } @@ -4846,174 +4913,129 @@ func TestSuspendActor_DanglingWorker(t *testing.T) { } } -// TestSuspendActor_FromPaused suspends a paused actor end-to-end: instead of -// checkpointing a running workload, ateapi asks the atelet on the node -// holding the pause snapshot to upload it, then finalizes the actor with a -// durable ActorSnapshot and no node pinning left behind. +// TestSuspendActor_FromPaused verifies SuspendActor, in either mode, returns a +// paused actor as it is: atelet is not called, and the actor keeps its +// node-local snapshot and stays pinned to its node. func TestSuspendActor_FromPaused(t *testing.T) { - ns := namespaceForTest("ns-suspend-paused") - tc := setupTest(t, ns) - defer tc.cleanup() - - createTemplate(t, tc, ns) - createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") - - name := "id1" - _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name}, - ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, - }}) - if err != nil { - t.Fatalf("CreateActor failed: %v", err) - } - if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, - }); err != nil { - t.Fatalf("ResumeActor failed: %v", err) - } - if _, err := tc.client.PauseActor(context.Background(), &ateapipb.PauseActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, - }); err != nil { - t.Fatalf("PauseActor failed: %v", err) - } - paused, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, - }) - if err != nil { - t.Fatalf("GetActor failed: %v", err) + tests := []struct { + name string + opts *ateapipb.SuspendOptions + }{ + {name: "sync", opts: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}, + {name: "async", opts: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}}, } - // Drop the pause's Checkpoint call so the suspend's atelet traffic is - // observable in isolation. - tc.fakeAtelet.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ns := namespaceForTest("ns-suspend-paused-" + tt.name) + tc := setupTest(t, ns) + defer tc.cleanup() - suspended, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, - }) - if err != nil { - t.Fatalf("SuspendActor failed: %v", err) - } + createTemplate(t, tc, ns) + createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") - if !tc.fakeAtelet.PromoteCalled { - t.Fatal("expected atelet PromoteSnapshot to be called") - } - if tc.fakeAtelet.CheckpointCalled { - t.Error("atelet Checkpoint called for a paused actor; there is no workload to checkpoint") - } - promote := tc.fakeAtelet.PromoteRequest - pausedLS := localSnapshot(paused.GetStatus()) - if pausedLS == nil { - t.Fatal("paused actor has no local snapshot") - } - if got, want := promote.GetSnapshot().GetSnapshotUuid(), pausedLS.GetUuid(); got != want { - t.Errorf("promote snapshot_uuid = %q, want %q", got, want) - } - if got, want := promote.GetSnapshot().GetFidelity(), ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY; got != want { - t.Errorf("promote fidelity = %v, want %v", got, want) - } - if got, want := promote.GetAtespace(), testAtespace; got != want { - t.Errorf("promote atespace = %q, want %q", got, want) - } - if got, want := promote.GetActorName(), name; got != want { - t.Errorf("promote actor_name = %q, want %q", got, want) - } + ref := &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"} + if _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: ref.GetAtespace(), Name: ref.GetName()}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }}); err != nil { + t.Fatalf("CreateActor failed: %v", err) + } + if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: ref}); err != nil { + t.Fatalf("ResumeActor failed: %v", err) + } + if _, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{ + Actor: ref, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, + }); err != nil { + t.Fatalf("pause failed: %v", err) + } + paused, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{Actor: ref}) + if err != nil { + t.Fatalf("GetActor failed: %v", err) + } + if localSnapshot(paused.GetStatus()) == nil { + t.Fatal("paused actor has no local snapshot") + } + // Drop the pause's Checkpoint call so the suspend's atelet traffic is + // observable in isolation. + tc.fakeAtelet.Reset() - actor := suspended.GetActor() - if actor.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { - t.Errorf("state = %v, want SUSPENDED", actor.GetStatus().GetState()) - } - if ls := localSnapshot(actor.GetStatus()); ls != nil { - t.Errorf("local snapshot = %v, want cleared (node pinning must not survive suspend)", ls) - } - if got, want := promote.GetSnapshot().GetObject().GetSnapshotUri(), durableSnapshotURI(actor.GetStatus()); got != want { - t.Errorf("promote snapshot_uri = %q, want %q", got, want) - } - if got, want := snapshotName(t, durableSnapshotURI(actor.GetStatus())), pausedLS.GetUuid(); got != want { - t.Errorf("snapshot name = %q, want the pause snapshot uuid %q", got, want) - } - // The suspend promotes the pause snapshot rather than taking a new one. - durable := durableSnapshot(actor.GetStatus()) - if durable.GetGeneration() != pausedLS.GetGeneration() || durable.GetUuid() != pausedLS.GetUuid() { - t.Errorf("durable snapshot = generation %d, uuid %q; want the pause snapshot's generation %d, uuid %q", durable.GetGeneration(), durable.GetUuid(), pausedLS.GetGeneration(), pausedLS.GetUuid()) - } - if got := durable.GetFidelity(); got != ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { - t.Errorf("snapshot Fidelity = %v, want FULL", got) + suspended, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: ref, Options: tt.opts}) + if err != nil { + t.Fatalf("SuspendActor failed: %v", err) + } + if tc.fakeAtelet.CheckpointCalled || tc.fakeAtelet.PromoteCalled { + t.Errorf("atelet called (Checkpoint %t, PromoteSnapshot %t), want neither for a paused actor", tc.fakeAtelet.CheckpointCalled, tc.fakeAtelet.PromoteCalled) + } + if diff := cmp.Diff(paused, suspended.GetActor(), protocmp.Transform()); diff != "" { + t.Errorf("SuspendActor changed the paused actor (-paused +returned):\n%s", diff) + } + }) } } -// TestSuspendActor_FromPaused_UploadFailureCrashes: an atelet error while -// uploading the paused snapshot crashes the actor like any other atelet -// error, releasing its worker and making it unsuspendable. -func TestSuspendActor_FromPaused_UploadFailureCrashes(t *testing.T) { - ns := namespaceForTest("ns-suspend-paused-crash") - tc := setupTest(t, ns) - defer tc.cleanup() - - createTemplate(t, tc, ns) - podUID := createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") - - name := "id1" - _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name}, - ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, - }}) - if err != nil { - t.Fatalf("CreateActor failed: %v", err) - } - if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, - }); err != nil { - t.Fatalf("ResumeActor failed: %v", err) - } - if _, err := tc.client.PauseActor(context.Background(), &ateapipb.PauseActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, - }); err != nil { - t.Fatalf("PauseActor failed: %v", err) - } - - tc.fakeAtelet.Reset() - tc.fakeAtelet.FailPromote = status.Error(codes.Internal, "injected upload failure") - _, err = tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, - }) - if err == nil { - t.Fatal("SuspendActor succeeded despite failing upload") - } - // Atelet's code does not reach the caller. - if got := status.Code(err); got != codes.Internal { - t.Errorf("status code = %v, want %v (err: %v)", got, codes.Internal, err) +// TestSuspendActor_Mode verifies the snapshot a running actor's suspend leaves +// in each mode. An unset mode suspends asynchronously. +func TestSuspendActor_Mode(t *testing.T) { + tests := []struct { + name string + opts *ateapipb.SuspendOptions + wantUploadMode ateletpb.UploadMode + wantLocal bool + // wantNode is the node an async suspend keeps the actor pinned to. + wantNode string + }{ + {name: "unset", opts: nil, wantUploadMode: ateletpb.UploadMode_UPLOAD_MODE_DEFERRED, wantLocal: true, wantNode: "node1"}, + {name: "unspecified", opts: &ateapipb.SuspendOptions{}, wantUploadMode: ateletpb.UploadMode_UPLOAD_MODE_DEFERRED, wantLocal: true, wantNode: "node1"}, + {name: "async", opts: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, wantUploadMode: ateletpb.UploadMode_UPLOAD_MODE_DEFERRED, wantLocal: true, wantNode: "node1"}, + {name: "sync", opts: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, wantUploadMode: ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE, wantLocal: false}, } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ns := namespaceForTest("ns-suspend-mode-" + tt.name) + tc := setupTest(t, ns) + defer tc.cleanup() - crashed, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, - }) - if err != nil { - t.Fatalf("GetActor failed: %v", err) - } - if crashed.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_CRASHED { - t.Fatalf("state after failed upload = %v, want CRASHED", crashed.GetStatus().GetState()) - } - if crashed.GetStatus().GetWorkerAssignment() != nil { - t.Errorf("expected worker assignment to be cleared, got %v", crashed.GetStatus().GetWorkerAssignment()) - } - assertActorCrashStatus(t, tc, name, "suspend failed: atelet PromoteSnapshot: injected upload failure") + createTemplate(t, tc, ns) + createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") - worker, err := tc.persistence.GetWorker(context.Background(), podUID) - if err != nil { - t.Fatalf("GetWorker(%s) failed: %v", podUID, err) - } - if n := worker.GetStatus().GetAllocated().GetActors(); n != 0 { - t.Errorf("expected worker to be released after crash, still holds %d actors", n) - } + ref := &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"} + if _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: ref.GetAtespace(), Name: ref.GetName()}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }}); err != nil { + t.Fatalf("CreateActor failed: %v", err) + } + if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: ref}); err != nil { + t.Fatalf("ResumeActor failed: %v", err) + } - // The crash is terminal: a healthy atelet does not make the actor suspendable. - tc.fakeAtelet.Lock.Lock() - tc.fakeAtelet.FailPromote = nil - tc.fakeAtelet.Lock.Unlock() - _, err = tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, - }) - if status.Code(err) != codes.FailedPrecondition || !strings.Contains(err.Error(), "ACTOR_STATE_CRASHED") { - t.Errorf("expected FailedPrecondition/ACTOR_STATE_CRASHED error, got %v", err) + suspended, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: ref, Options: tt.opts}) + if err != nil { + t.Fatalf("SuspendActor failed: %v", err) + } + tc.fakeAtelet.Lock.Lock() + gotUploadMode := tc.fakeAtelet.CheckpointRequest.GetUploadConfig().GetMode() + tc.fakeAtelet.Lock.Unlock() + if gotUploadMode != tt.wantUploadMode { + t.Errorf("Checkpoint upload mode = %v, want %v", gotUploadMode, tt.wantUploadMode) + } + status := suspended.GetActor().GetStatus() + if got := status.GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { + t.Errorf("state = %v, want %v", got, ateapipb.ActorState_ACTOR_STATE_SUSPENDED) + } + // A sync suspend leaves a durable snapshot, an async one a node-local + // snapshot whose upload is still pending. + if gotLocal := localSnapshot(status) != nil; gotLocal != tt.wantLocal { + t.Errorf("has local snapshot = %t, want %t", gotLocal, tt.wantLocal) + } + if gotDurable := durableSnapshot(status).GetOwner() == ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR; gotDurable == tt.wantLocal { + t.Errorf("has completed durable snapshot = %t, want %t", gotDurable, !tt.wantLocal) + } + if got := status.GetAssignedNode(); got != tt.wantNode { + t.Errorf("assigned node = %q, want %q", got, tt.wantNode) + } + }) } } @@ -5029,7 +5051,7 @@ func TestCheckpointFailureCrashes(t *testing.T) { { name: "pause", call: func(tc *testContext, ref *ateapipb.ObjectRef) error { - _, err := tc.client.PauseActor(context.Background(), &ateapipb.PauseActorRequest{Actor: ref}) + _, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: ref, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}}) return err }, wantMessage: "suspend failed: atelet Checkpoint: injected checkpoint failure", @@ -5037,7 +5059,7 @@ func TestCheckpointFailureCrashes(t *testing.T) { { name: "suspend", call: func(tc *testContext, ref *ateapipb.ObjectRef) error { - _, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: ref}) + _, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: ref, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}) return err }, wantMessage: "suspend failed: atelet Checkpoint: injected checkpoint failure", @@ -5097,12 +5119,12 @@ func TestCheckpointFailureCrashes(t *testing.T) { } } -// TestResumeActor_RelocatesAfterSuspendFromPaused covers the capacity-recovery -// flow: a paused actor is pinned to the node holding its local snapshot, so it -// cannot resume while that node is full. Suspending it uploads the snapshot and -// drops the node pinning, after which it is scheduled onto a worker on a different -// node. -func TestResumeActor_RelocatesAfterSuspendFromPaused(t *testing.T) { +// TestResumeActor_RelocatesAfterDetach covers the capacity-recovery flow: a +// paused actor is pinned to the node holding its local snapshot, so it cannot +// resume while that node is full. Once atelet reports the snapshot uploaded and +// detached from the node, the pin is dropped and the actor is scheduled onto a +// worker on a different node. +func TestResumeActor_RelocatesAfterDetach(t *testing.T) { ns := namespaceForTest("ns-resume-relocate") tc := setupTest(t, ns) defer tc.cleanup() @@ -5127,10 +5149,11 @@ func TestResumeActor_RelocatesAfterSuspendFromPaused(t *testing.T) { }); err != nil { t.Fatalf("ResumeActor(%s) failed: %v", pinned, err) } - if _, err := tc.client.PauseActor(context.Background(), &ateapipb.PauseActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: pinned}, + if _, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: pinned}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, }); err != nil { - t.Fatalf("PauseActor(%s) failed: %v", pinned, err) + t.Fatalf("SuspendActor(%s) failed: %v", pinned, err) } paused, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: pinned}, @@ -5159,25 +5182,38 @@ func TestResumeActor_RelocatesAfterSuspendFromPaused(t *testing.T) { }) assertGrpcError(t, err, codes.ResourceExhausted, "no worker has room for the actor") - suspended, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{ + // node1's atelet reports the paused snapshot uploaded and detached: a report + // with only latest_durable. + ls := localSnapshot(paused.GetStatus()) + if err := tc.service.ApplyActorSnapshotState(context.Background(), resources.ActorRefFromActor(paused), paused.GetMetadata().GetUid(), "node1", &ateapipb.ActorSnapshotState{ + LatestDurable: &ateapipb.SnapshotReport{ + Uuid: ls.GetUuid(), + Generation: ls.GetGeneration(), + Fidelity: ls.GetFidelity(), + Object: ls.GetDurableSnapshot().GetObject(), + }, + }); err != nil { + t.Fatalf("ApplyActorSnapshotState(%s) failed: %v", pinned, err) + } + detached, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: pinned}, }) if err != nil { - t.Fatalf("SuspendActor(%s) failed: %v", pinned, err) - } - if got := suspended.GetActor().GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { - t.Fatalf("state after suspend = %v, want SUSPENDED", got) + t.Fatalf("GetActor(%s) failed: %v", pinned, err) } - if ls := localSnapshot(suspended.GetActor().GetStatus()); ls != nil { + if ls := localSnapshot(detached.GetStatus()); ls != nil { t.Fatalf("local snapshot = %v, want cleared so the actor can be scheduled anywhere", ls) } + if got := detached.GetStatus().GetAssignedNode(); got != "" { + t.Fatalf("detached actor pinned to %q, want no node", got) + } // Resume should succeed now and the actor scheduled on node2. resumed, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: pinned}, }) if err != nil { - t.Fatalf("ResumeActor(%s) after suspend failed: %v", pinned, err) + t.Fatalf("ResumeActor(%s) after detach failed: %v", pinned, err) } if got := resumed.GetActor().GetStatus().GetWorkerAssignment().GetWorkerPod(); got != "worker-2" { t.Errorf("resumed onto worker %q, want worker-2 (the worker on node2)", got) @@ -5207,7 +5243,7 @@ func TestLifecycleOpPoolAttributesOnSuccess(t *testing.T) { op: ateattr.OperationSuspend, run: func(t *testing.T, tc *testContext, actor *ateapipb.ObjectRef) { t.Helper() - if _, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: actor}); err != nil { + if _, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: actor, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}); err != nil { t.Fatalf("SuspendActor failed: %v", err) } }, @@ -5217,8 +5253,8 @@ func TestLifecycleOpPoolAttributesOnSuccess(t *testing.T) { op: ateattr.OperationSuspend, run: func(t *testing.T, tc *testContext, actor *ateapipb.ObjectRef) { t.Helper() - if _, err := tc.client.PauseActor(context.Background(), &ateapipb.PauseActorRequest{Actor: actor}); err != nil { - t.Fatalf("PauseActor failed: %v", err) + if _, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: actor, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}}); err != nil { + t.Fatalf("SuspendActor failed: %v", err) } }, }, @@ -5478,7 +5514,7 @@ func TestRevertActor(t *testing.T) { if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { t.Fatalf("ResumeActor failed: %v", err) } - suspended, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef}) + suspended, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}) if err != nil { t.Fatalf("SuspendActor failed: %v", err) } @@ -5546,7 +5582,7 @@ func TestRevertActor_FromPaused(t *testing.T) { if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { t.Fatalf("ResumeActor failed: %v", err) } - suspended, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef}) + suspended, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}) if err != nil { t.Fatalf("SuspendActor failed: %v", err) } @@ -5559,8 +5595,8 @@ func TestRevertActor_FromPaused(t *testing.T) { if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { t.Fatalf("second ResumeActor failed: %v", err) } - if _, err := tc.client.PauseActor(ctx, &ateapipb.PauseActorRequest{Actor: actorRef}); err != nil { - t.Fatalf("PauseActor failed: %v", err) + if _, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}}); err != nil { + t.Fatalf("SuspendActor failed: %v", err) } waitForWorkerAvailable(t, tc, workerName) tc.fakeAtelet.Lock.Lock() @@ -5624,7 +5660,7 @@ func TestRevertActor_FromCrashed(t *testing.T) { if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { t.Fatalf("ResumeActor failed: %v", err) } - suspended, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef}) + suspended, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}) if err != nil { t.Fatalf("SuspendActor failed: %v", err) } @@ -5983,7 +6019,7 @@ func reportWorkerRuntime(t *testing.T, tc *testContext, workerName string, defau // 4. When the other worker advertises the snapshot's runtime in // restorable_runtimes, ResumeActor restores at full MEMORY fidelity, // records the matched restorable runtime on WorkerAssignment, and preserves -// that stamp through PauseActor and SuspendActor-from-PAUSED. +// that stamp through async SuspendActor and sync SuspendActor-from-PAUSED. func TestResumeActor_SandboxRuntimePlacementAndFallback(t *testing.T) { ns := namespaceForTest("ns-runtime-fallback") tc := setupTest(t, ns) @@ -5998,7 +6034,7 @@ func TestResumeActor_SandboxRuntimePlacementAndFallback(t *testing.T) { SchemaVersion: "v1", Attributes: []*ateapipb.AttributeEntry{ {Key: "architecture", Value: "amd64"}, - {Key: "gvisor_asset_hash", Value: "hash-a"}, + {Key: "sandbox_bundle_hash", Value: "hash-a"}, }, }, } @@ -6008,7 +6044,7 @@ func TestResumeActor_SandboxRuntimePlacementAndFallback(t *testing.T) { SchemaVersion: "v1", Attributes: []*ateapipb.AttributeEntry{ {Key: "architecture", Value: "amd64"}, - {Key: "gvisor_asset_hash", Value: "hash-b"}, + {Key: "sandbox_bundle_hash", Value: "hash-b"}, }, }, } @@ -6033,7 +6069,7 @@ func TestResumeActor_SandboxRuntimePlacementAndFallback(t *testing.T) { t.Fatalf("initial WorkerAssignment.SandboxRuntime = %v, want %v", got, rtA) } - suspended1, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef}) + suspended1, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}) if err != nil { t.Fatalf("first SuspendActor failed: %v", err) } @@ -6061,7 +6097,7 @@ func TestResumeActor_SandboxRuntimePlacementAndFallback(t *testing.T) { t.Fatalf("second RestoreRequest.Fidelity = %v, want MEMORY", got) } - if _, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef}); err != nil { + if _, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}); err != nil { t.Fatalf("second SuspendActor failed: %v", err) } waitForWorkerAvailable(t, tc, workerA) @@ -6115,7 +6151,7 @@ func TestResumeActor_SandboxRuntimePlacementAndFallback(t *testing.T) { t.Fatalf("fallback RestoreRequest.Fidelity = %v, want VOLUMES", got) } - suspended3, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef}) + suspended3, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}) if err != nil { t.Fatalf("SuspendActor after fallback failed: %v", err) } @@ -6131,8 +6167,8 @@ func TestResumeActor_SandboxRuntimePlacementAndFallback(t *testing.T) { goldenSnap := newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), goldenSnapshotName, goldenSnapshotURI(t)) goldenSnap.SandboxRuntime = proto.Clone(rtA).(*ateapipb.SandboxRuntime) stampedGoldenTag, err := tc.persistence.CreateTag(ctx, &ateapipb.Tag{ - Metadata: &ateapipb.ResourceMetadata{Atespace: resources.GoldenActorAtespace, Name: "golden-rt-a"}, - SourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: tmpl.GetMetadata().GetUid()}, + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "golden-rt-a"}, + SourceActor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: tmpl.GetMetadata().GetUid()}, Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapipb.TagStatus{ Snapshot: goldenSnap, @@ -6199,7 +6235,7 @@ func TestResumeActor_SandboxRuntimePlacementAndFallback(t *testing.T) { // keeping rtB as its default. Resuming an actor whose snapshot requires rtA // onto worker-b (with worker-a still full) must restore at MEMORY fidelity, // record the matched restorable entry (rtA, not rtB) on WorkerAssignment, - // and preserve rtA through PauseActor and SuspendActor-from-PAUSED. + // and preserve rtA through async SuspendActor and sync SuspendActor-from-PAUSED. if _, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: goldenActorRef}); err != nil { t.Fatalf("SuspendActor(golden-actor) failed: %v", err) } @@ -6228,20 +6264,22 @@ func TestResumeActor_SandboxRuntimePlacementAndFallback(t *testing.T) { t.Fatalf("restorable-actor RestoreRequest.Fidelity = %v, want MEMORY", got) } - pausedResp, err := tc.client.PauseActor(ctx, &ateapipb.PauseActorRequest{Actor: restorableRef}) + pausedResp, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: restorableRef, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}}) if err != nil { - t.Fatalf("PauseActor(restorable-actor) failed: %v", err) + t.Fatalf("SuspendActor(restorable-actor) failed: %v", err) } waitForWorkerAvailable(t, tc, workerB) if got := localSnapshot(pausedResp.GetActor().GetStatus()).GetSandboxRuntime(); !proto.Equal(got, rtA) { t.Fatalf("paused local Snapshot.SandboxRuntime = %v, want %v", got, rtA) } - suspendedFromPaused, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: restorableRef}) + // A sync suspend of the paused actor returns it as it is: the node-local + // snapshot keeps its runtime stamp. + suspendedFromPaused, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: restorableRef, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}) if err != nil { t.Fatalf("SuspendActor from PAUSED(restorable-actor) failed: %v", err) } - if got := durableSnapshot(suspendedFromPaused.GetActor().GetStatus()).GetSandboxRuntime(); !proto.Equal(got, rtA) { - t.Fatalf("durable Snapshot.SandboxRuntime after SuspendActor from PAUSED = %v, want %v", got, rtA) + if got := localSnapshot(suspendedFromPaused.GetActor().GetStatus()).GetSandboxRuntime(); !proto.Equal(got, rtA) { + t.Fatalf("local Snapshot.SandboxRuntime after SuspendActor from PAUSED = %v, want %v", got, rtA) } } diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index 4454282ed8..a7978c0db3 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -239,6 +239,7 @@ func setupTestWithOptions(t *testing.T, ns string, options testOptions) *testCon service := controlapi.NewRPCService( persistence, wc, + workerPoolLister, sandboxConfigLister, csiDriverConfigLister, scLister, @@ -381,7 +382,7 @@ const goldenSnapshotName = "9c2f7b41-6d05-4e83-a1f7-3b8c0d5e2a94" // goldenSnapshotURI is the snapshot owned by the test template's golden tag. func goldenSnapshotURI(t *testing.T) string { t.Helper() - uri, err := resources.NewTagSnapshotURI(testStorageLocation, resources.GoldenActorAtespace, goldenSnapshotName) + uri, err := resources.NewTagSnapshotURI(testStorageLocation, testAtespace, goldenSnapshotName) if err != nil { t.Fatalf("NewTagSnapshotURI: %v", err) } @@ -470,9 +471,9 @@ func createTemplateWithVolumes(t *testing.T, tc *testContext, ns string, volumes func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns string, containers []*ateapipb.Container, volumes []*ateapipb.Volume) *ateapipb.ActorTemplate { t.Helper() - // Sandbox binaries live on a (cluster-scoped) SandboxConfig the template - // names. Create the gvisor-default SandboxConfig so a boot-from-spec Run - // can resolve its assets. + // Sandbox binaries live on the (cluster-scoped) SandboxConfig the + // actor's WorkerPool resolves. Create the gvisor-default SandboxConfig so + // a boot-from-spec Run can resolve its assets. ensureDefaultGvisorSandboxConfig(t, tc) createWorkerPool(t, tc, ns, "pool1", map[string]string{poolLabelKey: ns}) @@ -485,10 +486,6 @@ func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns st SnapshotConfig: &ateapipb.SnapshotConfig{ StorageLocation: testStorageLocation, }, - SandboxConfig: &ateapipb.SandboxConfig{ - SandboxClass: "gvisor", - ConfigName: "gvisor-default", - }, Containers: containers, Volumes: volumes, WorkerSelector: &ateapipb.Selector{ @@ -501,10 +498,9 @@ func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns st t.Fatalf("failed to create actor template: %v", err) } - createAtespace(t, tc, resources.GoldenActorAtespace) tag, err := tc.persistence.CreateTag(context.Background(), &ateapipb.Tag{ - Metadata: &ateapipb.ResourceMetadata{Atespace: resources.GoldenActorAtespace, Name: created.GetMetadata().GetUid()}, - SourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: created.GetMetadata().GetUid()}, + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: created.GetMetadata().GetUid()}, + SourceActor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: created.GetMetadata().GetUid()}, Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapipb.TagStatus{ Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, created.GetMetadata().GetUid(), goldenSnapshotName, goldenSnapshotURI(t)), @@ -548,7 +544,14 @@ func newDurableSnapshot(gen int32, owner ateapipb.SnapshotOwner, fidelity ateapi } } -func newLocalSnapshot(gen int32, fidelity ateapipb.SnapshotFidelity, templateUID, uuid, locality string) *ateapipb.Snapshot { +// newPausedSnapshot is the snapshot a pause leaves on node locality: its +// durable copy, under the actor's own prefix, is PENDING until atelet uploads it. +func newPausedSnapshot(t *testing.T, gen int32, fidelity ateapipb.SnapshotFidelity, templateUID, actorUID, uuid, locality string) *ateapipb.Snapshot { + t.Helper() + uri, err := resources.NewActorSnapshotURI(testStorageLocation, testAtespace, actorUID, uuid) + if err != nil { + t.Fatalf("NewActorSnapshotURI: %v", err) + } return &ateapipb.Snapshot{ Uuid: uuid, Generation: gen, @@ -556,6 +559,10 @@ func newLocalSnapshot(gen int32, fidelity ateapipb.SnapshotFidelity, templateUID Fidelity: fidelity, ActorTemplateUid: templateUID, Locality: locality, + DurableSnapshot: &ateapipb.SnapshotStorage{ + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: uri.String()}, + }, } } @@ -621,13 +628,13 @@ func ensureGvisorSandboxConfig(t *testing.T, tc *testContext, name string) { Name: "v1", PauseImage: testPauseImage, Assets: map[string]map[string]atev1alpha1.AssetFile{ - "amd64": {"runsc": { - URL: "gs://gvisor/releases/nightly/2026-05-19/x86_64/runsc", - SHA256: "a397be1abc2420d26bce6c70e6e2ff96c73aaaab929756c56f5e2089ea842b63", + "amd64": {"gvisor": { + URL: "gs://gvisor/releases/nightly/2026-10-10/x86_64/gvisor.tar.zstd", + SHA256: "4ae7fab988c3647d2a835d6c3c70c6a99021c562db97827592562b6c51c3e88c", }}, - "arm64": {"runsc": { - URL: "gs://gvisor/releases/nightly/2026-05-19/aarch64/runsc", - SHA256: "1ba2366ae2efceba166046f51a4104f9261c9cb72c6db8f5b3fe2dc57dea86b9", + "arm64": {"gvisor": { + URL: "gs://gvisor/releases/nightly/2026-10-10/aarch64/gvisor.tar.zstd", + SHA256: "aaf414f06eea7960238e43165c691f0a36cefc0593fb141e011e96211c7b6f57", }}, }, }}, @@ -658,22 +665,32 @@ func createWorkerPool(t *testing.T, tc *testContext, ns string, name string, lab SandboxClasses: []atev1alpha1.WorkerPoolSandboxClass{{Name: atev1alpha1.SandboxClassGvisor}}, }, } - _, err := tc.substrateClient.ApiV1alpha1().WorkerPools(ns).Create(context.Background(), wp, metav1.CreateOptions{}) + created, err := tc.substrateClient.ApiV1alpha1().WorkerPools(ns).Create(context.Background(), wp, metav1.CreateOptions{}) if err != nil { t.Fatalf("failed to create WorkerPool: %v", err) } + // The status the controller would write once it confirmed the pool's + // SandboxConfig: resume resolves the sandbox assets through it. Status is + // a subresource, so it takes its own write. + created.Status.SandboxClasses = []atev1alpha1.WorkerPoolSandboxClassStatus{{ + Name: atev1alpha1.SandboxClassGvisor, + ConfigRef: &atev1alpha1.SandboxConfigReference{Name: "gvisor-default"}, + }} + if _, err := tc.substrateClient.ApiV1alpha1().WorkerPools(ns).UpdateStatus(context.Background(), created, metav1.UpdateOptions{}); err != nil { + t.Fatalf("failed to update WorkerPool status: %v", err) + } waitForWorkerPoolInInformer(t, tc, ns, name) } -// waitForWorkerPoolInInformer blocks until the pool the test just created is -// visible to the scheduler, which reads it through an informer rather than the -// API. +// waitForWorkerPoolInInformer blocks until the pool the test just created, +// with its resolved SandboxConfig, is visible to the scheduler and the resume +// workflow, which read it through an informer rather than the API. func waitForWorkerPoolInInformer(t *testing.T, tc *testContext, ns, name string) { t.Helper() err := wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { - _, err := tc.workerPoolLister.WorkerPools(ns).Get(name) - return err == nil, nil + pool, err := tc.workerPoolLister.WorkerPools(ns).Get(name) + return err == nil && pool.ResolvedSandboxConfig() != "", nil }) if err != nil { t.Fatalf("failed to wait for WorkerPool %s/%s in informer: %v", ns, name, err) @@ -694,10 +711,6 @@ func createTemplateWithSelector(t *testing.T, tc *testContext, name string, sele SnapshotConfig: &ateapipb.SnapshotConfig{ StorageLocation: testStorageLocation, }, - SandboxConfig: &ateapipb.SandboxConfig{ - SandboxClass: "gvisor", - ConfigName: "gvisor-default", - }, Containers: []*ateapipb.Container{ {Name: "main", Image: "main@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", Command: []string{"/main"}}, }, diff --git a/cmd/ateapi/internal/controlapi/functionaltest/main_test.go b/cmd/ateapi/internal/controlapi/functionaltest/main_test.go index 9ea0ae0df7..0444467da0 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/main_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/main_test.go @@ -20,11 +20,13 @@ import ( "log" "net" "os" + "strconv" "sync" "testing" "time" "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/internal/atelet" "go.ate.dev/substrate/internal/objectstore/objectstoretest" "go.ate.dev/substrate/internal/proto/ateletpb" "go.ate.dev/substrate/internal/serverboot" @@ -85,12 +87,13 @@ func TestMain(m *testing.M) { log.Fatalf("%v", err) } - // Start Fake Atelet Server on port 8085 + // Start Fake Atelet Server on the port ateapi dials. ateletGrpcServer := grpc.NewServer() ateletpb.RegisterAteletServer(ateletGrpcServer, fakeAtelet) - ateletLis, err := net.Listen("tcp", "127.0.0.1:8085") + ateletAddr := net.JoinHostPort("127.0.0.1", strconv.Itoa(atelet.DefaultPort)) + ateletLis, err := net.Listen("tcp", ateletAddr) if err != nil { - log.Fatalf("listen on 127.0.0.1:8085: %v", err) + log.Fatalf("listen on %s: %v", ateletAddr, err) } go func() { if err := ateletGrpcServer.Serve(ateletLis); err != nil { @@ -275,3 +278,12 @@ func (f *FakeAteletServer) lastRestoreRequest() *ateletpb.RestoreRequest { } return proto.Clone(f.RestoreRequest).(*ateletpb.RestoreRequest) } + +func (f *FakeAteletServer) lastRestoreVolumePublishContext(volumeName string) map[string]string { + for _, v := range f.lastRestoreRequest().GetSpec().GetVolumes() { + if v.GetName() == volumeName { + return v.GetExternal().GetPublishContext() + } + } + return nil +} diff --git a/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go b/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go index 4972f12fe7..8f2ca73aa4 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go @@ -172,7 +172,8 @@ func suspendActorForTest(t *testing.T, tc *testContext, workerName, name string) t.Fatalf("ResumeActor(%s) failed: %v", name, err) } suspended, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, }) if err != nil { t.Fatalf("SuspendActor(%s) failed: %v", name, err) diff --git a/cmd/ateapi/internal/controlapi/hosting.go b/cmd/ateapi/internal/controlapi/hosting.go index 852fb8c149..a0c93b1f6f 100644 --- a/cmd/ateapi/internal/controlapi/hosting.go +++ b/cmd/ateapi/internal/controlapi/hosting.go @@ -29,9 +29,9 @@ import ( // worker (delete its row in the store's worker assignments) before they clear // status.worker_assignment, and a crash keeps status.assigned_node as a record // of the last node. The rules: -// - SUSPENDED or SUSPENDING with no worker and a local-only latest -// snapshot, the shape of a paused actor: assigned_node, the node holding -// that snapshot. +// - SUSPENDED with no worker: assigned_node. It is set only while a +// snapshot whose locality is that node still holds files there, uploaded +// or not, and cleared once none does. // - RESUMING, RUNNING and SUSPENDING with status.worker_assignment set: the // worker's node, but only while the store's worker assignment row binds // that worker to this actor's UID; "" once it is released or the worker @@ -44,13 +44,10 @@ func HostingNode(ctx context.Context, st store.Interface, actor *ateapipb.Actor) status := actor.GetStatus() assignment := status.GetWorkerAssignment() if assignment == nil { - switch status.GetState() { - case ateapipb.ActorState_ACTOR_STATE_SUSPENDED, ateapipb.ActorState_ACTOR_STATE_SUSPENDING: - if latestSnapshotIsLocalOnly(status) { - return status.GetAssignedNode(), nil - } + if status.GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { + return "", nil } - return "", nil + return status.GetAssignedNode(), nil } switch status.GetState() { case ateapipb.ActorState_ACTOR_STATE_RESUMING, ateapipb.ActorState_ACTOR_STATE_RUNNING, diff --git a/cmd/ateapi/internal/controlapi/hosting_test.go b/cmd/ateapi/internal/controlapi/hosting_test.go index 0480a8c2b1..2e6556bf1c 100644 --- a/cmd/ateapi/internal/controlapi/hosting_test.go +++ b/cmd/ateapi/internal/controlapi/hosting_test.go @@ -59,7 +59,7 @@ func TestHostingNodeAfterWorkerRelease(t *testing.T) { WorkerPod: wa.GetWorkerPod(), WorkerPodUid: wa.GetWorkerPodUid(), NodeName: node, - SandboxClass: tc.WorkerClass, + SandboxClass: "gvisor", Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}}, }); err != nil { t.Fatalf("CreateWorker: %v", err) @@ -176,6 +176,7 @@ func TestHostingNodeAfterResumeOntoNewWorker(t *testing.T) { WorkerPodUid: testWorkerUID("free-pod"), NodeName: "node2", SandboxClass: "gvisor", + Labels: map[string]string{"pool": "b"}, Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}}, } for _, w := range []*ateapipb.Worker{stale, free} { @@ -195,7 +196,9 @@ func TestHostingNodeAfterResumeOntoNewWorker(t *testing.T) { t.Fatalf("workercache.Start: %v", err) } w := &ActorWorkflow{store: persistence, workerCache: wc, scheduler: scheduling.New(wc)} - tmpl := &ateapipb.ActorTemplate{SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor"}} + tmpl := &ateapipb.ActorTemplate{ + WorkerSelector: &ateapipb.Selector{MatchLabels: map[string]string{"pool": "b"}}, + } resuming, _, err := w.assignWorkerAttempt(ctx, actorRef, actor, tmpl) if err != nil { t.Fatalf("assignWorkerAttempt: %v", err) @@ -219,3 +222,45 @@ func TestHostingNodeAfterResumeOntoNewWorker(t *testing.T) { t.Errorf("HostingNode naming the released stale worker = %q, %v; want \"\", nil", got, err) } } + +// TestHostingNodeWithoutWorker pins which no-worker shapes count assigned_node +// as hosting the actor. None of them reads the store. +func TestHostingNodeWithoutWorker(t *testing.T) { + durable := func(st ateapipb.SnapshotStorageStatus) *ateapipb.SnapshotStorage { + return &ateapipb.SnapshotStorage{Status: st, Object: &ateapipb.ObjectSnapshot{SnapshotUri: "gs://snaps/root/atespaces/a/actors/uid/snap-1"}} + } + status := func(state ateapipb.ActorState, locality string, d *ateapipb.SnapshotStorage) *ateapipb.ActorStatus { + return &ateapipb.ActorStatus{ + State: state, + AssignedNode: "node-1", + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{{Uuid: "snap-1", Generation: 1, Owner: ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, Locality: locality, DurableSnapshot: d}}, + } + } + olderOnNode := status(ateapipb.ActorState_ACTOR_STATE_SUSPENDED, "node-1", durable(ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED)) + olderOnNode.LastAssignedGeneration = 2 + olderOnNode.Snapshots = append(olderOnNode.Snapshots, &ateapipb.Snapshot{Uuid: "snap-2", Generation: 2, Owner: ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, DurableSnapshot: durable(ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED)}) + noAssignedNode := status(ateapipb.ActorState_ACTOR_STATE_SUSPENDED, "node-1", nil) + noAssignedNode.AssignedNode = "" + for _, tc := range []struct { + name string + status *ateapipb.ActorStatus + want string + }{ + {"paused", status(ateapipb.ActorState_ACTOR_STATE_SUSPENDED, "node-1", nil), "node-1"}, + {"paused, upload pending", status(ateapipb.ActorState_ACTOR_STATE_SUSPENDED, "node-1", durable(ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING)), "node-1"}, + {"paused, uploaded, local copy kept", status(ateapipb.ActorState_ACTOR_STATE_SUSPENDED, "node-1", durable(ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED)), "node-1"}, + {"older snapshot on the node, latest not", olderOnNode, "node-1"}, + {"local copy but no assigned node", noAssignedNode, ""}, + {"suspending without a worker", status(ateapipb.ActorState_ACTOR_STATE_SUSPENDING, "node-1", durable(ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING)), ""}, + {"crashed with a local copy", status(ateapipb.ActorState_ACTOR_STATE_CRASHED, "node-1", nil), ""}, + } { + t.Run(tc.name, func(t *testing.T) { + actor := &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: "a", Name: "n", Uid: "uid"}, Status: tc.status} + got, err := HostingNode(context.Background(), nil, actor) + if err != nil || got != tc.want { + t.Fatalf("HostingNode = %q, %v, want %q", got, err, tc.want) + } + }) + } +} diff --git a/cmd/ateapi/internal/controlapi/metrics.go b/cmd/ateapi/internal/controlapi/metrics.go index 67d898e185..f523743627 100644 --- a/cmd/ateapi/internal/controlapi/metrics.go +++ b/cmd/ateapi/internal/controlapi/metrics.go @@ -202,13 +202,14 @@ func (i *Instruments) recordLifecycleOp(ctx context.Context, op string, start ti } // lifecycleOpAttrs builds the resume/suspend/pause/revert dimensions from -// workflow state. Nil-safe, and omits the template, pool, snapshot-kind and -// snapshot-fidelity labels until they are known so a failure before the -// load/assign/restore steps never emits an empty-string series. snapshotKind is empty for suspend/pause, which do not -// restore; snapshotScope applies to all three and separates a full restore -// from a data-only one of the same kind. +// workflow state. Nil-safe, and omits the template, pool, sandbox-class, +// snapshot-kind and snapshot-fidelity labels until they are known so a failure +// before the load/assign/restore steps never emits an empty-string series. +// snapshotKind is empty for suspend/pause, which do not restore; snapshotScope +// applies to all three and separates a full restore from a data-only one of the +// same kind. // The pool keys are set together or not at all; see ateattr.WorkerPoolAttributes. -func lifecycleOpAttrs(actor *ateapipb.Actor, template *ateapipb.ActorTemplate, snapshotKind, snapshotScope string) []attribute.KeyValue { +func lifecycleOpAttrs(actor *ateapipb.Actor, snapshotKind, snapshotScope string) []attribute.KeyValue { var attrs []attribute.KeyValue if actor != nil { attrs = append(attrs, @@ -218,8 +219,12 @@ func lifecycleOpAttrs(actor *ateapipb.Actor, template *ateapipb.ActorTemplate, s } ass := actor.GetStatus().GetWorkerAssignment() attrs = append(attrs, ateattr.WorkerPoolAttributes(ass.GetWorkerNamespace(), ass.GetWorkerPool())...) - if template != nil { - attrs = append(attrs, ateattr.SandboxClassAttribute(template.GetSandboxConfig().GetSandboxClass())) + class := ass.GetSandboxRuntime().GetSandboxClass() + if class == "" { + class = snapshotAtLatestGeneration(actor.GetStatus()).GetSandboxRuntime().GetSandboxClass() + } + if class != "" { + attrs = append(attrs, ateattr.SandboxClassAttribute(class)) } if snapshotKind != "" { attrs = append(attrs, ateattr.SnapshotKindKey.String(snapshotKind)) diff --git a/cmd/ateapi/internal/controlapi/metrics_test.go b/cmd/ateapi/internal/controlapi/metrics_test.go index eb8bfe7654..a2f271b574 100644 --- a/cmd/ateapi/internal/controlapi/metrics_test.go +++ b/cmd/ateapi/internal/controlapi/metrics_test.go @@ -242,14 +242,15 @@ func TestLifecycleOpDurationShape(t *testing.T) { actor := &ateapipb.Actor{ ActorTemplate: &ateapipb.ObjectRef{Atespace: "ate-agents", Name: "support-agent"}, Status: &ateapipb.ActorStatus{ - WorkerAssignment: &ateapipb.WorkerAssignment{WorkerNamespace: "ate-workers", WorkerPool: "pool-a"}, + WorkerAssignment: &ateapipb.WorkerAssignment{ + WorkerNamespace: "ate-workers", + WorkerPool: "pool-a", + SandboxRuntime: &ateapipb.SandboxRuntime{SandboxClass: "gvisor"}, + }, }, } - template := &ateapipb.ActorTemplate{ - SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor"}, - } inst.recordLifecycleOp(context.Background(), ateattr.OperationResume, time.Now(), nil, - lifecycleOpAttrs(actor, template, ateattr.SnapshotKindLatest, ateattr.SnapshotFidelityVolumes)...) + lifecycleOpAttrs(actor, ateattr.SnapshotKindLatest, ateattr.SnapshotFidelityVolumes)...) dp := singleHistogramDP(t, reader, lifecycleOpDurationMetric) assertAttrKeys(t, dp, @@ -286,9 +287,9 @@ func TestLifecycleOpDurationShape(t *testing.T) { // pins the pool pair: a failure before the assign step emits neither key. func TestLifecycleOpAttrsOmitsUnknownScope(t *testing.T) { actor := &ateapipb.Actor{ActorTemplate: &ateapipb.ObjectRef{Atespace: "ate-agents", Name: "support-agent"}} - for _, kv := range lifecycleOpAttrs(actor, nil, "", "") { + for _, kv := range lifecycleOpAttrs(actor, "", "") { switch kv.Key { - case ateattr.SnapshotFidelityKey, ateattr.SnapshotKindKey, ateattr.WorkerPoolNamespaceKey, ateattr.WorkerPoolNameKey: + case ateattr.SnapshotFidelityKey, ateattr.SnapshotKindKey, ateattr.WorkerPoolNamespaceKey, ateattr.WorkerPoolNameKey, ateattr.SandboxClassKey: t.Errorf("attribute %s must be omitted while unknown, got %q", kv.Key, kv.Value.AsString()) } } @@ -299,7 +300,7 @@ func TestLifecycleOpAttrsOmitsUnknownScope(t *testing.T) { // holds no actor, so the template keys are omitted, as on a failed delete, // rather than recorded as the empty string. func TestLifecycleOpAttrsOmitsTemplateBeforeLoad(t *testing.T) { - if got := lifecycleOpAttrs(nil, nil, "", ""); len(got) != 0 { + if got := lifecycleOpAttrs(nil, "", ""); len(got) != 0 { t.Errorf("lifecycleOpAttrs(nil actor) = %v, want no attributes", got) } } diff --git a/cmd/ateapi/internal/controlapi/mint_after_release_test.go b/cmd/ateapi/internal/controlapi/mint_after_release_test.go index 369f0bb616..6ca42106cd 100644 --- a/cmd/ateapi/internal/controlapi/mint_after_release_test.go +++ b/cmd/ateapi/internal/controlapi/mint_after_release_test.go @@ -70,7 +70,7 @@ func TestMintWorkerActorCertificateAfterWorkerRelease(t *testing.T) { WorkerPod: w.pod, WorkerPodUid: w.podUID, NodeName: node, - SandboxClass: wf.WorkerClass, + SandboxClass: "gvisor", Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}}, }); err != nil { t.Fatalf("CreateWorker(%s): %v", w.name, err) diff --git a/cmd/ateapi/internal/controlapi/sandbox_assets.go b/cmd/ateapi/internal/controlapi/sandbox_assets.go index 47593c8462..ea8fdb83c6 100644 --- a/cmd/ateapi/internal/controlapi/sandbox_assets.go +++ b/cmd/ateapi/internal/controlapi/sandbox_assets.go @@ -25,45 +25,57 @@ import ( k8serrors "k8s.io/apimachinery/pkg/api/errors" ) -// resolveTemplateSandboxConfig resolves the SandboxConfig the ActorTemplate -// names via sandbox_config.config_name and checks that its class matches the -// template's sandbox_class. -func resolveTemplateSandboxConfig( +// resolvePoolSandboxConfig resolves the SandboxConfig the controller confirmed +// for the WorkerPool of the actor's assigned worker. +func resolvePoolSandboxConfig( + workerPoolLister listersv1alpha1.WorkerPoolLister, sandboxConfigLister listersv1alpha1.SandboxConfigLister, - templateSandbox *ateapipb.SandboxConfig, + assignment *ateapipb.WorkerAssignment, ) (*atev1alpha1.SandboxConfig, error) { - name := templateSandbox.GetConfigName() - sc, err := sandboxConfigLister.Get(name) + namespace, poolName := assignment.GetWorkerNamespace(), assignment.GetWorkerPool() + if namespace == "" || poolName == "" { + return nil, fmt.Errorf("worker assignment names no worker pool") + } + pool, err := workerPoolLister.WorkerPools(namespace).Get(poolName) if k8serrors.IsNotFound(err) { - return nil, apierror.FailedPrecondition("SandboxConfig %q not found", name) + return nil, apierror.FailedPrecondition("WorkerPool %s/%s not found", namespace, poolName) } if err != nil { - return nil, fmt.Errorf("while getting SandboxConfig %q: %w", name, err) + return nil, fmt.Errorf("while getting WorkerPool %s/%s: %w", namespace, poolName, err) } - if class := templateSandbox.GetSandboxClass(); string(sc.Spec.SandboxClass) != class { + configName := pool.ResolvedSandboxConfig() + if configName == "" { return nil, apierror.FailedPrecondition( - "SandboxConfig %q has class %q but sandbox_config.sandbox_class is %q", name, sc.Spec.SandboxClass, class) + "WorkerPool %s/%s has not resolved a SandboxConfig for sandbox class %q", namespace, poolName, pool.Spec.DefaultSandboxClass()) + } + sc, err := sandboxConfigLister.Get(configName) + if k8serrors.IsNotFound(err) { + return nil, apierror.FailedPrecondition("SandboxConfig %q of WorkerPool %s/%s not found", configName, namespace, poolName) + } + if err != nil { + return nil, fmt.Errorf("while getting SandboxConfig %q: %w", configName, err) } return sc, nil } // resolveSandboxAssets determines the sandbox binaries and pause image an actor -// should boot with and projects them onto the ateletpb.SandboxAssets atelet -// fetches: the default version of the SandboxConfig the ActorTemplate names via -// sandbox_config.config_name (required; enforced by CreateActorTemplate), -// checked against the template's sandbox_class. +// boots with on its assigned worker and projects them onto the +// ateletpb.SandboxAssets atelet fetches: the default version of the +// SandboxConfig the worker's pool resolved for its sandbox class, which is the +// runtime the worker reports as its default. +// +// TODO: Remove with sandbox_assets. atelet already resolves the pool's +// SandboxConfig and its versions for every worker on the node (its sandbox +// runtimes syncer), and Run and Restore carry sandbox_runtime; once atelet +// resolves the version from that field instead of taking the assets from the +// request, this lookup, the WorkerPool and SandboxConfig listers on +// ActorWorkflow, and ateapi's sandboxconfigs RBAC go. func resolveSandboxAssets( + workerPoolLister listersv1alpha1.WorkerPoolLister, sandboxConfigLister listersv1alpha1.SandboxConfigLister, - templateSandbox *ateapipb.SandboxConfig, + assignment *ateapipb.WorkerAssignment, ) (*ateletpb.SandboxAssets, error) { - if templateSandbox.GetSandboxClass() == "" { - return nil, fmt.Errorf("ActorTemplate names no sandbox_config.sandbox_class") - } - if templateSandbox.GetConfigName() == "" { - return nil, fmt.Errorf("ActorTemplate names no sandbox_config.config_name") - } - - sc, err := resolveTemplateSandboxConfig(sandboxConfigLister, templateSandbox) + sc, err := resolvePoolSandboxConfig(workerPoolLister, sandboxConfigLister, assignment) if err != nil { return nil, err } diff --git a/cmd/ateapi/internal/controlapi/sandbox_assets_test.go b/cmd/ateapi/internal/controlapi/sandbox_assets_test.go index 4bc227eb55..ae915b2eef 100644 --- a/cmd/ateapi/internal/controlapi/sandbox_assets_test.go +++ b/cmd/ateapi/internal/controlapi/sandbox_assets_test.go @@ -39,15 +39,45 @@ func sandboxConfigListerFor(t *testing.T, configs []*atev1alpha1.SandboxConfig) return listersv1alpha1.NewSandboxConfigLister(configIdx) } +// workerPoolListerFor builds a lister over the given WorkerPools, using the +// same key function the informers use. +func workerPoolListerFor(t *testing.T, pools []*atev1alpha1.WorkerPool) listersv1alpha1.WorkerPoolLister { + t.Helper() + poolIdx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) + for _, p := range pools { + if err := poolIdx.Add(p); err != nil { + t.Fatalf("adding WorkerPool: %v", err) + } + } + return listersv1alpha1.NewWorkerPoolLister(poolIdx) +} + +// resolvedWorkerPool is a gVisor WorkerPool whose controller-confirmed +// SandboxConfig is configName; "" is a pool that has not resolved one yet. +func resolvedWorkerPool(namespace, name, configName string) *atev1alpha1.WorkerPool { + pool := &atev1alpha1.WorkerPool{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}, + Spec: atev1alpha1.WorkerPoolSpec{ + SandboxClasses: []atev1alpha1.WorkerPoolSandboxClass{{Name: atev1alpha1.SandboxClassGvisor}}, + }, + } + status := atev1alpha1.WorkerPoolSandboxClassStatus{Name: atev1alpha1.SandboxClassGvisor} + if configName != "" { + status.ConfigRef = &atev1alpha1.SandboxConfigReference{Name: configName} + } + pool.Status.SandboxClasses = []atev1alpha1.WorkerPoolSandboxClassStatus{status} + return pool +} + func testAssets() map[string]map[string]atev1alpha1.AssetFile { return map[string]map[string]atev1alpha1.AssetFile{ "amd64": {"gvisor": {URL: "gs://bucket/gvisor.tar.bz2", SHA256: "abc"}}, } } -// TestResolveSandboxAssets pins the template-side resolution: the config the -// template names is resolved (with its class checked) to its default version, -// a missing default version, an empty name or an unrecognized class is an +// TestResolveSandboxAssets pins the pool-side resolution: the SandboxConfig +// the assigned worker's pool resolved is taken at its default version, a +// missing default version, an unresolved pool, a missing pool or config is an // error, and the pause image travels with the sandbox binaries. func TestResolveSandboxAssets(t *testing.T) { const namedPause = "gcr.io/gke-release/pause@sha256:named" @@ -88,68 +118,57 @@ func TestResolveSandboxAssets(t *testing.T) { }}, }, } + pools := []*atev1alpha1.WorkerPool{ + resolvedWorkerPool("worker-ns", "custom", "gvisor-custom"), + resolvedWorkerPool("worker-ns", "no-default", "gvisor-no-default"), + resolvedWorkerPool("worker-ns", "disabled-default", "gvisor-disabled-default"), + resolvedWorkerPool("worker-ns", "missing-config", "does-not-exist"), + resolvedWorkerPool("worker-ns", "unresolved", ""), + } + assignment := func(pool string) *ateapipb.WorkerAssignment { + return &ateapipb.WorkerAssignment{WorkerNamespace: "worker-ns", WorkerPool: pool} + } tests := []struct { name string - sandbox *ateapipb.SandboxConfig + assignment *ateapipb.WorkerAssignment wantPauseImage string wantErr string }{{ - name: "named config", - sandbox: &ateapipb.SandboxConfig{ - SandboxClass: "gvisor", - ConfigName: "gvisor-custom", - }, + name: "pool with a resolved config", + assignment: assignment("custom"), wantPauseImage: namedPause, }, { - name: "named config without its default version", - sandbox: &ateapipb.SandboxConfig{ - SandboxClass: "gvisor", - ConfigName: "gvisor-no-default", - }, - wantErr: `SandboxConfig "gvisor-no-default" has no enabled version ""`, + name: "config without its default version", + assignment: assignment("no-default"), + wantErr: `SandboxConfig "gvisor-no-default" has no enabled version ""`, }, { - name: "named config whose default version is disabled", - sandbox: &ateapipb.SandboxConfig{ - SandboxClass: "gvisor", - ConfigName: "gvisor-disabled-default", - }, - wantErr: `SandboxConfig "gvisor-disabled-default" has no enabled version "v1"`, + name: "config whose default version is disabled", + assignment: assignment("disabled-default"), + wantErr: `SandboxConfig "gvisor-disabled-default" has no enabled version "v1"`, }, { - name: "named config class mismatch", - sandbox: &ateapipb.SandboxConfig{ - SandboxClass: "microvm", - ConfigName: "gvisor-custom", - }, - wantErr: `has class "gvisor"`, - }, { - name: "missing named config", - sandbox: &ateapipb.SandboxConfig{ - SandboxClass: "gvisor", - ConfigName: "does-not-exist", - }, - wantErr: `SandboxConfig "does-not-exist" not found`, + name: "pool whose config is missing", + assignment: assignment("missing-config"), + wantErr: `SandboxConfig "does-not-exist" of WorkerPool worker-ns/missing-config not found`, }, { - name: "unrecognized sandbox class", - sandbox: &ateapipb.SandboxConfig{ - SandboxClass: "kata", - ConfigName: "gvisor-custom", - }, - wantErr: `has class "gvisor" but sandbox_config.sandbox_class is "kata"`, + name: "pool that has not resolved a config", + assignment: assignment("unresolved"), + wantErr: `WorkerPool worker-ns/unresolved has not resolved a SandboxConfig`, }, { - name: "empty sandbox class", - sandbox: &ateapipb.SandboxConfig{ConfigName: "gvisor-custom"}, - wantErr: "names no sandbox_config.sandbox_class", + name: "missing pool", + assignment: assignment("does-not-exist"), + wantErr: `WorkerPool worker-ns/does-not-exist not found`, }, { - name: "empty config name", - sandbox: &ateapipb.SandboxConfig{SandboxClass: "gvisor"}, - wantErr: "names no sandbox_config.config_name", + name: "assignment without a pool", + assignment: &ateapipb.WorkerAssignment{}, + wantErr: "names no worker pool", }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + poolLister := workerPoolListerFor(t, pools) configLister := sandboxConfigListerFor(t, []*atev1alpha1.SandboxConfig{namedConfig, noDefaultConfig, disabledDefaultConfig}) - got, err := resolveSandboxAssets(configLister, tt.sandbox) + got, err := resolveSandboxAssets(poolLister, configLister, tt.assignment) if tt.wantErr != "" { if err == nil || !strings.Contains(err.Error(), tt.wantErr) { t.Fatalf("resolveSandboxAssets() error = %v, want it to contain %q", err, tt.wantErr) diff --git a/cmd/ateapi/internal/controlapi/service.go b/cmd/ateapi/internal/controlapi/service.go index c109ddf5ed..713dc3a6ec 100644 --- a/cmd/ateapi/internal/controlapi/service.go +++ b/cmd/ateapi/internal/controlapi/service.go @@ -43,7 +43,6 @@ type RPCService struct { persistence serviceStore workerCache *workercache.Cache dialer *AteletDialer - sandboxConfigLister listersv1alpha1.SandboxConfigLister csiDriverConfigLister listersv1alpha1.CSIDriverConfigLister actorWorkflow *ActorWorkflow workerWorkflow *WorkerWorkflow @@ -78,6 +77,7 @@ type VolumePluginRegistry interface { func NewRPCService( persistence store.Interface, workerCache *workercache.Cache, + workerPoolLister listersv1alpha1.WorkerPoolLister, sandboxConfigLister listersv1alpha1.SandboxConfigLister, csiDriverConfigLister listersv1alpha1.CSIDriverConfigLister, storageClassLister storagev1listers.StorageClassLister, @@ -96,7 +96,6 @@ func NewRPCService( impl: impl, persistence: persistence, workerCache: workerCache, - sandboxConfigLister: sandboxConfigLister, csiDriverConfigLister: csiDriverConfigLister, dialer: dialer, instruments: instruments, @@ -106,7 +105,7 @@ func NewRPCService( actorIDJWTPool: actorIDJWTPool, actorIDCAPool: actorIDCAPool, } - s.actorWorkflow = NewActorWorkflow(impl, workerCache, dialer, sandboxConfigLister, storageClassLister, instruments, egressGatewayAddress, s, actorWorkflowDeadline, objectStorePlugin) + s.actorWorkflow = NewActorWorkflow(impl, workerCache, dialer, workerPoolLister, sandboxConfigLister, storageClassLister, instruments, egressGatewayAddress, s, actorWorkflowDeadline, objectStorePlugin) s.workerWorkflow = NewWorkerWorkflow(impl) return s } diff --git a/cmd/ateapi/internal/controlapi/snapshot_helpers.go b/cmd/ateapi/internal/controlapi/snapshot_helpers.go index d952125079..f8ee69e529 100644 --- a/cmd/ateapi/internal/controlapi/snapshot_helpers.go +++ b/cmd/ateapi/internal/controlapi/snapshot_helpers.go @@ -67,26 +67,23 @@ func findLatestLocalSnapshot(status *ateapipb.ActorStatus) *ateapipb.Snapshot { }) } -// latestSnapshotIsLocalOnly reports whether the snapshot at the actor's latest -// generation is node-local with no durable copy: the shape a pause leaves -// behind. A paused actor is SUSPENDED with this shape, and resumes only on the -// node named by Locality; a suspend of such an actor uploads that snapshot -// instead of checkpointing a workload. A RUNNING actor resumed from pause -// keeps the local snapshot too, so callers that need "at rest" check the state -// as well. -func latestSnapshotIsLocalOnly(status *ateapipb.ActorStatus) bool { - snap := snapshotAtLatestGeneration(status) - return snap.GetLocality() != "" && snap.GetDurableSnapshot() == nil +// latestSnapshotIsLocal reports whether the snapshot at the actor's latest +// generation is node-local: the shape a pause leaves behind, which atelet's +// upload of the snapshot doesn't change. A paused actor is SUSPENDED with this +// shape, and resumes only on the node named by Locality. A RUNNING actor +// resumed from pause keeps the local snapshot too, so callers that need "at +// rest" check the state as well. +func latestSnapshotIsLocal(status *ateapipb.ActorStatus) bool { + return snapshotAtLatestGeneration(status).GetLocality() != "" } -// latestSnapshotIsPendingLocal reports whether the snapshot at the actor's -// latest generation is node-local but not yet captured: it has no locality and -// no durable copy. This is the shape a pause in progress has, and what tells a -// SUSPENDING pause from a SUSPENDING suspend, whose latest snapshot has a -// pending durable copy. -func latestSnapshotIsPendingLocal(status *ateapipb.ActorStatus) bool { +// latestSnapshotIsLocalOnly reports whether the snapshot at the actor's latest +// generation is node-local with no COMPLETED durable copy: a pause's snapshot +// whose upload is not recorded yet. +func latestSnapshotIsLocalOnly(status *ateapipb.ActorStatus) bool { snap := snapshotAtLatestGeneration(status) - return snap != nil && snap.GetLocality() == "" && snap.GetDurableSnapshot() == nil + return snap.GetLocality() != "" && + snap.GetDurableSnapshot().GetStatus() != ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED } // clearLocalSnapshots clears Locality across all Snapshots in status.Snapshots @@ -108,11 +105,11 @@ func clearLocalSnapshots(status *ateapipb.ActorStatus) { // beforehand. // // Dropping PENDING durable snapshots is safe because: -// - On pause finalization, pause is only reachable from RUNNING (a failed -// suspend cannot resume without reverting first, which discards any -// in-flight upload), so no uncleaned PENDING durable snapshot exists. +// - On pause finalization, the new snapshot is kept with its PENDING copy. +// An earlier pause's PENDING copy is dropped: whatever atelet uploaded of +// it stays under the actor's prefix until DeleteActor collects it. // - On revert finalization, ensureInProgressSnapshotDiscarded has already -// deleted any in-flight durable snapshot objects. +// deleted the objects of every PENDING durable snapshot. // - On suspend finalization, the latest snapshot has already been marked // COMPLETED. // @@ -151,19 +148,6 @@ func newDurableSnapshot(gen int32, owner ateapipb.SnapshotOwner, fidelity ateapi } } -// newLocalSnapshot constructs an actor Snapshot for a local pause checkpoint -// with Uuid, Fidelity, and Locality. -func newLocalSnapshot(gen int32, fidelity ateapipb.SnapshotFidelity, templateUID, uuid, locality string) *ateapipb.Snapshot { - return &ateapipb.Snapshot{ - Uuid: uuid, - Generation: gen, - Owner: ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, - Fidelity: fidelity, - ActorTemplateUid: templateUID, - Locality: locality, - } -} - // tagDurableSnapshotURI returns the durable snapshot URI on tag matching // storageStatus (or any status if storageStatus is UNSPECIFIED), or "" if none // exists. diff --git a/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go b/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go index 647d44b5fc..b535d70072 100644 --- a/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go +++ b/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go @@ -30,7 +30,7 @@ func TestFindSnapshotByGenerationAndSnapshotAtLatestGeneration(t *testing.T) { t.Errorf("findSnapshotByGeneration(nil, 1) = %v, want nil", got) } - s1 := newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-1", "node-1") + s1 := newPausedSnapshot(t, 1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-1", "node-1") s2 := newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-2", "gs://b/snap-2", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) status := &ateapipb.ActorStatus{ LastAssignedGeneration: 2, @@ -59,7 +59,7 @@ func TestFindLatestDurableAndLocalSnapshots(t *testing.T) { s1 := newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) s2 := newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-2", "gs://b/snap-2", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) s3 := newDurableSnapshot(3, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-3", "gs://b/snap-3", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) - s4 := newLocalSnapshot(4, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-4", "node-1") + s4 := newPausedSnapshot(t, 4, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-4", "node-1") // Place s2 before s1 to verify generation comparison rather than slice order. status := &ateapipb.ActorStatus{ @@ -71,9 +71,14 @@ func TestFindLatestDurableAndLocalSnapshots(t *testing.T) { t.Errorf("findLatestDurableSnapshot(COMPLETED) = %v, want gen 2 (gs://b/snap-2)", gotSnap) } + // A paused snapshot's durable copy is PENDING until atelet uploads it. gotSnap = findLatestDurableSnapshot(status, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) + if gotSnap != s4 { + t.Errorf("findLatestDurableSnapshot(PENDING) = %v, want gen 4", gotSnap) + } + gotSnap = findLatestDurableSnapshot(&ateapipb.ActorStatus{Snapshots: []*ateapipb.Snapshot{s2, s1, s3}}, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) if gotSnap != s3 || gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri() != "gs://b/snap-3" { - t.Errorf("findLatestDurableSnapshot(PENDING) = %v, want gen 3 (gs://b/snap-3)", gotSnap) + t.Errorf("findLatestDurableSnapshot(PENDING) without the paused snapshot = %v, want gen 3 (gs://b/snap-3)", gotSnap) } gotLocal := findLatestLocalSnapshot(status) @@ -88,8 +93,8 @@ func TestSnapshotPruningHelpers(t *testing.T) { pruneSnapshots(nil, nil) t.Run("clearLocalSnapshots clears locality without dropping snapshots", func(t *testing.T) { - localGen1 := newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-1", "node-1") - localGen2 := newLocalSnapshot(2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-2", "node-2") + localGen1 := newPausedSnapshot(t, 1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-1", "node-1") + localGen2 := newPausedSnapshot(t, 2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-2", "node-2") status := &ateapipb.ActorStatus{ Snapshots: []*ateapipb.Snapshot{localGen1, localGen2}, } @@ -97,8 +102,8 @@ func TestSnapshotPruningHelpers(t *testing.T) { clearLocalSnapshots(status) want := []*ateapipb.Snapshot{ - newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-1", ""), - newLocalSnapshot(2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-2", ""), + newPausedSnapshot(t, 1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-1", ""), + newPausedSnapshot(t, 2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-2", ""), } if diff := cmp.Diff(want, status.GetSnapshots(), protocmp.Transform()); diff != "" { t.Errorf("status.Snapshots mismatch (-want +got):\n%s", diff) @@ -108,8 +113,8 @@ func TestSnapshotPruningHelpers(t *testing.T) { t.Run("pause flow: pruneSnapshots keeps the latest COMPLETED durable and the new local snapshot", func(t *testing.T) { completedGen1 := newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) completedGen1.Locality = "node-1" - localGen2 := newLocalSnapshot(2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-2", "node-2") - localGen3 := newLocalSnapshot(3, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-3", "") + localGen2 := newPausedSnapshot(t, 2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-2", "node-2") + localGen3 := newPausedSnapshot(t, 3, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-3", "") status := &ateapipb.ActorStatus{ LastAssignedGeneration: 3, @@ -121,7 +126,7 @@ func TestSnapshotPruningHelpers(t *testing.T) { want := []*ateapipb.Snapshot{ newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), - newLocalSnapshot(3, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-3", "node-3"), + newPausedSnapshot(t, 3, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-3", "node-3"), } if diff := cmp.Diff(want, status.GetSnapshots(), protocmp.Transform()); diff != "" { t.Errorf("status.Snapshots mismatch (-want +got):\n%s", diff) @@ -134,7 +139,7 @@ func TestSnapshotPruningHelpers(t *testing.T) { t.Run("pruneSnapshots drops a PENDING snapshot that still names a node", func(t *testing.T) { pendingWithLocality := newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-2", "gs://b/snap-2", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) pendingWithLocality.Locality = "node-1" - local := newLocalSnapshot(3, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-3", "node-1") + local := newPausedSnapshot(t, 3, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-3", "node-1") status := &ateapipb.ActorStatus{ Snapshots: []*ateapipb.Snapshot{ newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), @@ -147,7 +152,7 @@ func TestSnapshotPruningHelpers(t *testing.T) { want := []*ateapipb.Snapshot{ newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), - newLocalSnapshot(3, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-3", "node-1"), + newPausedSnapshot(t, 3, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-3", "node-1"), } if diff := cmp.Diff(want, status.GetSnapshots(), protocmp.Transform()); diff != "" { t.Errorf("status.Snapshots mismatch (-want +got):\n%s", diff) @@ -163,7 +168,7 @@ func TestSnapshotPruningHelpers(t *testing.T) { status := &ateapipb.ActorStatus{ Snapshots: []*ateapipb.Snapshot{ completedWithLocality, - newLocalSnapshot(2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-2", "node-2"), + newPausedSnapshot(t, 2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-2", "node-2"), newDurableSnapshot(3, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-3", "gs://b/snap-3", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), }, } @@ -182,7 +187,7 @@ func TestSnapshotPruningHelpers(t *testing.T) { status := &ateapipb.ActorStatus{ Snapshots: []*ateapipb.Snapshot{ newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), - newLocalSnapshot(2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-2", "node-1"), + newPausedSnapshot(t, 2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-2", "node-1"), newDurableSnapshot(3, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-3", "gs://b/snap-3", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), }, } @@ -199,14 +204,14 @@ func TestSnapshotPruningHelpers(t *testing.T) { }) } -// TestLatestSnapshotIsLocalOnly pins the shape a pause leaves behind: the -// latest snapshot is node-local with no durable copy. A durable latest -// snapshot, a pending upload of the local one, or a local snapshot that is -// not captured yet do not count; a RUNNING actor that kept its local snapshot -// across a resume does. +// TestLatestSnapshotIsLocalOnly pins a pause's snapshot whose upload is not +// recorded yet: the latest snapshot is node-local with no COMPLETED durable +// copy, whether it has no copy or a PENDING one. A durable latest snapshot, or +// a local snapshot that is not captured yet, do not count; a RUNNING actor +// that kept its local snapshot across a resume does. func TestLatestSnapshotIsLocalOnly(t *testing.T) { local := func(gen int32) *ateapipb.Snapshot { - return newLocalSnapshot(gen, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local", "node-1") + return newPausedSnapshot(t, gen, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local", "node-1") } durable := func(gen int32, st ateapipb.SnapshotStorageStatus) *ateapipb.Snapshot { return newDurableSnapshot(gen, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "durable", "gs://b/durable", st) @@ -224,9 +229,9 @@ func TestLatestSnapshotIsLocalOnly(t *testing.T) { {"local only above a durable one", &ateapipb.ActorStatus{LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{durable(1, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), local(2)}}, true}, {"local only alone", &ateapipb.ActorStatus{LastAssignedGeneration: 1, Snapshots: []*ateapipb.Snapshot{local(1)}}, true}, {"durable only", &ateapipb.ActorStatus{LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{durable(2, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED)}}, false}, - {"local with an upload pending", &ateapipb.ActorStatus{LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{uploading}}, false}, - {"local not yet captured", &ateapipb.ActorStatus{LastAssignedGeneration: 1, Snapshots: []*ateapipb.Snapshot{newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local", "")}}, false}, - {"older local only under a pause in progress", &ateapipb.ActorStatus{LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{local(1), newLocalSnapshot(2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local", "")}}, false}, + {"local with an upload pending", &ateapipb.ActorStatus{LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{uploading}}, true}, + {"local not yet captured", &ateapipb.ActorStatus{LastAssignedGeneration: 1, Snapshots: []*ateapipb.Snapshot{newPausedSnapshot(t, 1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local", "")}}, false}, + {"older local only under a pause in progress", &ateapipb.ActorStatus{LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{local(1), newPausedSnapshot(t, 2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local", "")}}, false}, {"latest generation dropped by a revert", &ateapipb.ActorStatus{LastAssignedGeneration: 3, Snapshots: []*ateapipb.Snapshot{local(2)}}, false}, } for _, tc := range tests { @@ -238,17 +243,21 @@ func TestLatestSnapshotIsLocalOnly(t *testing.T) { } } -// TestLatestSnapshotIsPendingLocal pins what tells a SUSPENDING pause from a -// SUSPENDING suspend: the latest snapshot is node-local with no locality yet -// and no durable copy. A running-origin suspend has a pending durable copy, a -// paused-origin suspend has both locality and a pending durable copy, and an -// actor with no snapshot at all has nothing pending. -func TestLatestSnapshotIsPendingLocal(t *testing.T) { - pendingLocal := newLocalSnapshot(2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local", "") - capturedLocal := newLocalSnapshot(2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local", "node-1") - pendingDurable := newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "durable", "gs://b/d", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) - uploadingLocal := newLocalSnapshot(2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local", "node-1") - uploadingLocal.DurableSnapshot = &ateapipb.SnapshotStorage{Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING, Object: &ateapipb.ObjectSnapshot{SnapshotUri: "gs://b/up"}} +// TestLatestSnapshotIsLocal pins the shape a paused actor has: the latest +// snapshot is node-local, whether or not atelet has uploaded it. A latest +// snapshot with only a durable copy, or a local one not captured yet, does not +// count. +func TestLatestSnapshotIsLocal(t *testing.T) { + local := func(gen int32) *ateapipb.Snapshot { + return newPausedSnapshot(t, gen, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local", "node-1") + } + withCopy := func(st ateapipb.SnapshotStorageStatus) *ateapipb.Snapshot { + snap := local(1) + snap.DurableSnapshot = &ateapipb.SnapshotStorage{Status: st, Object: &ateapipb.ObjectSnapshot{SnapshotUri: "gs://b/up"}} + return snap + } + durable := newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "durable", "gs://b/durable", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + notCaptured := newPausedSnapshot(t, 2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local", "") tests := []struct { name string @@ -257,16 +266,16 @@ func TestLatestSnapshotIsPendingLocal(t *testing.T) { }{ {"nil status", nil, false}, {"no snapshots", &ateapipb.ActorStatus{}, false}, - {"local not yet captured", &ateapipb.ActorStatus{LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{pendingLocal}}, true}, - {"durable upload pending", &ateapipb.ActorStatus{LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{pendingDurable}}, false}, - {"local with an upload pending", &ateapipb.ActorStatus{LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{uploadingLocal}}, false}, - {"local only", &ateapipb.ActorStatus{LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{capturedLocal}}, false}, - {"latest generation dropped by a revert", &ateapipb.ActorStatus{LastAssignedGeneration: 3, Snapshots: []*ateapipb.Snapshot{pendingLocal}}, false}, + {"local only", &ateapipb.ActorStatus{LastAssignedGeneration: 1, Snapshots: []*ateapipb.Snapshot{local(1)}}, true}, + {"local and uploaded", &ateapipb.ActorStatus{LastAssignedGeneration: 1, Snapshots: []*ateapipb.Snapshot{withCopy(ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED)}}, true}, + {"local with an upload pending", &ateapipb.ActorStatus{LastAssignedGeneration: 1, Snapshots: []*ateapipb.Snapshot{withCopy(ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING)}}, true}, + {"durable only", &ateapipb.ActorStatus{LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{durable}}, false}, + {"older local under a pause in progress", &ateapipb.ActorStatus{LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{local(1), notCaptured}}, false}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - if got := latestSnapshotIsPendingLocal(tc.status); got != tc.want { - t.Errorf("latestSnapshotIsPendingLocal = %t, want %t", got, tc.want) + if got := latestSnapshotIsLocal(tc.status); got != tc.want { + t.Errorf("latestSnapshotIsLocal = %t, want %t", got, tc.want) } }) } diff --git a/cmd/ateapi/internal/controlapi/snapshot_report.go b/cmd/ateapi/internal/controlapi/snapshot_report.go index 01a2b5fa30..ca7355d7b5 100644 --- a/cmd/ateapi/internal/controlapi/snapshot_report.go +++ b/cmd/ateapi/internal/controlapi/snapshot_report.go @@ -119,8 +119,9 @@ func (w *ActorWorkflow) loadReportedActor(ctx context.Context, actorRef resource } // actorOnNode reports whether the Actor is placed on node or has a local -// snapshot there. Unlike HostingNode, it includes a node that only holds the -// Actor's snapshots, since that node's atelet reports them. +// snapshot there. Unlike HostingNode, it includes a node that holds only +// older snapshots of an Actor placed elsewhere, since that node's atelet +// reports them. func actorOnNode(actor *ateapipb.Actor, node string) bool { if actor.GetStatus().GetAssignedNode() == node { return true @@ -153,21 +154,23 @@ func applyActorSnapshotState(ctx context.Context, status *ateapipb.ActorStatus, } // applyCheckpointState records an upload that atelet's state, as returned by -// Checkpoint or PromoteSnapshot, says finished, with the same rules as a -// report. atelet echoes the Snapshot it was asked for, so the fidelity recorded -// is the one requested, not read from the manifest. Unlike a report, a state -// with no local snapshot is not a detach: the workflow that made the call -// records where its snapshot lives. +// Checkpoint, says finished, with the same rules as a report. It records the +// fidelity atelet reports, which for Checkpoint is the one requested. Unlike a +// report, a state with no local snapshot is not a detach. The workflow that +// made the call records where its snapshot lives. func applyCheckpointState(ctx context.Context, status *ateapipb.ActorStatus, state *ateletpb.ActorSnapshotState) error { return applyLatestDurable(ctx, status, snapshotReportFromAtelet(state.GetLatestDurable())) } -// applyLatestDurable marks the reported durable snapshot COMPLETED and records -// the fidelity from its manifest. Its uuid and upload URI must match the -// recorded ones. A generation at or below the latest COMPLETED one changes -// nothing, and so does one status no longer has, such as one a revert -// discarded. A late report is expected, so an older generation and a -// discarded one are logged at Info. A repeat of the latest isn't logged. +// applyLatestDurable marks the reported durable snapshot COMPLETED, and records +// its fidelity. Its uuid and upload URI must match the recorded ones, and its +// fidelity must be set. The snapshot keeps its locality, so a paused Actor +// stays on its node. +// +// A late report changes nothing and is logged at Info. A report is late when +// its generation is older than the latest COMPLETED one, or is no longer in +// status.snapshots, for example because a revert removed it. A repeat of the +// latest COMPLETED generation also changes nothing, and isn't logged. func applyLatestDurable(ctx context.Context, status *ateapipb.ActorStatus, reported *ateapipb.SnapshotReport) error { if reported == nil { return nil @@ -194,13 +197,16 @@ func applyLatestDurable(ctx context.Context, status *ateapipb.ActorStatus, repor if snap.GetUuid() != reported.GetUuid() { return apierror.FailedPrecondition("snapshot generation %d is recorded as %s, not %s", reported.GetGeneration(), snap.GetUuid(), reported.GetUuid()) } - want := snap.GetDurableSnapshot().GetObject().GetSnapshotUri() - if want == "" { - return apierror.FailedPrecondition("snapshot generation %d has no upload location recorded", reported.GetGeneration()) + if snap.GetDurableSnapshot() == nil { + return apierror.FailedPrecondition("snapshot generation %d records no upload", reported.GetGeneration()) } + want := snap.GetDurableSnapshot().GetObject().GetSnapshotUri() if got := reported.GetObject().GetSnapshotUri(); got != want { return apierror.InvalidArgument("snapshot generation %d was uploaded to %q, want %q", reported.GetGeneration(), got, want) } + if reported.GetFidelity() == ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED { + return apierror.InvalidArgument("snapshot generation %d reports no fidelity", reported.GetGeneration()) + } snap.Fidelity = reported.GetFidelity() snap.DurableSnapshot.Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED return nil diff --git a/cmd/ateapi/internal/controlapi/snapshot_report_test.go b/cmd/ateapi/internal/controlapi/snapshot_report_test.go index 5e0a0f261e..19228f3cf4 100644 --- a/cmd/ateapi/internal/controlapi/snapshot_report_test.go +++ b/cmd/ateapi/internal/controlapi/snapshot_report_test.go @@ -50,16 +50,11 @@ func ownedURI(t *testing.T, actorUID string, gen int32) string { return uri.String() } -// localOnly returns a Snapshot held on locality with no durable copy. -func localOnly(gen int32, locality string) *ateapipb.Snapshot { - return &ateapipb.Snapshot{Uuid: snapName(gen), Generation: gen, Locality: locality, Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY} -} - -// withDurable returns a Snapshot with a durable copy in the given storage -// status. +// withDurable returns a Snapshot held on locality ("" for none) with a durable +// copy in the given storage status. func withDurable(t *testing.T, gen int32, locality string, storage ateapipb.SnapshotStorageStatus) *ateapipb.Snapshot { t.Helper() - snap := localOnly(gen, locality) + snap := &ateapipb.Snapshot{Uuid: snapName(gen), Generation: gen, Locality: locality, Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY} snap.DurableSnapshot = &ateapipb.SnapshotStorage{ Status: storage, Object: &ateapipb.ObjectSnapshot{SnapshotUri: ownedURI(t, reportActorUID, gen)}, @@ -121,6 +116,17 @@ func TestApplyActorSnapshotState(t *testing.T) { return snap }(), }}, + }, { + name: "an unset fidelity is refused", + status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, reportNode, pending), + }}, + state: &ateapipb.ActorSnapshotState{LatestLocal: reported(t, 2), LatestDurable: func() *ateapipb.SnapshotReport { + r := reported(t, 2) + r.Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED + return r + }()}, + wantCode: codes.InvalidArgument, }, { name: "a snapshot already COMPLETED changes nothing", status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ @@ -173,9 +179,20 @@ func TestApplyActorSnapshotState(t *testing.T) { }()}, wantCode: codes.FailedPrecondition, }, { - name: "a snapshot with no recorded upload URI is refused", - status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ - localOnly(2, reportNode), + name: "an older snapshot's upload is recorded and the latest stays pending", + status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 1, reportNode, pending), + withDurable(t, 2, reportNode, pending), + }}, + state: &ateapipb.ActorSnapshotState{LatestLocal: reported(t, 2), LatestDurable: reported(t, 1)}, + want: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 1, reportNode, completed), + withDurable(t, 2, reportNode, pending), + }}, + }, { + name: "a snapshot that records no upload is refused", + status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ + {Uuid: snapName(2), Generation: 2, Locality: reportNode, Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY}, }}, state: &ateapipb.ActorSnapshotState{LatestLocal: reported(t, 2), LatestDurable: reported(t, 2)}, wantCode: codes.FailedPrecondition, @@ -236,20 +253,20 @@ func TestApplyActorSnapshotState(t *testing.T) { withDurable(t, 3, reportNode, completed), }}, }, { - name: "a late detach leaves a newer local-only snapshot on the node", + name: "a late detach leaves a newer snapshot whose upload is pending on the node", status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ withDurable(t, 1, "", completed), - localOnly(2, reportNode), + withDurable(t, 2, reportNode, pending), }}, state: &ateapipb.ActorSnapshotState{LatestDurable: reported(t, 1)}, want: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ withDurable(t, 1, "", completed), - localOnly(2, reportNode), + withDurable(t, 2, reportNode, pending), }}, }, { name: "a detach of a snapshot that isn't durable is refused", status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ - localOnly(1, reportNode), + withDurable(t, 1, reportNode, pending), }}, state: &ateapipb.ActorSnapshotState{LatestDurable: reported(t, 2)}, wantCode: codes.FailedPrecondition, @@ -275,12 +292,12 @@ func TestApplyActorSnapshotState(t *testing.T) { name: "a detach from one node leaves another node alone", status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportOther, Snapshots: []*ateapipb.Snapshot{ withDurable(t, 2, reportNode, completed), - localOnly(3, reportOther), + withDurable(t, 3, reportOther, pending), }}, state: &ateapipb.ActorSnapshotState{LatestDurable: reported(t, 2)}, want: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportOther, Snapshots: []*ateapipb.Snapshot{ withDurable(t, 2, "", completed), - localOnly(3, reportOther), + withDurable(t, 3, reportOther, pending), }}, }} for _, tt := range tests { @@ -388,12 +405,19 @@ func TestApplyReportedSnapshotState(t *testing.T) { ctx := context.Background() persistence := newTestPersistence(t) w := newMarkOnlyWorkflow(persistence) - tmpl := &ateapipb.ActorTemplate{SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: testStorageLocation}} + tmpl, err := persistence.CreateActorTemplate(ctx, &ateapipb.ActorTemplate{ + Metadata: &ateapipb.ResourceMetadata{Atespace: reportAtespace, Name: "tmpl"}, + SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: testStorageLocation}, + }) + if err != nil { + t.Fatalf("CreateActorTemplate() failed: %v", err) + } actorRef := resources.ActorRef{Atespace: reportAtespace, Name: "actor-1"} actor := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, - Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}, + Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: reportAtespace, Name: "tmpl"}, + Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}, }) uid := actor.GetMetadata().GetUid() pending := mustActorSnapshotURI(t, tmpl, actor, "snap-2") @@ -499,7 +523,7 @@ func TestActorOnNode(t *testing.T) { actor := func(assigned string, localities ...string) *ateapipb.Actor { status := &ateapipb.ActorStatus{AssignedNode: assigned} for i, l := range localities { - status.Snapshots = append(status.Snapshots, localOnly(int32(i+1), l)) + status.Snapshots = append(status.Snapshots, withDurable(t, int32(i+1), l, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING)) } return &ateapipb.Actor{Status: status} } @@ -537,7 +561,21 @@ func TestApplyCheckpointState(t *testing.T) { completed = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED suspended = ateapipb.ActorState_ACTOR_STATE_SUSPENDED suspending = ateapipb.ActorState_ACTOR_STATE_SUSPENDING + + otherActorUID = "1a7e4c83-6d20-4f95-b3c8-9e0a2f6d4b17" ) + // previousPause is a pausing actor's status, holding the snapshot of the + // pause before, whose upload is pending. + previousPause := func() *ateapipb.ActorStatus { + return &ateapipb.ActorStatus{State: suspending, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 1, reportNode, pending), + withDurable(t, 2, "", pending), + }} + } + elsewhere, err := resources.NewActorSnapshotURI("gs://elsewhere", reportAtespace, reportActorUID, snapName(1)) + if err != nil { + t.Fatal(err) + } tests := []struct { name string status *ateapipb.ActorStatus @@ -573,23 +611,38 @@ func TestApplyCheckpointState(t *testing.T) { }, { name: "a deferred checkpoint changes nothing", status: &ateapipb.ActorStatus{State: suspending, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ - localOnly(2, ""), + withDurable(t, 2, "", pending), }}, state: &ateletpb.ActorSnapshotState{LatestLocal: ateletReported(t, 2)}, want: &ateapipb.ActorStatus{State: suspending, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ - localOnly(2, ""), + withDurable(t, 2, "", pending), }}, }, { - name: "a deferred checkpoint records an older upload that finished", - status: &ateapipb.ActorStatus{State: suspending, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ - withDurable(t, 1, reportNode, pending), - localOnly(2, ""), - }}, - state: &ateletpb.ActorSnapshotState{LatestLocal: ateletReported(t, 2), LatestDurable: ateletReported(t, 1)}, + name: "a deferred checkpoint records the upload of the previous pause's snapshot", + status: previousPause(), + state: &ateletpb.ActorSnapshotState{LatestLocal: ateletReported(t, 2), LatestDurable: ateletReported(t, 1)}, want: &ateapipb.ActorStatus{State: suspending, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ withDurable(t, 1, reportNode, completed), - localOnly(2, ""), + withDurable(t, 2, "", pending), }}, + }, { + name: "the previous pause's snapshot uploaded under another location is refused", + status: previousPause(), + state: &ateletpb.ActorSnapshotState{LatestLocal: ateletReported(t, 2), LatestDurable: func() *ateletpb.SnapshotReport { + r := ateletReported(t, 1) + r.Snapshot.Object.SnapshotUri = elsewhere.String() + return r + }()}, + wantCode: codes.InvalidArgument, + }, { + name: "the previous pause's snapshot uploaded under another actor's prefix is refused", + status: previousPause(), + state: &ateletpb.ActorSnapshotState{LatestLocal: ateletReported(t, 2), LatestDurable: func() *ateletpb.SnapshotReport { + r := ateletReported(t, 1) + r.Snapshot.Object.SnapshotUri = ownedURI(t, otherActorUID, 1) + return r + }()}, + wantCode: codes.InvalidArgument, }, { name: "no local snapshot is not a detach", status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ diff --git a/cmd/ateapi/internal/controlapi/tag.go b/cmd/ateapi/internal/controlapi/tag.go index cd6187076e..4bff1c88f3 100644 --- a/cmd/ateapi/internal/controlapi/tag.go +++ b/cmd/ateapi/internal/controlapi/tag.go @@ -177,10 +177,8 @@ func (s *ServiceImpl) UpdateTag(ctx context.Context, tagRef resources.TagRef, pr // Validate the merged tag against the one it replaces. This is where // the rules the request could not be checked against land: scope, and // the immutability of metadata and source_actor. - if errs := apivalidation.ValidateTagUpdate(ctx, field.NewPath("tag"), toUpdate, oldVal); len(errs) > 0 { - return resources.ToAPIError(errs) - } - return nil + errs := apivalidation.ValidateTagUpdate(ctx, field.NewPath("tag"), toUpdate, oldVal) + return resources.ToAPIError(errs) }) } diff --git a/cmd/ateapi/internal/controlapi/template_reconciler.go b/cmd/ateapi/internal/controlapi/template_reconciler.go index cbb583de08..350f6954ef 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler.go @@ -72,7 +72,6 @@ type goldenActorControl interface { CreateTag(ctx context.Context, req *ateapipb.CreateTagRequest) (*ateapipb.Tag, error) DeleteTag(ctx context.Context, req *ateapipb.DeleteTagRequest) (*ateapipb.Tag, error) DeleteActor(ctx context.Context, req *ateapipb.DeleteActorRequest) (*ateapipb.Actor, error) - CreateAtespace(ctx context.Context, req *ateapipb.CreateAtespaceRequest) (*ateapipb.Atespace, error) CreateActor(ctx context.Context, req *ateapipb.CreateActorRequest) (*ateapipb.Actor, error) GetActor(ctx context.Context, req *ateapipb.GetActorRequest) (*ateapipb.Actor, error) ResumeActor(ctx context.Context, req *ateapipb.ResumeActorRequest) (*ateapipb.ResumeActorResponse, error) @@ -198,11 +197,9 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource } goldenActorRef := &ateapipb.ObjectRef{ - // Golden actors live in the reserved ate-golden atespace, because - // the suspend workflow relies on the ate-golden system atespace to - // always take a full snapshot of the golden actor. - // https://github.com/agent-substrate/substrate/blob/cb7c8385ef2bb489c3d5f7bfa71820fd33935d91/cmd/ateapi/internal/controlapi/workflow_suspend.go#L170-L173 - Atespace: resources.GoldenActorAtespace, + // Golden actors live in the template's atespace, under the same + // policies and storage grants as ordinary actors. + Atespace: tmpl.GetMetadata().GetAtespace(), // Use the template's UID as golden actor's name to prevent collision // when templates are recreated with the same name. Name: tmpl.GetMetadata().GetUid(), @@ -321,7 +318,10 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource // suspendActor waits for the golden actor to produce an external snapshot. // SuspendActor completes an in-flight suspend and is a no-op if already suspended. func (r *ActorTemplateReconciler) suspendActor(ctx context.Context, goldenRef *ateapipb.ObjectRef) error { - resp, err := r.control.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: goldenRef}) + resp, err := r.control.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: goldenRef, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, + }) if err != nil { // A crash during suspend is observed as CRASHED on the retry. return fmt.Errorf("while suspending golden actor: %w", err) @@ -432,13 +432,6 @@ func (r *ActorTemplateReconciler) ensureActorExists(ctx context.Context, tmpl *a if apierror.Code(err) != codes.NotFound { return nil, fmt.Errorf("while getting golden actor: %w", err) } - // Golden actor has not yet been created. Its reserved atespace is - // system-owned, so ensure it exists rather than assuming bootstrap did. - if _, err := r.control.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{ - Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: goldenActorRef.GetAtespace()}}, - }); err != nil && apierror.Code(err) != codes.AlreadyExists { - return nil, fmt.Errorf("while ensuring atespace %q: %w", goldenActorRef.GetAtespace(), err) - } actor, err = r.control.CreateActor(ctx, &ateapipb.CreateActorRequest{ Actor: &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{ diff --git a/cmd/ateapi/internal/controlapi/template_reconciler_test.go b/cmd/ateapi/internal/controlapi/template_reconciler_test.go index 274aae5a45..8a53f51c89 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler_test.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler_test.go @@ -363,7 +363,7 @@ func withSnapshotDeadline(at time.Time) func(*ateapipb.ActorTemplate) { func withGoldenTag() func(*ateapipb.ActorTemplate) { return func(tmpl *ateapipb.ActorTemplate) { - seededGoldenStatus(tmpl).GoldenTag = &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: testTemplateUID} + seededGoldenStatus(tmpl).GoldenTag = &ateapipb.ObjectRef{Atespace: testTemplateRef.Atespace, Name: testTemplateUID} } } @@ -398,7 +398,7 @@ func TestGoldenSnapshotWarmupFor(t *testing.T) { } func TestReconcileOne(t *testing.T) { - goldenSnapshot := "gs://bucket/root/atespaces/ate-golden/actors/" + someActorUID + "/snapshots/snap-1" + goldenSnapshot := "gs://bucket/root/atespaces/ns/actors/" + someActorUID + "/snapshots/snap-1" tests := []struct { name string template *ateapipb.ActorTemplate // nil: template absent from the store @@ -641,13 +641,12 @@ func TestReconcileOne(t *testing.T) { // TestReconcileOne_GoldenActorRequests pins the shape of the control-plane // requests the happy path issues: the golden actor is named after the // template UID so recreated templates with the same name never collide, and -// lives in the reserved ate-golden atespace so the suspend workflow commits -// it Full regardless of the template's preferredFidelity scope. +// lives in the template's atespace under ordinary tenant policies. func TestReconcileOne_GoldenActorRequests(t *testing.T) { ctx := context.Background() st := newFakeTemplateStore(testTemplate()) - goldenSnapshot := "gs://bucket/root/atespaces/ate-golden/actors/" + someActorUID + "/snapshots/snap-1" + goldenSnapshot := "gs://bucket/root/atespaces/ns/actors/" + someActorUID + "/snapshots/snap-1" control := &fakeGoldenControl{snapshot: goldenSnapshot} r := newTestTemplateReconciler(st, control) @@ -659,14 +658,14 @@ func TestReconcileOne_GoldenActorRequests(t *testing.T) { if got := created.GetMetadata().GetName(); got != testTemplateUID { t.Errorf("golden actor name = %q, want template UID %q", got, testTemplateUID) } - if got := created.GetMetadata().GetAtespace(); got != resources.GoldenActorAtespace { - t.Errorf("golden actor atespace = %q, want %q", got, resources.GoldenActorAtespace) + if got := created.GetMetadata().GetAtespace(); got != testTemplateRef.Atespace { + t.Errorf("golden actor atespace = %q, want %q", got, testTemplateRef.Atespace) } if got := created.GetActorTemplate().GetName(); got != testTemplateName { t.Errorf("golden actor template ref = %q, want %q", got, testTemplateName) } - if len(control.atespaceReqs) != 1 || control.atespaceReqs[0].GetAtespace().GetMetadata().GetName() != resources.GoldenActorAtespace { - t.Errorf("atespace ensure requests = %v, want one for %q", control.atespaceReqs, resources.GoldenActorAtespace) + if len(control.atespaceReqs) != 0 { + t.Errorf("unexpected atespace creation requests: %v", control.atespaceReqs) } if got := control.resumeReqs[0].GetActor().GetName(); got != testTemplateUID { t.Errorf("resumed actor = %q, want %q", got, testTemplateUID) @@ -674,6 +673,9 @@ func TestReconcileOne_GoldenActorRequests(t *testing.T) { if got := control.suspendReqs[0].GetActor().GetName(); got != testTemplateUID { t.Errorf("suspended actor = %q, want %q", got, testTemplateUID) } + if got := control.suspendReqs[0].GetOptions().GetMode(); got != ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS { + t.Errorf("golden suspend mode = %v, want %v", got, ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS) + } if st.storedStatus(t, testTemplateRef).GetGoldenSnapshotStatus().GetSnapshotTime() == nil { t.Error("stored snapshot_time is nil, want set") } @@ -888,7 +890,7 @@ func TestResync_FollowsPagination(t *testing.T) { } func TestReconcileOne_GoldenTagRecovery(t *testing.T) { - ref := &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: testTemplateUID} + ref := &ateapipb.ObjectRef{Atespace: testTemplateRef.Atespace, Name: testTemplateUID} completed := &ateapipb.Tag{ Metadata: &ateapipb.ResourceMetadata{Atespace: ref.Atespace, Name: ref.Name}, SourceActor: ref, @@ -977,8 +979,8 @@ func TestReconcileOne_GoldenTagConflict(t *testing.T) { templateUID string sourceActor *ateapipb.ObjectRef }{ - {name: "template", templateUID: "another-template", sourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: testTemplateUID}}, - {name: "actor name", templateUID: testTemplateUID, sourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: "another-actor"}}, + {name: "template", templateUID: "another-template", sourceActor: &ateapipb.ObjectRef{Atespace: testTemplateRef.Atespace, Name: testTemplateUID}}, + {name: "actor name", templateUID: testTemplateUID, sourceActor: &ateapipb.ObjectRef{Atespace: testTemplateRef.Atespace, Name: "another-actor"}}, {name: "actor atespace", templateUID: testTemplateUID, sourceActor: &ateapipb.ObjectRef{Atespace: "another-atespace", Name: testTemplateUID}}, } for _, tt := range tests { diff --git a/cmd/ateapi/internal/controlapi/workflow.go b/cmd/ateapi/internal/controlapi/workflow.go index 488135cbb9..832d616bc3 100644 --- a/cmd/ateapi/internal/controlapi/workflow.go +++ b/cmd/ateapi/internal/controlapi/workflow.go @@ -110,6 +110,7 @@ type ActorWorkflow struct { workerCache *workercache.Cache scheduler scheduling.Scheduler dialer *AteletDialer + workerPoolLister listersv1alpha1.WorkerPoolLister sandboxConfigLister listersv1alpha1.SandboxConfigLister storageClassLister storagev1listers.StorageClassLister instruments *Instruments @@ -129,6 +130,7 @@ func NewActorWorkflow( store actorWorkflowStore, workerCache *workercache.Cache, dialer *AteletDialer, + workerPoolLister listersv1alpha1.WorkerPoolLister, sandboxConfigLister listersv1alpha1.SandboxConfigLister, storageClassLister storagev1listers.StorageClassLister, instruments *Instruments, @@ -142,6 +144,7 @@ func NewActorWorkflow( workerCache: workerCache, scheduler: scheduling.New(workerCache), dialer: dialer, + workerPoolLister: workerPoolLister, sandboxConfigLister: sandboxConfigLister, storageClassLister: storageClassLister, instruments: instruments, diff --git a/cmd/ateapi/internal/controlapi/workflow_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_delete_test.go index 0b2b3fe84a..dedabf7635 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete_test.go @@ -334,7 +334,7 @@ func TestDeleteActor_CollectsInFlightSnapshotWithoutTemplate(t *testing.T) { ctx := context.Background() persistence := newTestPersistence(t) objects := objectstoretest.New() - w := NewActorWorkflow(persistence, nil, nil, nil, nil, nil, "", nil, time.Minute, objectstoreplugintest.ControlClient(objects)) + w := NewActorWorkflow(persistence, nil, nil, nil, nil, nil, nil, "", nil, time.Minute, objectstoreplugintest.ControlClient(objects)) actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"} actor := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ @@ -490,7 +490,7 @@ func TestDeleteActor_CollectsSnapshotsAfterWorkerDelete(t *testing.T) { } return status.Error(codes.Unavailable, "worker pod is gone") } - if _, err := actorWorkflow.SuspendActor(ctx, actorRef); apierror.Code(err) != codes.Unavailable { + if _, err := actorWorkflow.SuspendActor(ctx, actorRef, suspendSync); apierror.Code(err) != codes.Unavailable { t.Fatalf("SuspendActor err = %v, want Unavailable from the failed checkpoint", err) } diff --git a/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go b/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go index 20c9b04e9f..3fcf4e5f7f 100644 --- a/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go @@ -179,7 +179,7 @@ func TestReconcileAssignments_CrashesEarlierActors(t *testing.T) { actor := seedAPIActor(t, ctx, persistence, ateapipb.ActorState_ACTOR_STATE_RUNNING, func(a *ateapipb.Actor) { a.Status.LastAssignedGeneration = 1 a.Status.Snapshots = []*ateapipb.Snapshot{ - newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "partial-local-snapshot", ""), + newPausedSnapshot(t, 1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "partial-local-snapshot", ""), } }) assignAPIWorker(t, ctx, persistence, apiWorkerName, actor.GetMetadata().GetUid()) @@ -600,7 +600,7 @@ func TestAssignWorkerAttempt_StampsWorkerEpoch(t *testing.T) { } w := &ActorWorkflow{store: persistence, workerCache: wc, scheduler: scheduling.New(wc)} - tmpl := &ateapipb.ActorTemplate{SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor"}} + tmpl := &ateapipb.ActorTemplate{} stored, _, err := w.assignWorkerAttempt(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, tmpl) if err != nil { t.Fatalf("assignWorkerAttempt: %v", err) diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index a28938a7be..ad8e18048b 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -86,7 +86,7 @@ func (w *ActorWorkflow) ResumeActor(ctx context.Context, actorRef resources.Acto return } w.instruments.recordLifecycleOp(ctx, ateattr.OperationResume, start, err, - lifecycleOpAttrs(actor, actorTemplate, tele.SnapshotKind, tele.WireFidelity)...) + lifecycleOpAttrs(actor, tele.SnapshotKind, tele.WireFidelity)...) }() // Routed requests call ResumeActor even when the actor is already running. @@ -141,20 +141,6 @@ func (w *ActorWorkflow) ResumeActor(ctx context.Context, actorRef resources.Acto return actor, true, nil } -// validateGoldenSnapshotFidelity rejects a golden snapshot that does not carry -// the guest state (memory + fs delta) a restore needs. Golden actors always -// commit MEMORY (preferredFidelity), so this only trips on a golden snapshot -// whose record drifted — surface a clear error instead of shipping a restore -// request atelet would reject (or that would boot an empty guest). -func validateGoldenSnapshotFidelity(snapshot *ateapipb.Snapshot) error { - if fidelity := snapshot.GetFidelity(); fidelity != ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { - return apierror.FailedPrecondition( - "ActorTemplate golden snapshot %q was taken with fidelity %s, not MEMORY; regenerate the golden snapshot", - snapshot.GetDurableSnapshot().GetObject().GetSnapshotUri(), fidelity) - } - return nil -} - // loadActorForResume fetches the current actor record and its template, and // resolves the boot source for the pending restore. func (w *ActorWorkflow) loadActorForResume(ctx context.Context, actorRef resources.ActorRef) (_ *ateapipb.Actor, _ *ateapipb.ActorTemplate, _ resumeSnapshotSource, err error) { @@ -319,12 +305,12 @@ func (w *ActorWorkflow) ensureWorkerAssigned(ctx context.Context, actorRef resou return assignedActor, assignedWorker, nil case ctxErr != nil: if attempted { - w.recordExhaustedAssignment(ctx, lastAttempt, actorTemplate, ctxErr) + w.recordExhaustedAssignment(ctx, lastAttempt, actor, ctxErr) } return nil, nil, err case exhausted: err = apierror.Aborted("concurrent update conflict, please retry: %w", store.ErrVersionConflict) - w.recordExhaustedAssignment(ctx, lastAttempt, actorTemplate, err) + w.recordExhaustedAssignment(ctx, lastAttempt, actor, err) return nil, nil, err default: // assignWorkerAttempt recorded this one itself. @@ -345,12 +331,8 @@ var assignmentBackoff = wait.Backoff{ // recordExhaustedAssignment writes the one record of an assignment loop that // ended without assigning. No worker was assigned, so no pool is named. -func (w *ActorWorkflow) recordExhaustedAssignment(ctx context.Context, elapsed time.Duration, actorTemplate *ateapipb.ActorTemplate, err error) { - class := "" - if actorTemplate != nil { - class = actorTemplate.GetSandboxConfig().GetSandboxClass() - } - w.instruments.recordSchedulerAssignment(ctx, elapsed, ateattr.SchedulerOutcomeError, "", "", class, err) +func (w *ActorWorkflow) recordExhaustedAssignment(ctx context.Context, elapsed time.Duration, actor *ateapipb.Actor, err error) { + w.instruments.recordSchedulerAssignment(ctx, elapsed, ateattr.SchedulerOutcomeError, "", "", snapshotSandboxClass(actor), err) } // validateAssignedWorker checks a RESUMING actor's persisted assignment @@ -485,9 +467,6 @@ func (w *ActorWorkflow) assignWorkerAttempt(ctx context.Context, actorRef resour poolNamespace := "" pool := "" class := "" - if actorTemplate != nil { - class = actorTemplate.GetSandboxConfig().GetSandboxClass() - } defer func() { if schedulerRecordable(err) { w.instruments.recordSchedulerAssignment(ctx, time.Since(start), outcome, poolNamespace, pool, class, err) @@ -499,6 +478,7 @@ func (w *ActorWorkflow) assignWorkerAttempt(ctx context.Context, actorRef resour return nil, nil, err } snapRuntime := constraints.RequiredRuntime + class = constraints.SandboxClass relaxable := canRelaxRuntime(actor, constraints) assignedWorker, err := w.workerHoldingStaleClaim(ctx, actor, constraints, relaxable) @@ -591,6 +571,7 @@ func (w *ActorWorkflow) assignWorkerAttempt(ctx context.Context, actorRef resour } poolNamespace = bound.GetWorkerNamespace() pool = bound.GetWorkerPool() + class = bound.GetSandboxClass() outcome = ateattr.SchedulerOutcomeAssigned logActorStateChanged(ctx, storedActor, ateattr.OperationResume) return storedActor, bound, nil @@ -616,18 +597,34 @@ func workerAssignmentFrom(w *ateapipb.Worker, snapRuntime hardware.Requirement) // changes its default does not relabel state captured on the old one. // // TODO: Workers report restorable_runtimes, so a match on one records - // that older runtime here, but resolveSandboxAssets still ships the - // template's SandboxConfig default version and atelet restores with - // the assets in the request: the restore runs on the newer version and - // fails, or its next checkpoint is stamped with a runtime that did not - // produce it. A follow-up replaces the assets in Run and Restore with - // this sandbox_runtime, which atelet resolves to the matching version - // of the worker pool's SandboxConfig; until it lands, keep a config's + // that older runtime here and ensureAteletRestored sends it on Run and + // Restore, but atelet still restores with the assets in the request + // (the worker pool's SandboxConfig default version): the restore runs on + // the newer version and fails, or its next checkpoint is stamped with a + // runtime that did not produce it. A follow-up resolves the matching + // version of the worker pool's SandboxConfig from sandbox_runtime in + // atelet and removes sandbox_assets; until it lands, keep a config's // default version the only enabled one. SandboxRuntime: proto.CloneOf(snapRuntime.RuntimeFor(w.GetStatus())), } } +// toAteletSandboxRuntime converts the control plane's SandboxRuntime to +// atelet's, which it mirrors field for field. +func toAteletSandboxRuntime(in *ateapipb.SandboxRuntime) *ateletpb.SandboxRuntime { + if in == nil { + return nil + } + out := &ateletpb.SandboxRuntime{SandboxClass: in.GetSandboxClass()} + if v := in.GetVersion(); v != nil { + out.Version = &ateletpb.VersionedSandboxCompat{SchemaVersion: v.GetSchemaVersion()} + for _, a := range v.GetAttributes() { + out.Version.Attributes = append(out.Version.Attributes, &ateletpb.AttributeEntry{Key: a.GetKey(), Value: a.GetValue()}) + } + } + return out +} + // actorResourceLimits returns the actor's declared CPU (millicores) and memory // (bytes) limits from its ActorTemplate, or 0 for a dimension the template did // not set. These size the sandbox, which takes the two scalars the runtimes @@ -656,11 +653,15 @@ func schedulingConstraints(actor *ateapipb.Actor, tmpl *ateapipb.ActorTemplate) return scheduling.Constraints{}, fmt.Errorf("invalid template resource limits: %w", err) } var requiredNodes []string + // A paused Actor stays pinned to its node until atelet detaches the local + // snapshot, so it cannot resume while that node is full. + // TODO: when the node has no room, detach the Actor there and resume it on + // another node from the durable snapshot. if node := actor.GetStatus().GetAssignedNode(); node != "" { requiredNodes = []string{node} } c := scheduling.Constraints{ - SandboxClass: tmpl.GetSandboxConfig().GetSandboxClass(), + SandboxClass: snapshotSandboxClass(actor), RequiredRuntime: hardware.NewRequirement(requiredSnapshotRuntime(actor, tmpl)), ActorSelector: labels.SelectorFromSet(labels.Set(actor.GetWorkerSelector().GetMatchLabels())), RequiredNodes: requiredNodes, @@ -672,6 +673,22 @@ func schedulingConstraints(actor *ateapipb.Actor, tmpl *ateapipb.ActorTemplate) return c, nil } +// snapshotSandboxClass returns the sandbox class of the snapshot ResumeActor +// will restore from, the actor's local snapshot or else its latest durable +// one (a golden tag borrowed at creation included), or "" when the actor has +// none and cold-boots. Unlike requiredSnapshotRuntime it does not depend on +// fidelity: a worker of another class cannot read the snapshot's files even +// for a volumes-only restore. +func snapshotSandboxClass(actor *ateapipb.Actor) string { + if localSnap := findLatestLocalSnapshot(actor.GetStatus()); localSnap != nil { + return localSnap.GetSandboxRuntime().GetSandboxClass() + } + if snap := findLatestDurableSnapshot(actor.GetStatus(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); snap != nil { + return snap.GetSandboxRuntime().GetSandboxClass() + } + return "" +} + // requiredSnapshotRuntime returns the SandboxRuntime stamped on the snapshot // that ResumeActor will restore guest state from, or nil when the resume does // not restore guest state (cold boot, VOLUMES fidelity, or replaced template). @@ -704,7 +721,8 @@ func requiredSnapshotRuntime(actor *ateapipb.Actor, tmpl *ateapipb.ActorTemplate // runtime-compatible worker has capacity. A local pause checkpoint is not // relaxed: the resume stays pinned to its node, the local restore has no // fidelity downgrade, and nothing invalidates the checkpoint files if the actor -// boots elsewhere. Suspending the actor uploads it and lifts the pin. +// boots elsewhere. The pin lifts once atelet uploads the checkpoint and +// detaches it from the node. func canRelaxRuntime(actor *ateapipb.Actor, constraints scheduling.Constraints) bool { if constraints.RequiredRuntime.Runtime() == nil { return false @@ -791,15 +809,16 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou return tele, err } - // The sandbox binaries and pause image come from the template's - // SandboxConfig on every path, restores included. - sandboxAssets, err := resolveSandboxAssets(w.sandboxConfigLister, actorTemplate.GetSandboxConfig()) + // The sandbox binaries and pause image come from the assigned worker's + // pool SandboxConfig on every path, restores included. + sandboxAssets, err := resolveSandboxAssets(w.workerPoolLister, w.sandboxConfigLister, assignment) if err != nil { return tele, fmt.Errorf("while resolving sandbox assets: %w", err) } // The assignment is what the next checkpoint is stamped from, so it is also // what decides how this restore runs. + sandboxRuntime := toAteletSandboxRuntime(assignment.GetSandboxRuntime()) runtimeIncompatible := src.RequiredRuntime != nil && !hardware.Matches(assignment.GetSandboxRuntime(), src.RequiredRuntime) if localSnap := findLatestLocalSnapshot(actor.GetStatus()); localSnap != nil { slog.InfoContext(ctx, "Actor has snapshot; Restoring from snapshot") @@ -818,11 +837,17 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou Generation: localSnap.GetGeneration(), Fidelity: fidelity, }, - SandboxAssets: sandboxAssets, - ActorUid: actor.GetMetadata().Uid, - EgressGateway: egressGateway, - CpuMilli: cpuMilli, - MemoryBytes: memBytes, + SandboxAssets: sandboxAssets, + SandboxRuntime: sandboxRuntime, + ActorUid: actor.GetMetadata().Uid, + EgressGateway: egressGateway, + CpuMilli: cpuMilli, + MemoryBytes: memBytes, + } + // atelet uploads a pause's snapshot in the background. Once that upload + // is recorded, atelet can download it if its local copy is gone. + if durable := localSnap.GetDurableSnapshot(); durable.GetStatus() == ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + req.Snapshot.Object = &ateletpb.ObjectStorage{SnapshotUri: durable.GetObject().GetSnapshotUri()} } tele.WireFidelity = ateattr.SnapshotFidelityValue(fidelity) @@ -860,11 +885,12 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou SnapshotUri: src.SnapshotURI.String(), }, }, - SandboxAssets: sandboxAssets, - ActorUid: actor.GetMetadata().Uid, - EgressGateway: egressGateway, - CpuMilli: cpuMilli, - MemoryBytes: memBytes, + SandboxAssets: sandboxAssets, + SandboxRuntime: sandboxRuntime, + ActorUid: actor.GetMetadata().Uid, + EgressGateway: egressGateway, + CpuMilli: cpuMilli, + MemoryBytes: memBytes, } if _, err = client.Restore(ctx, req); err != nil { return tele, handleAteletError(ctx, w.store, actorRef, ateattr.OperationResume, "Restore", false, err) @@ -881,6 +907,7 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou ActorTemplateAtespace: actor.GetActorTemplate().GetAtespace(), ActorTemplateName: actor.GetActorTemplate().GetName(), SandboxAssets: sandboxAssets, + SandboxRuntime: sandboxRuntime, Spec: workloadSpec, ActorUid: actor.GetMetadata().Uid, EgressGateway: egressGateway, diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index a355de1958..fde73568e0 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -129,7 +129,7 @@ func TestFinalizeRunning_CommitsRunning(t *testing.T) { AssignedNode: "node-1", LastAssignedGeneration: 1, Snapshots: []*ateapipb.Snapshot{ - newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-1", "node-1"), + newPausedSnapshot(t, 1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-1", "node-1"), }, }, }) @@ -164,7 +164,7 @@ func TestAssignWorkerAttempt_MissingSelectedWorkerIsRetried(t *testing.T) { actor, wc := seedAssignFixture(t, ctx, persistence) st := &bindErrorStore{Interface: persistence, err: store.ErrNotFound} w := &ActorWorkflow{store: st, workerCache: wc, scheduler: scheduling.New(wc)} - tmpl := &ateapipb.ActorTemplate{SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor"}} + tmpl := &ateapipb.ActorTemplate{} _, _, err := w.assignWorkerAttempt(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, tmpl) if !errors.Is(err, store.ErrVersionConflict) { @@ -185,7 +185,7 @@ func TestEnsureWorkerAssigned_ConflictExhaustionIsRetryable(t *testing.T) { actor, wc := seedAssignFixture(t, ctx, persistence) st := &bindErrorStore{Interface: persistence, err: store.ErrVersionConflict} w := &ActorWorkflow{store: st, workerCache: wc, scheduler: scheduling.New(wc)} - tmpl := &ateapipb.ActorTemplate{SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor"}} + tmpl := &ateapipb.ActorTemplate{} _, _, err := w.ensureWorkerAssigned(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, tmpl) if !errors.Is(err, store.ErrVersionConflict) { @@ -227,9 +227,7 @@ func TestAssignWorkerAttempt_SkipsWorkerAssignedInOtherAtespace(t *testing.T) { actor := &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "shared", Uid: "actor-uid"}, } - tmpl := &ateapipb.ActorTemplate{ - SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor"}, - } + tmpl := &ateapipb.ActorTemplate{} _, _, err := w.assignWorkerAttempt(ctx, resources.ActorRef{Atespace: "team-a", Name: "shared"}, actor, tmpl) if apierror.Code(err) != codes.ResourceExhausted { t.Fatalf("assignWorkerAttempt() error = %v, want ResourceExhausted (no worker has room)", err) @@ -257,15 +255,15 @@ func TestAssignWorkerAttempt_ReleasesIneligibleStaleWorker(t *testing.T) { Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}, }) - // stale-pod is claimed by this actor from a failed attempt but its sandbox - // class no longer matches the template; free-pod is eligible and free. + // stale-pod is claimed by this actor from a failed attempt but its labels + // do not match the template's selector; free-pod is eligible and free. stale := &ateapipb.Worker{ Metadata: &ateapipb.ResourceMetadata{Name: testWorkerUID("stale-pod")}, WorkerNamespace: "worker-ns", WorkerPool: "pool-a", WorkerPod: "stale-pod", WorkerPodUid: testWorkerUID("stale-pod"), - SandboxClass: "microvm", + SandboxClass: "gvisor", Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}}, } free := &ateapipb.Worker{ @@ -275,6 +273,7 @@ func TestAssignWorkerAttempt_ReleasesIneligibleStaleWorker(t *testing.T) { WorkerPod: "free-pod", WorkerPodUid: testWorkerUID("free-pod"), SandboxClass: "gvisor", + Labels: map[string]string{"pool": "b"}, Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}}, } for _, w := range []*ateapipb.Worker{stale, free} { @@ -296,7 +295,7 @@ func TestAssignWorkerAttempt_ReleasesIneligibleStaleWorker(t *testing.T) { w := &ActorWorkflow{store: persistence, workerCache: wc, scheduler: scheduling.New(wc)} tmpl := &ateapipb.ActorTemplate{ - SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor"}, + WorkerSelector: &ateapipb.Selector{MatchLabels: map[string]string{"pool": "b"}}, } _, worker, err := w.assignWorkerAttempt(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, tmpl) if err != nil { @@ -372,9 +371,7 @@ func TestAssignWorkerAttempt_RetryAfterConflictPicksFreshWorker(t *testing.T) { } w := &ActorWorkflow{store: persistence, workerCache: wc, scheduler: scheduling.New(wc)} - tmpl := &ateapipb.ActorTemplate{ - SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor"}, - } + tmpl := &ateapipb.ActorTemplate{} _, worker, err := w.assignWorkerAttempt(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, tmpl) if err != nil { t.Fatalf("assignWorkerAttempt() on retry = %v, want nil (must re-pick a free worker)", err) @@ -513,9 +510,7 @@ func TestAssignWorkerAttempt_ConflictRefreshesActor(t *testing.T) { }} w := &ActorWorkflow{store: st, workerCache: wc, scheduler: scheduling.New(wc)} - tmpl := &ateapipb.ActorTemplate{ - SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor"}, - } + tmpl := &ateapipb.ActorTemplate{} refreshed, _, err := w.assignWorkerAttempt(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, tmpl) if tc.wantRetry { @@ -754,6 +749,7 @@ func TestValidateAssignedWorker(t *testing.T) { // is gone. workerStatus *ateapipb.WorkerStatus sandboxClass string + tmplSelector map[string]string assignment *ateapipb.ActorAssignment // wantCode is codes.OK when validateAssignedWorker must return nil. wantCode codes.Code @@ -833,7 +829,8 @@ func TestValidateAssignedWorker(t *testing.T) { { name: "releases own ineligible worker and crashes actor", workerStatus: activeStatus, - sandboxClass: "microvm", + sandboxClass: "gvisor", + tmplSelector: map[string]string{"pool": "other"}, assignment: ownAssignment, wantCode: codes.Aborted, wantActorState: ateapipb.ActorState_ACTOR_STATE_CRASHED, @@ -886,7 +883,10 @@ func TestValidateAssignedWorker(t *testing.T) { }, }, } - tmpl := &ateapipb.ActorTemplate{SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor"}} + tmpl := &ateapipb.ActorTemplate{} + if tt.tmplSelector != nil { + tmpl.WorkerSelector = &ateapipb.Selector{MatchLabels: tt.tmplSelector} + } _, err := w.validateAssignedWorker(ctx, resources.ActorRef{Atespace: "team-a", Name: "shared"}, resumingActor, tmpl) if got := apierror.Code(err); got != tt.wantCode { t.Fatalf("apierror.Code(err) = %v, want %v (err: %v)", got, tt.wantCode, err) @@ -1066,6 +1066,10 @@ type capturingAtelet struct { // whatever else happens while the workload checkpoints. Its error fails // the RPC. onCheckpoint func(ctx context.Context) error + // checkpointState is the snapshot state Checkpoint returns. + checkpointState *ateletpb.ActorSnapshotState + // promoteState is the snapshot state PromoteSnapshot returns. + promoteState *ateletpb.ActorSnapshotState } func (f *capturingAtelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (*ateletpb.RestoreResponse, error) { @@ -1093,7 +1097,9 @@ func (f *capturingAtelet) Checkpoint(ctx context.Context, req *ateletpb.Checkpoi return nil, err } } - return &ateletpb.CheckpointResponse{}, nil + f.mu.Lock() + defer f.mu.Unlock() + return &ateletpb.CheckpointResponse{State: proto.CloneOf(f.checkpointState)}, nil } func (f *capturingAtelet) PromoteSnapshot(ctx context.Context, req *ateletpb.PromoteSnapshotRequest) (*ateletpb.PromoteSnapshotResponse, error) { @@ -1101,7 +1107,7 @@ func (f *capturingAtelet) PromoteSnapshot(ctx context.Context, req *ateletpb.Pro defer f.mu.Unlock() f.promote = proto.Clone(req).(*ateletpb.PromoteSnapshotRequest) f.promotes++ - return &ateletpb.PromoteSnapshotResponse{}, nil + return &ateletpb.PromoteSnapshotResponse{State: proto.CloneOf(f.promoteState)}, nil } // calls returns how many Checkpoint and PromoteSnapshot RPCs were received. @@ -1208,7 +1214,9 @@ func newWireCaptureWorkflow(t *testing.T, persistence store.Interface) (*ActorWo }, }}) - return &ActorWorkflow{store: persistence, dialer: dialer, sandboxConfigLister: lister, workflowDeadline: time.Minute}, fake + pools := workerPoolListerFor(t, []*atev1alpha1.WorkerPool{resolvedWorkerPool("worker-ns", "pool", "gvisor")}) + + return &ActorWorkflow{store: persistence, dialer: dialer, workerPoolLister: pools, sandboxConfigLister: lister, workflowDeadline: time.Minute}, fake } // TestResumeActor_AteletWireRequest is the characteristic test for the @@ -1226,7 +1234,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { const malformedURI = "not-a-valid-snapshot-uri" actorURI := someActorSnapshotURI(t, testStorageLocation, "team-a", durableSnapshotName) - goldenURI := someActorSnapshotURI(t, "gs://bucket/golden-root", "ate-golden", "golden-1") + goldenURI := someActorSnapshotURI(t, "gs://bucket/golden-root", "team-a", "golden-1") fullScope := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY rootfsScope := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS @@ -1237,7 +1245,17 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { return newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, fidelity, "", durableSnapshotName, uri, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) } localSnap := func(name string, fidelity ateapipb.SnapshotFidelity) *ateapipb.Snapshot { - return newLocalSnapshot(1, fidelity, "", name, "node-1") + return newPausedSnapshot(t, 1, fidelity, "", name, "node-1") + } + // pauseUploadURI is where atelet uploads the pause snapshot. + pauseUploadURI := someActorSnapshotURI(t, testStorageLocation, "team-a", localSnapshotName) + uploadedLocalSnap := func(name string, fidelity ateapipb.SnapshotFidelity) *ateapipb.Snapshot { + snap := localSnap(name, fidelity) + snap.DurableSnapshot = &ateapipb.SnapshotStorage{ + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: pauseUploadURI}, + } + return snap } // actorSeed is the actor status a row persists before resuming. @@ -1256,14 +1274,15 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { sourceTag *ateapipb.ObjectRef // assignedRuntime seeds WorkerAssignment.SandboxRuntime. assignedRuntime *ateapipb.SandboxRuntime + // poolConfigName, when set, is the SandboxConfig the assigned + // worker's pool resolves instead of the "gvisor" config the + // workflow's lister serves. + poolConfigName string } // templateSeed is the ActorTemplate configuration a row persists. type templateSeed struct { // golden seeds the template's golden tag snapshot. golden *ateapipb.Snapshot - // configName is the SandboxConfig the template names; "" means the - // "gvisor" config the workflow's lister serves. - configName string } // restoreWant pins the request atelet receives. On a non-OK code neither // Restore nor Run may reach atelet; with run set the Run RPC (cold boot) @@ -1274,7 +1293,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { run bool // snapshotName is Snapshot.SnapshotUuid and snapshotURI is // Snapshot.Object.SnapshotUri. Both are asserted on every Restore, so - // a local row also pins that no durable copy is sent. + // a local row also pins which durable copy, if any, is sent. snapshotName string snapshotURI string scope ateletpb.SnapshotFidelity @@ -1282,6 +1301,22 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { amd64RT := testSandboxRuntime() arm64RT := testRuntime("arm64", "") + wantAteletRuntime := map[*ateapipb.SandboxRuntime]*ateletpb.SandboxRuntime{ + amd64RT: { + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateletpb.AttributeEntry{{Key: "architecture", Value: "amd64"}}, + }, + }, + arm64RT: { + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateletpb.AttributeEntry{{Key: "architecture", Value: "arm64"}}, + }, + }, + } stampedExtSnap := func(owner ateapipb.SnapshotOwner, uri string, fidelity ateapipb.SnapshotFidelity, rt *ateapipb.SandboxRuntime) *ateapipb.Snapshot { snap := newDurableSnapshot(1, owner, fidelity, "", durableSnapshotName, uri, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) snap.SandboxRuntime = rt @@ -1295,8 +1330,9 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { want restoreWant }{ { - name: "01 nothing to restore cold-boots from the spec", - want: restoreWant{run: true}, + name: "01 nothing to restore cold-boots from the spec", + actor: actorSeed{assignedRuntime: amd64RT}, + want: restoreWant{run: true}, }, { name: "02 inherited golden snapshot restores in Full", @@ -1425,7 +1461,8 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { { name: "13 Full pause snapshot restores locally as Full", actor: actorSeed{ - localSnapshot: localSnap(localSnapshotName, fullScope), + localSnapshot: localSnap(localSnapshotName, fullScope), + assignedRuntime: amd64RT, }, want: restoreWant{ snapshotName: localSnapshotName, @@ -1499,17 +1536,16 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, }, { - // Restores take their sandbox from the template, like cold boots, - // so an unresolvable SandboxConfig stops them before atelet. - name: "19 durable snapshot restore with a missing SandboxConfig is rejected", - actor: actorSeed{externalSnapshot: extSnap(actorURI, fullScope)}, - tmpl: templateSeed{configName: "missing"}, + // Restores take their sandbox from the assigned worker's pool, + // like cold boots, so an unresolvable SandboxConfig stops them + // before atelet. + name: "19 durable snapshot restore with a missing pool SandboxConfig is rejected", + actor: actorSeed{externalSnapshot: extSnap(actorURI, fullScope), poolConfigName: "missing"}, want: restoreWant{code: codes.FailedPrecondition}, }, { - name: "20 local snapshot restore with a missing SandboxConfig is rejected", - actor: actorSeed{localSnapshot: localSnap(localSnapshotName, fullScope)}, - tmpl: templateSeed{configName: "missing"}, + name: "20 local snapshot restore with a missing pool SandboxConfig is rejected", + actor: actorSeed{localSnapshot: localSnap(localSnapshotName, fullScope), poolConfigName: "missing"}, want: restoreWant{code: codes.FailedPrecondition}, }, { @@ -1566,10 +1602,24 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, }, + { + // atelet uploads a pause's snapshot in the background. Once that + // upload is recorded, a local restore sends it too, so atelet can + // download the snapshot if its local copy is gone. + name: "25 uploaded pause snapshot restores locally and sends its upload", + actor: actorSeed{ + localSnapshot: uploadedLocalSnap(localSnapshotName, fullScope), + }, + want: restoreWant{ + snapshotName: localSnapshotName, + snapshotURI: pauseUploadURI, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + }, + }, } // Every request atelet receives, Run or Restore, carries the sandbox the - // template's SandboxConfig resolves to. + // assigned worker's pool SandboxConfig resolves to. wantSandboxAssets := sandboxAssetsProto( &atev1alpha1.SandboxConfig{Spec: atev1alpha1.SandboxConfigSpec{SandboxClass: atev1alpha1.SandboxClassGvisor}}, &atev1alpha1.SandboxVersionConfig{Name: "v1", PauseImage: "pause@sha256:abc", Assets: testAssets()}, @@ -1582,19 +1632,14 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { w, atelet := newWireCaptureWorkflow(t, persistence) storetest.MustCreateAtespace(t, ctx, persistence, "ns") - configName := tt.tmpl.configName - if configName == "" { - configName = "gvisor" + if tt.actor.poolConfigName != "" { + w.workerPoolLister = workerPoolListerFor(t, []*atev1alpha1.WorkerPool{resolvedWorkerPool("worker-ns", "pool", tt.actor.poolConfigName)}) } tmpl := &ateapipb.ActorTemplate{ Metadata: &ateapipb.ResourceMetadata{Atespace: "ns", Name: "tmpl1"}, SnapshotConfig: &ateapipb.SnapshotConfig{ StorageLocation: testStorageLocation, }, - SandboxConfig: &ateapipb.SandboxConfig{ - SandboxClass: "gvisor", - ConfigName: configName, - }, } if tt.tmpl.golden != nil { tmpl.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ @@ -1672,6 +1717,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { return } + wantSandboxRuntime := wantAteletRuntime[tt.actor.assignedRuntime] if tt.want.run { if run == nil || restore != nil { t.Fatalf("atelet requests = (restore=%v, run=%v), want exactly one Run", restore, run) @@ -1679,6 +1725,9 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { if !proto.Equal(run.GetSandboxAssets(), wantSandboxAssets) { t.Errorf("run SandboxAssets = %v, want %v", run.GetSandboxAssets(), wantSandboxAssets) } + if !proto.Equal(run.GetSandboxRuntime(), wantSandboxRuntime) { + t.Errorf("run SandboxRuntime = %v, want %v", run.GetSandboxRuntime(), wantSandboxRuntime) + } return } if restore == nil || run != nil { @@ -1687,6 +1736,9 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { if !proto.Equal(restore.GetSandboxAssets(), wantSandboxAssets) { t.Errorf("restore SandboxAssets = %v, want %v", restore.GetSandboxAssets(), wantSandboxAssets) } + if !proto.Equal(restore.GetSandboxRuntime(), wantSandboxRuntime) { + t.Errorf("restore SandboxRuntime = %v, want %v", restore.GetSandboxRuntime(), wantSandboxRuntime) + } if got := restore.GetSnapshot().GetSnapshotUuid(); got != tt.want.snapshotName { t.Errorf("Snapshot.SnapshotUuid = %q, want %q", got, tt.want.snapshotName) } @@ -1707,14 +1759,17 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { } } -// publishContextVolumePlugin hands back a fixed publish context, standing in -// for a driver whose node plugin needs attachment metadata. +// publishContextVolumePlugin records AttachVolume requests and hands back a +// fixed publish context, standing in for a driver whose node plugin needs +// attachment metadata. type publishContextVolumePlugin struct { volume.VolumePluginControlPlane publishContext map[string]string + attachCalls []volume.AttachVolumeRequest } func (p *publishContextVolumePlugin) AttachVolume(ctx context.Context, req volume.AttachVolumeRequest) (volume.AttachVolumeResponse, error) { + p.attachCalls = append(p.attachCalls, req) return volume.AttachVolumeResponse{PublishContext: p.publishContext}, nil } @@ -1740,10 +1795,11 @@ func TestEnsureVolumesAttached_ReturnsPublishContext(t *testing.T) { t.Fatalf("GetActor: %v", err) } + plugin := &publishContextVolumePlugin{publishContext: map[string]string{"devicePath": "/dev/xvdba"}} w := &ActorWorkflow{ store: persistence, pluginRegistry: &mockPluginRegistry{plugins: map[string]volume.VolumePluginControlPlane{ - "mock": &publishContextVolumePlugin{publishContext: map[string]string{"devicePath": "/dev/xvdba"}}, + "mock": plugin, }}, } worker := &ateapipb.Worker{NodeName: "node-1"} @@ -1765,6 +1821,10 @@ func TestEnsureVolumesAttached_ReturnsPublishContext(t *testing.T) { if diff := cmp.Diff(want, got); diff != "" { t.Errorf("publish contexts mismatch (-want +got):\n%s", diff) } + wantCalls := []volume.AttachVolumeRequest{{VolumeID: "storage-mounted", Node: "node-1"}} + if diff := cmp.Diff(wantCalls, plugin.attachCalls); diff != "" { + t.Errorf("attachCalls mismatch (-want +got):\n%s", diff) + } stored, err := persistence.GetActor(ctx, actorRef) if err != nil { @@ -2036,12 +2096,6 @@ func newContendedWorkflow(t *testing.T, sched *filledUpScheduler) (*ActorWorkflo return w, actor, reader } -func gvisorTemplate() *ateapipb.ActorTemplate { - return &ateapipb.ActorTemplate{ - SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor"}, - } -} - // TestEnsureWorkerAssigned_ContentionRecordsOneObservation holds the note on // ate.scheduler.assignment.duration: ateapi records only the last attempt. A // resume that loses a race for the last slot on a worker retries, and the @@ -2054,7 +2108,7 @@ func TestEnsureWorkerAssigned_ContentionRecordsOneObservation(t *testing.T) { } w, actor, reader := newContendedWorkflow(t, sched) - _, worker, err := w.ensureWorkerAssigned(context.Background(), resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, gvisorTemplate()) + _, worker, err := w.ensureWorkerAssigned(context.Background(), resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, &ateapipb.ActorTemplate{}) if err != nil { t.Fatalf("ensureWorkerAssigned: %v", err) } @@ -2088,7 +2142,7 @@ func TestEnsureWorkerAssigned_ExhaustedBackoffRecordsOneError(t *testing.T) { } w, actor, reader := newContendedWorkflow(t, sched) - _, _, err := w.ensureWorkerAssigned(context.Background(), resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, gvisorTemplate()) + _, _, err := w.ensureWorkerAssigned(context.Background(), resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, &ateapipb.ActorTemplate{}) if err == nil { t.Fatal("ensureWorkerAssigned succeeded, want a failure after the backoff ended") } @@ -2157,7 +2211,7 @@ func TestEnsureWorkerAssigned_CancelledDuringBackoffRecordsOne(t *testing.T) { } defer cancel() - _, _, err := w.ensureWorkerAssigned(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, gvisorTemplate()) + _, _, err := w.ensureWorkerAssigned(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, &ateapipb.ActorTemplate{}) if !errors.Is(err, context.Canceled) { t.Fatalf("ensureWorkerAssigned error = %v, want context.Canceled", err) } @@ -2187,7 +2241,7 @@ func TestEnsureWorkerAssigned_CancelledBeforeFirstAttemptRecordsNothing(t *testi ctx, cancel := context.WithCancel(context.Background()) cancel() - _, _, err := w.ensureWorkerAssigned(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, gvisorTemplate()) + _, _, err := w.ensureWorkerAssigned(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, &ateapipb.ActorTemplate{}) if !errors.Is(err, context.Canceled) { t.Fatalf("ensureWorkerAssigned error = %v, want context.Canceled", err) } @@ -2219,7 +2273,7 @@ func TestEnsureWorkerAssigned_CancelledInLastSleepRecordsOne(t *testing.T) { } defer cancel() - _, _, err := w.ensureWorkerAssigned(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, gvisorTemplate()) + _, _, err := w.ensureWorkerAssigned(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, &ateapipb.ActorTemplate{}) if !errors.Is(err, context.Canceled) { t.Fatalf("ensureWorkerAssigned error = %v, want context.Canceled", err) } @@ -2248,7 +2302,7 @@ func TestEnsureWorkerAssigned_ContextEndsInsideAttemptRecordsOne(t *testing.T) { } w, actor, reader := newContendedWorkflow(t, sched) - _, _, err := w.ensureWorkerAssigned(context.Background(), resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, gvisorTemplate()) + _, _, err := w.ensureWorkerAssigned(context.Background(), resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, &ateapipb.ActorTemplate{}) if !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("ensureWorkerAssigned error = %v, want context.DeadlineExceeded", err) } @@ -2267,13 +2321,13 @@ func TestEnsureWorkerAssigned_ContextEndsInsideAttemptRecordsOne(t *testing.T) { } // testRuntime is a gvisor SandboxRuntime for arch under compatibility schema -// v1. A non-empty assetHash adds a gvisor_asset_hash attribute, so two +// v1. A non-empty bundleHash adds a sandbox_bundle_hash attribute, so two // runtimes can share an architecture and still differ, as two versions of // one SandboxConfig do. -func testRuntime(arch, assetHash string) *ateapipb.SandboxRuntime { +func testRuntime(arch, bundleHash string) *ateapipb.SandboxRuntime { attrs := []*ateapipb.AttributeEntry{{Key: "architecture", Value: arch}} - if assetHash != "" { - attrs = append(attrs, &ateapipb.AttributeEntry{Key: "gvisor_asset_hash", Value: assetHash}) + if bundleHash != "" { + attrs = append(attrs, &ateapipb.AttributeEntry{Key: hardware.AttrSandboxBundleHash, Value: bundleHash}) } return &ateapipb.SandboxRuntime{ SandboxClass: "gvisor", @@ -2320,6 +2374,35 @@ func durableMemorySnapshot(t *testing.T, runtime *ateapipb.SandboxRuntime) *atea return snap } +// seedMicroVMWorker stores an ACTIVE microvm worker on node with room for one +// actor, reporting a microvm default runtime. +func seedMicroVMWorker(t *testing.T, ctx context.Context, st store.Interface, pod, node string) *ateapipb.Worker { + t.Helper() + worker, err := st.CreateWorker(ctx, &ateapipb.Worker{ + Metadata: &ateapipb.ResourceMetadata{Name: testWorkerUID(pod)}, + WorkerNamespace: "worker-ns", + WorkerPool: "vm-pool", + WorkerPod: pod, + WorkerPodUid: testWorkerUID(pod), + SandboxClass: "microvm", + NodeName: node, + Status: &ateapipb.WorkerStatus{ + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + DefaultRuntime: &ateapipb.SandboxRuntime{ + SandboxClass: "microvm", + Version: &ateapipb.VersionedSandboxCompat{SchemaVersion: "v1", Attributes: []*ateapipb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + }}, + }, + }, + }) + if err != nil { + t.Fatalf("CreateWorker(%s): %v", pod, err) + } + return worker +} + // seedActorWithSnapshot stores actor team-a/id1 in state holding snap as its // only snapshot. node is the AssignedNode a PAUSED actor keeps, which pins // its resume to that node. @@ -2392,7 +2475,7 @@ func TestAssignWorkerAttempt_PrefersRuntimeMatchingWorkerAndFallsBack(t *testing seedRuntimeWorker(t, ctx, st, "amd-pod", "node-amd", amd64RT) actor := seedActorWithSnapshot(t, ctx, st, suspended, "", durableMemorySnapshot(t, amd64RT)) - stored, worker, err := newSchedulingWorkflow(t, ctx, st).assignWorkerAttempt(ctx, actorRef, actor, gvisorTemplate()) + stored, worker, err := newSchedulingWorkflow(t, ctx, st).assignWorkerAttempt(ctx, actorRef, actor, &ateapipb.ActorTemplate{}) if err != nil { t.Fatalf("assignWorkerAttempt: %v", err) } @@ -2408,7 +2491,7 @@ func TestAssignWorkerAttempt_PrefersRuntimeMatchingWorkerAndFallsBack(t *testing seedRuntimeWorker(t, ctx, st, "amd-pod", "node-amd", newDefaultRT, oldRestorableRT) actor := seedActorWithSnapshot(t, ctx, st, suspended, "", durableMemorySnapshot(t, oldRestorableRT)) - stored, _, err := newSchedulingWorkflow(t, ctx, st).assignWorkerAttempt(ctx, actorRef, actor, gvisorTemplate()) + stored, _, err := newSchedulingWorkflow(t, ctx, st).assignWorkerAttempt(ctx, actorRef, actor, &ateapipb.ActorTemplate{}) if err != nil { t.Fatalf("assignWorkerAttempt: %v", err) } @@ -2423,7 +2506,7 @@ func TestAssignWorkerAttempt_PrefersRuntimeMatchingWorkerAndFallsBack(t *testing seedRuntimeWorker(t, ctx, st, "arm-pod", "node-arm", arm64RT) actor := seedActorWithSnapshot(t, ctx, st, suspended, "", durableMemorySnapshot(t, amd64RT)) - stored, worker, err := newSchedulingWorkflow(t, ctx, st).assignWorkerAttempt(ctx, actorRef, actor, gvisorTemplate()) + stored, worker, err := newSchedulingWorkflow(t, ctx, st).assignWorkerAttempt(ctx, actorRef, actor, &ateapipb.ActorTemplate{}) if err != nil { t.Fatalf("assignWorkerAttempt: %v", err) } @@ -2434,16 +2517,40 @@ func TestAssignWorkerAttempt_PrefersRuntimeMatchingWorkerAndFallsBack(t *testing assertAssignedRuntime(t, stored, arm64RT) }) + t.Run("does not fall back to a worker of another sandbox class", func(t *testing.T) { + // A snapshot's files are written by one class's worker and read back + // by the same, so relaxing the runtime never crosses classes, not + // even for a volumes-only restore that needs no runtime at all. + for _, fidelity := range []ateapipb.SnapshotFidelity{ + ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, + } { + t.Run(fidelity.String(), func(t *testing.T) { + ctx := context.Background() + st := newTestPersistence(t) + seedMicroVMWorker(t, ctx, st, "vm-pod", "node-vm") + snap := durableMemorySnapshot(t, amd64RT) + snap.Fidelity = fidelity + actor := seedActorWithSnapshot(t, ctx, st, suspended, "", snap) + + _, _, err := newSchedulingWorkflow(t, ctx, st).assignWorkerAttempt(ctx, actorRef, actor, &ateapipb.ActorTemplate{}) + if apierror.Code(err) != codes.ResourceExhausted { + t.Fatalf("assignWorkerAttempt with only a microvm worker = %v, want ResourceExhausted", err) + } + }) + } + }) + t.Run("does not relax runtime constraint for local pause snapshot", func(t *testing.T) { ctx := context.Background() st := newTestPersistence(t) seedRuntimeWorker(t, ctx, st, "node1-incompatible", "node-1", arm64RT) // A paused actor: SUSPENDED, pinned to its node by a local snapshot. - local := newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-1", "node-1") + local := newPausedSnapshot(t, 1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-1", "node-1") local.SandboxRuntime = amd64RT actor := seedActorWithSnapshot(t, ctx, st, suspended, "node-1", local) - _, _, err := newSchedulingWorkflow(t, ctx, st).assignWorkerAttempt(ctx, actorRef, actor, gvisorTemplate()) + _, _, err := newSchedulingWorkflow(t, ctx, st).assignWorkerAttempt(ctx, actorRef, actor, &ateapipb.ActorTemplate{}) if got := apierror.Code(err); got != codes.ResourceExhausted { t.Fatalf("apierror.Code(err) = %v, want %v (err: %v)", got, codes.ResourceExhausted, err) } @@ -2473,7 +2580,7 @@ func TestAssignWorkerAttempt_PrefersRuntimeMatchingWorkerAndFallsBack(t *testing actor := seedActorWithSnapshot(t, ctx, st, suspended, "", durableMemorySnapshot(t, oldRestorableRT)) w := &ActorWorkflow{store: st, scheduler: &relaxingTestScheduler{Scheduler: scheduling.New(nil), relaxedWorker: worker}} - stored, _, err := w.assignWorkerAttempt(ctx, actorRef, actor, gvisorTemplate()) + stored, _, err := w.assignWorkerAttempt(ctx, actorRef, actor, &ateapipb.ActorTemplate{}) if err != nil { t.Fatalf("assignWorkerAttempt: %v", err) } @@ -2493,7 +2600,7 @@ func TestAssignWorkerAttempt_PrefersRuntimeMatchingWorkerAndFallsBack(t *testing actor := seedActorWithSnapshot(t, ctx, st, suspended, "", durableMemorySnapshot(t, oldRestorableRT)) w := &ActorWorkflow{store: st, scheduler: &relaxingTestScheduler{Scheduler: scheduling.New(nil), relaxedWorker: stale}} - stored, bound, err := w.assignWorkerAttempt(ctx, actorRef, actor, gvisorTemplate()) + stored, bound, err := w.assignWorkerAttempt(ctx, actorRef, actor, &ateapipb.ActorTemplate{}) if err != nil { t.Fatalf("assignWorkerAttempt: %v", err) } @@ -2537,7 +2644,7 @@ func TestValidateAssignedWorker_RuntimeFallback(t *testing.T) { } w := &ActorWorkflow{store: st, scheduler: scheduling.New(nil)} - worker, err := w.validateAssignedWorker(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, resumingActor, gvisorTemplate()) + worker, err := w.validateAssignedWorker(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, resumingActor, &ateapipb.ActorTemplate{}) if err != nil { t.Fatalf("validateAssignedWorker: %v", err) } @@ -2566,7 +2673,7 @@ func TestRequiredSnapshotRuntime(t *testing.T) { return snap } local := func(gen int32, fidelity ateapipb.SnapshotFidelity, rt *ateapipb.SandboxRuntime) *ateapipb.Snapshot { - snap := newLocalSnapshot(gen, fidelity, tmplUID, "local", "node-1") + snap := newPausedSnapshot(t, gen, fidelity, tmplUID, "local", "node-1") snap.SandboxRuntime = rt return snap } @@ -2651,7 +2758,7 @@ func TestCanRelaxRuntime(t *testing.T) { return newDurableSnapshot(gen, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, memory, "", "snap", "gs://bucket/snap", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) } local := func(gen int32) *ateapipb.Snapshot { - return newLocalSnapshot(gen, memory, "", "local", "node-1") + return newPausedSnapshot(t, gen, memory, "", "local", "node-1") } tests := []struct { @@ -2700,3 +2807,48 @@ func TestWorkerAssignmentFromCopiesSPIFFEID(t *testing.T) { t.Errorf("worker SPIFFE ID = %q, want %q", got.GetWorkerSpiffeId(), id) } } + +func TestToAteletSandboxRuntime(t *testing.T) { + tests := []struct { + name string + in *ateapipb.SandboxRuntime + want *ateletpb.SandboxRuntime + }{ + {name: "nil"}, + { + name: "class only", + in: &ateapipb.SandboxRuntime{SandboxClass: "gvisor"}, + want: &ateletpb.SandboxRuntime{SandboxClass: "gvisor"}, + }, + { + name: "full", + in: &ateapipb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + {Key: "cpu_features", Value: "3f"}, + }, + }, + }, + want: &ateletpb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateletpb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + {Key: "cpu_features", Value: "3f"}, + }, + }, + }, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if diff := cmp.Diff(tc.want, toAteletSandboxRuntime(tc.in), protocmp.Transform()); diff != "" { + t.Errorf("toAteletSandboxRuntime mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/cmd/ateapi/internal/controlapi/workflow_revert.go b/cmd/ateapi/internal/controlapi/workflow_revert.go index ddc7e02552..dcae8cc7c0 100644 --- a/cmd/ateapi/internal/controlapi/workflow_revert.go +++ b/cmd/ateapi/internal/controlapi/workflow_revert.go @@ -33,8 +33,9 @@ import ( // revertableStates are the states a revert is accepted from. A SUSPENDED // actor is also accepted when it is paused, that is, when its latest snapshot -// is node-local: the revert discards that snapshot and returns the actor to -// its durable one. See isRevertable. +// is node-local: the revert discards the node-local copy and returns the actor +// to its latest durable snapshot, which is that same snapshot once atelet's +// upload of it is recorded. See isRevertable. var revertableStates = []ateapipb.ActorState{ ateapipb.ActorState_ACTOR_STATE_RUNNING, ateapipb.ActorState_ACTOR_STATE_CRASHED, @@ -45,7 +46,7 @@ var revertableStates = []ateapipb.ActorState{ func isRevertable(actor *ateapipb.Actor) bool { state := actor.GetStatus().GetState() return slices.Contains(revertableStates, state) || - (state == ateapipb.ActorState_ACTOR_STATE_SUSPENDED && latestSnapshotIsLocalOnly(actor.GetStatus())) + (state == ateapipb.ActorState_ACTOR_STATE_SUSPENDED && latestSnapshotIsLocal(actor.GetStatus())) } // RevertActor executes the workflow to discard an actor's current execution and @@ -68,7 +69,7 @@ func (w *ActorWorkflow) RevertActor(ctx context.Context, actorRef resources.Acto defer func() { attrs := finalAttrs if attrs == nil { - attrs = lifecycleOpAttrs(actor, actorTemplate, "", "") + attrs = lifecycleOpAttrs(actor, "", "") } w.instruments.recordLifecycleOp(ctx, ateattr.OperationRevert, start, err, attrs...) }() @@ -103,7 +104,7 @@ func (w *ActorWorkflow) RevertActor(ctx context.Context, actorRef resources.Acto // FinalizeReverted clears the WorkerAssignment the labels read, so snapshot // them here, as suspend and crash.go do. - finalAttrs = lifecycleOpAttrs(actor, actorTemplate, "", "") + finalAttrs = lifecycleOpAttrs(actor, "", "") var finalized *ateapipb.Actor if finalized, err = w.ensureRevertedFinalized(leaseCtx, actorRef); err != nil { return nil, err @@ -205,41 +206,60 @@ func (w *ActorWorkflow) ensureWorkerDiscarded(ctx context.Context, actorRef reso return w.releaseAssignmentWithoutBacklink(ctx, actor) } -// ensureInProgressSnapshotDiscarded deletes the objects a suspend was partway -// through writing when the actor was reverted. +// ensureInProgressSnapshotDiscarded deletes the objects of every PENDING +// durable snapshot on the actor: the one a suspend was partway through writing +// when the actor was reverted, and any earlier pause's whose upload was never +// recorded. The revert keeps none of them. atelet's upload of them has already +// been canceled when ensureWorkerDiscarded terminated the actor. func (w *ActorWorkflow) ensureInProgressSnapshotDiscarded(ctx context.Context, actor *ateapipb.Actor) (err error) { ctx, done := stepSpan(ctx, "DiscardInProgressSnapshot") defer func() { err = done(err) }() - inProgressSnap := findLatestDurableSnapshot(actor.GetStatus(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) - inProgress := inProgressSnap.GetDurableSnapshot().GetObject().GetSnapshotUri() + var inProgress []string + for _, snap := range actor.GetStatus().GetSnapshots() { + storage := snap.GetDurableSnapshot() + if storage.GetStatus() == ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING && storage.GetObject().GetSnapshotUri() != "" { + inProgress = append(inProgress, storage.GetObject().GetSnapshotUri()) + } + } switch { case w.objectStorePlugin == nil: markSkipped(ctx, "no object store configured") return nil - case inProgress == "": + case len(inProgress) == 0: markSkipped(ctx, "no in-progress snapshot recorded") return nil } - uri, err := resources.ParseSnapshotURI(inProgress) - if err != nil { - return fmt.Errorf("while parsing the in-progress snapshot %q: %w", inProgress, err) - } - // A suspend records the in-progress URI under the actor's own prefix - // before atelet writes the first object, so a URI owned by anything else - // is a corrupted record. + // Check every URI before deleting any, so a corrupted record deletes nothing. owner := actorSnapshotOwner(actor) - if !uri.OwnedBy(owner) { - return fmt.Errorf("the in-progress snapshot %q is not owned by actor %s", inProgress, owner) + prefixes := make([]resources.StoragePrefix, 0, len(inProgress)) + for _, raw := range inProgress { + uri, err := resources.ParseSnapshotURI(raw) + if err != nil { + return fmt.Errorf("while parsing the in-progress snapshot %q: %w", raw, err) + } + // A suspend records the in-progress URI under the actor's own prefix + // before atelet writes the first object, so a URI owned by anything + // else is a corrupted record. + if !uri.OwnedBy(owner) { + return fmt.Errorf("the in-progress snapshot %q is not owned by actor %s", raw, owner) + } + prefixes = append(prefixes, uri.Prefix()) } - // Only the abandoned snapshot goes, not the actor's whole prefix: the + // Only the abandoned snapshots go, not the actor's whole prefix: the // external snapshot the revert returns the actor to lives under it too. - return w.cleanupSnapshot(ctx, uri.Prefix()) + for _, prefix := range prefixes { + if err := w.cleanupSnapshot(ctx, prefix); err != nil { + return err + } + } + return nil } // ensureRevertedFinalized commits SUSPENDED and drops every pointer to the -// execution that was discarded, in a single update. +// execution that was discarded, in a single update, then releases the +// external snapshots the update dropped. // It re-reads the actor first so an out-of-band transition is not overwritten, // and so the version precondition guards what was actually observed. func (w *ActorWorkflow) ensureRevertedFinalized(ctx context.Context, actorRef resources.ActorRef) (_ *ateapipb.Actor, err error) { @@ -275,5 +295,6 @@ func (w *ActorWorkflow) ensureRevertedFinalized(ctx context.Context, actorRef re return nil, err } logActorStateChanged(ctx, storedActor, ateattr.OperationRevert) + w.releaseDroppedSnapshots(ctx, latestActor, storedActor) return storedActor, nil } diff --git a/cmd/ateapi/internal/controlapi/workflow_revert_test.go b/cmd/ateapi/internal/controlapi/workflow_revert_test.go index 32c65a2d70..64de763c38 100644 --- a/cmd/ateapi/internal/controlapi/workflow_revert_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_revert_test.go @@ -18,8 +18,11 @@ import ( "context" "testing" + "github.com/google/go-cmp/cmp" "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/objectstore/objectstoretest" + "go.ate.dev/substrate/internal/objectstoreplugin/objectstoreplugintest" "go.ate.dev/substrate/internal/resources" "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" @@ -76,7 +79,7 @@ func TestEnsureMarkedReverting_AcceptsPaused(t *testing.T) { w := newTestActorWorkflow(t, st, "ns", "tmpl1") actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} - seedWorkflowActor(t, ctx, st, actorRef, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_SUSPENDED, withPausedShape("node-1")) + seedWorkflowActor(t, ctx, st, actorRef, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_SUSPENDED, withPausedShape(t, "node-1")) actor, err := st.GetActor(ctx, actorRef) if err != nil { t.Fatalf("get seeded actor: %v", err) @@ -132,12 +135,17 @@ func TestRevertActor_ReturnsActorToItsSnapshot(t *testing.T) { // snapshot is the completed local one; the other origins also carry // the half-written local snapshot an interrupted pause leaves. const keptURI = "gs://snapshots/team-a/actors/keep/snapshot" + tmpl := &ateapipb.ActorTemplate{SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: "gs://snapshots"}} mustUpdateActorStatus(t, ctx, st, actor, func(s *ateapipb.ActorStatus) { + local1 := newPausedSnapshot(t, 2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-1", "node-1") + local1.DurableSnapshot.Object.SnapshotUri = mustActorSnapshotURI(t, tmpl, actor, "local-1").String() + inProgress := newPausedSnapshot(t, 3, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-in-progress", "") + inProgress.DurableSnapshot.Object.SnapshotUri = mustActorSnapshotURI(t, tmpl, actor, "local-in-progress").String() s.LastAssignedGeneration = 3 s.Snapshots = []*ateapipb.Snapshot{ newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "kept", keptURI, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), - newLocalSnapshot(2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-1", "node-1"), - newLocalSnapshot(3, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-in-progress", ""), + local1, + inProgress, } if tc.paused { s.Snapshots[2].Locality = "node-1" @@ -194,7 +202,7 @@ func TestRevertActor_ReturnsActorToItsSnapshot(t *testing.T) { }) seedWireWorker(t, ctx, st, assignmentOf(running)) wire, atelet := newWireCaptureWorkflow(t, st) - suspended, err := wire.SuspendActor(ctx, actorRef) + suspended, err := wire.SuspendActor(ctx, actorRef, suspendSync) if err != nil { t.Fatalf("SuspendActor after revert: %v", err) } @@ -212,6 +220,59 @@ func TestRevertActor_ReturnsActorToItsSnapshot(t *testing.T) { } } +// TestRevertActor_PausedSnapshotAlreadyUploaded verifies a revert of a paused +// actor whose snapshot atelet uploaded returns it to that upload, its latest +// durable snapshot, and collects the durable snapshot the upload replaced. The +// actor has no assigned node, so there is nothing to terminate, as in +// TestRevertActor_ReturnsActorToItsSnapshot. +func TestRevertActor_PausedSnapshotAlreadyUploaded(t *testing.T) { + memory := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY + completed := ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + ctx := context.Background() + st, cleanup := storetest.SetupTestStore(t) + defer cleanup() + w := newTestActorWorkflow(t, st, "ns", "tmpl1") + objects := objectstoretest.New() + w.objectStorePlugin = objectstoreplugintest.ControlClient(objects) + + actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} + seedWorkflowActor(t, ctx, st, actorRef, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_SUSPENDED) + actor, err := st.GetActor(ctx, actorRef) + if err != nil { + t.Fatalf("get seeded actor: %v", err) + } + // newTestActorWorkflow's template stores snapshots under gs://snapshots. + tmpl := &ateapipb.ActorTemplate{SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: "gs://snapshots"}} + old := mustActorSnapshotURI(t, tmpl, actor, "old") + uploaded := mustActorSnapshotURI(t, tmpl, actor, "pause-snap") + objects.PutSnapshot(t, old, "manifest.json") + objects.PutSnapshot(t, uploaded, "manifest.json") + mustUpdateActorStatus(t, ctx, st, actor, func(s *ateapipb.ActorStatus) { + paused := newPausedSnapshot(t, 2, memory, "", "pause-snap", "node-1") + paused.DurableSnapshot = &ateapipb.SnapshotStorage{Status: completed, Object: &ateapipb.ObjectSnapshot{SnapshotUri: uploaded.String()}} + s.LastAssignedGeneration = 2 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, memory, "", old.Name(), old.String(), completed), + paused, + } + }) + + reverted, err := w.RevertActor(ctx, actorRef) + if err != nil { + t.Fatalf("RevertActor: %v", err) + } + want := []snapshotOutcome{{generation: 2, fidelity: memory, durable: completed}} + if diff := cmp.Diff(want, describeSnapshots(reverted.GetStatus()), cmp.AllowUnexported(snapshotOutcome{})); diff != "" { + t.Errorf("snapshots mismatch (-want +got):\n%s", diff) + } + if len(objects.Snapshot(t, old)) != 0 { + t.Error("the durable snapshot the upload replaced was not released") + } + if len(objects.Snapshot(t, uploaded)) == 0 { + t.Error("the upload the revert returned the actor to was released") + } +} + // TestRevertActor_RejectsSuspended covers both ways an actor can already be // SUSPENDED when a revert arrives: it never ran, or a previous revert already // finished. Both are FailedPrecondition — revert is not a no-op success on its @@ -307,6 +368,7 @@ func TestRevertActor_NoSnapshotToRevertTo(t *testing.T) { // actor to lives under that same prefix. func TestEnsureInProgressSnapshotDiscarded(t *testing.T) { const inFlightSnapshotName = "2026-01-01t00-00-00z-abandoned" + const earlierPauseSnapshotName = "2025-12-31t00-00-00z-earlier-pause" tests := []struct { name string @@ -317,9 +379,13 @@ func TestEnsureInProgressSnapshotDiscarded(t *testing.T) { inFlight string // foreignInFlight records the in-progress snapshot under another // owner's prefix, which no suspend can produce. - foreignInFlight bool - wantInFlightDiscarded bool - wantErr bool + foreignInFlight bool + // earlierPause records an older PENDING snapshot: a pause whose upload + // atelet never reported before the actor resumed. + earlierPause bool + wantInFlightDiscarded bool + wantEarlierPauseDiscarded bool + wantErr bool }{ { name: "discards the snapshot an interrupted suspend was writing", @@ -342,6 +408,25 @@ func TestEnsureInProgressSnapshotDiscarded(t *testing.T) { foreignInFlight: true, wantErr: true, }, + { + name: "discards an earlier pause's unrecorded upload along with the in-flight snapshot", + inFlight: inFlightSnapshotName, + earlierPause: true, + wantInFlightDiscarded: true, + wantEarlierPauseDiscarded: true, + }, + { + name: "discards an earlier pause's unrecorded upload with no suspend in flight", + earlierPause: true, + wantEarlierPauseDiscarded: true, + }, + { + name: "discards nothing when any in-progress snapshot is not the actor's", + inFlight: inFlightSnapshotName, + foreignInFlight: true, + earlierPause: true, + wantErr: true, + }, } for _, tt := range tests { @@ -373,14 +458,22 @@ func TestEnsureInProgressSnapshotDiscarded(t *testing.T) { if tt.inFlight != "" { objects.PutSnapshot(t, inFlight, "manifest.json") } + earlierPause := mustActorSnapshotURI(t, template, actor, earlierPauseSnapshotName) + if tt.earlierPause { + objects.PutSnapshot(t, earlierPause, "manifest.json") + } actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { s.LastAssignedGeneration = 1 s.Snapshots = []*ateapipb.Snapshot{ newDurableSnapshot(1, currentType, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", current.Name(), current.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), } + if tt.earlierPause { + s.LastAssignedGeneration++ + s.Snapshots = append(s.Snapshots, newDurableSnapshot(s.LastAssignedGeneration, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", earlierPause.Name(), earlierPause.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING)) + } if tt.inFlight != "" { - s.LastAssignedGeneration = 2 - s.Snapshots = append(s.Snapshots, newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", inFlight.Name(), inFlight.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING)) + s.LastAssignedGeneration++ + s.Snapshots = append(s.Snapshots, newDurableSnapshot(s.LastAssignedGeneration, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", inFlight.Name(), inFlight.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING)) } }) @@ -392,6 +485,9 @@ func TestEnsureInProgressSnapshotDiscarded(t *testing.T) { if len(objects.Snapshot(t, inFlight)) == 0 { t.Errorf("snapshot %v was discarded, but it belongs to another owner", inFlight) } + if tt.earlierPause && len(objects.Snapshot(t, earlierPause)) == 0 { + t.Errorf("snapshot %v was discarded, but a corrupted record must discard nothing", earlierPause) + } return } if err != nil { @@ -409,6 +505,12 @@ func TestEnsureInProgressSnapshotDiscarded(t *testing.T) { t.Errorf("in-flight snapshot discarded = %v, want %v", discarded, tt.wantInFlightDiscarded) } } + if tt.earlierPause { + discarded := len(objects.Snapshot(t, earlierPause)) == 0 + if discarded != tt.wantEarlierPauseDiscarded { + t.Errorf("earlier pause snapshot discarded = %v, want %v", discarded, tt.wantEarlierPauseDiscarded) + } + } }) } } diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend.go b/cmd/ateapi/internal/controlapi/workflow_suspend.go index 14adde3c41..0b4a65cf98 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend.go @@ -39,14 +39,24 @@ type suspendMode int const ( // suspendSync makes the snapshot durable before the workflow returns: - // the actor ends SUSPENDED holding a durable snapshot (SuspendActor). + // the actor ends SUSPENDED holding a durable snapshot. suspendSync suspendMode = iota - // suspendAsync leaves the snapshot on the worker's node, to be uploaded - // later: the actor ends SUSPENDED holding a node-local snapshot and - // pinned to that node (PauseActor). Nothing uploads it yet. + // suspendAsync leaves the snapshot on the worker's node: the actor ends + // SUSPENDED holding a node-local snapshot whose durable copy is still + // pending, pinned to that node until atelet uploads it in the background + // and the detach report unpins it. suspendAsync ) +// suspendModeFromOptions maps a SuspendActor request's options to the +// workflow mode. An unset mode runs an async suspend. +func suspendModeFromOptions(opts *ateapipb.SuspendOptions) suspendMode { + if opts.GetMode() == ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS { + return suspendSync + } + return suspendAsync +} + // uploadMode is when atelet makes the checkpoint durable. func (m suspendMode) uploadMode() ateletpb.UploadMode { if m == suspendAsync { @@ -55,31 +65,21 @@ func (m suspendMode) uploadMode() ateletpb.UploadMode { return ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE } -// SuspendActor executes the workflow to suspend a running or paused actor: -// a running actor is checkpointed on its worker, a paused actor's node-local -// snapshot is uploaded. Idempotent: a re-entered workflow fast-forwards past -// the steps a previous attempt completed, deriving progress from the -// persisted actor alone. -func (w *ActorWorkflow) SuspendActor(ctx context.Context, actorRef resources.ActorRef) (*ateapipb.Actor, error) { +// SuspendActor executes the workflow to suspend an actor. A running actor is +// checkpointed on its worker: a sync suspend uploads the snapshot before +// returning, an async suspend keeps it on that node. A SUSPENDED actor is +// returned as it is, whatever the mode. Idempotent: a re-entered workflow +// fast-forwards past the steps a previous attempt completed, deriving +// progress from the persisted actor alone. +func (w *ActorWorkflow) SuspendActor(ctx context.Context, actorRef resources.ActorRef, mode suspendMode) (_ *ateapipb.Actor, err error) { ctx = preview.ContextWithOptIn(ctx) - return w.suspendActor(ctx, actorRef, suspendSync) -} - -// PauseActor executes the workflow to pause a running actor: it is -// checkpointed on its worker and the snapshot is kept on that node. -// Idempotent: a re-entered workflow fast-forwards past the steps a previous -// attempt completed, deriving progress from the persisted actor alone. -func (w *ActorWorkflow) PauseActor(ctx context.Context, actorRef resources.ActorRef) (*ateapipb.Actor, error) { - ctx = preview.ContextWithOptIn(ctx) - return w.suspendActor(ctx, actorRef, suspendAsync) -} - -// suspendActor runs the suspend workflow behind SuspendActor and PauseActor. -func (w *ActorWorkflow) suspendActor(ctx context.Context, actorRef resources.ActorRef, mode suspendMode) (_ *ateapipb.Actor, err error) { start := time.Now() var actor *ateapipb.Actor var actorTemplate *ateapipb.ActorTemplate var wireFidelity string + // ateletState is the state atelet returned from Checkpoint, which + // finalize applies. + var ateletState *ateletpb.ActorSnapshotState // Set just before finalize; nil until then, so earlier exits label // themselves from the record they hold. var finalAttrs []attribute.KeyValue @@ -87,7 +87,7 @@ func (w *ActorWorkflow) suspendActor(ctx context.Context, actorRef resources.Act defer func() { attrs := finalAttrs if attrs == nil { - attrs = lifecycleOpAttrs(actor, actorTemplate, "", wireFidelity) + attrs = lifecycleOpAttrs(actor, "", wireFidelity) } w.instruments.recordLifecycleOp(ctx, ateattr.OperationSuspend, start, err, attrs...) }() @@ -102,35 +102,19 @@ func (w *ActorWorkflow) suspendActor(ctx context.Context, actorRef resources.Act if err != nil { return nil, err } - // FinalizeSuspended commits SUSPENDED and the cleared worker assignment in - // a single update, so an actor already holding the snapshot this mode - // leaves behind has nothing left to do. These successes report no pool, - // and cannot: the previous attempt released the worker, so the record - // names none (#957). + // A SUSPENDED actor has nothing left to do, whatever the mode and wherever + // its snapshot is: atelet uploads a node-local snapshot in the background. + // These successes report no pool, and cannot: the previous attempt + // released the worker, so the record names none (#957). if actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_SUSPENDED { - if mode == suspendAsync && latestSnapshotIsLocalOnly(actor.GetStatus()) { - return actor, nil // fully paused - } - if mode == suspendSync && !latestSnapshotIsLocalOnly(actor.GetStatus()) { - return actor, nil // fully suspended - } - // A sync suspend of a paused actor goes on to upload its node-local - // snapshot. An async suspend of a suspended actor is rejected by MarkSuspending. + return actor, nil } - // Decided before marking: once SUSPENDING is committed, the loaded status - // alone can no longer tell the two origins apart. - // TODO(paused-suspend): remove the paused -> suspended transition before GA. - fromPaused := mode == suspendSync && isPausedOriginSuspend(actor) var marked *ateapipb.Actor if marked, err = w.ensureMarkedSuspending(leaseCtx, actorRef, actor, actorTemplate, mode); err != nil { return nil, err } actor = marked - if fromPaused { - wireFidelity, err = w.ensurePausedSnapshotUploaded(leaseCtx, actorRef, actor, actorTemplate) - } else { - wireFidelity, err = w.ensureAteletSuspended(leaseCtx, actorRef, actor, actorTemplate, mode) - } + wireFidelity, ateletState, err = w.ensureAteletSuspended(leaseCtx, actorRef, actor, actorTemplate, mode) if err != nil { return nil, err } @@ -139,10 +123,11 @@ func (w *ActorWorkflow) suspendActor(ctx context.Context, actorRef resources.Act return nil, err } // FinalizeSuspended clears the WorkerAssignment the labels read, so snapshot - // them here, as crash.go does for the crash counter. - finalAttrs = lifecycleOpAttrs(actor, actorTemplate, "", wireFidelity) + // them here, as crash.go does for the crash counter. The fidelity label is + // the one requested. + finalAttrs = lifecycleOpAttrs(actor, "", wireFidelity) var finalized *ateapipb.Actor - if finalized, err = w.ensureSuspendedFinalized(leaseCtx, actorRef, mode); err != nil { + if finalized, err = w.ensureSuspendedFinalized(leaseCtx, actorRef, mode, ateletState); err != nil { return nil, err } actor = finalized @@ -165,54 +150,25 @@ func (w *ActorWorkflow) loadActorForSuspend(ctx context.Context, actorRef resour return actor, actorTemplate, nil } -// ensureMarkedSuspending transitions a RUNNING or paused actor to -// SUSPENDING, minting the in-progress snapshot and recording the actor -// version the snapshot will capture. A sync suspend mints a pending durable -// snapshot; an async one mints a node-local snapshot with no locality yet. -// Skips when a previous attempt of the same mode already marked the actor; -// the persisted snapshot and source version then stay authoritative for the -// rest of the workflow. A mark left by the other mode is rejected: that -// workflow owns the actor until it finishes. +// ensureMarkedSuspending transitions a RUNNING actor to SUSPENDING, minting +// the in-progress snapshot and recording the actor version the snapshot will +// capture. Either mode mints a snapshot with a pending durable copy. Skips +// when a previous attempt already marked the actor, in either mode; the +// persisted snapshot and source version then stay authoritative for the rest +// of the workflow. func (w *ActorWorkflow) ensureMarkedSuspending(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate, mode suspendMode) (_ *ateapipb.Actor, err error) { ctx, done := stepSpan(ctx, "MarkSuspending") defer func() { err = done(err) }() - if actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_SUSPENDING { - // An async mark's latest snapshot is node-local with no locality yet, - // a sync mark's has a pending durable copy. - if latestSnapshotIsPendingLocal(actor.GetStatus()) != (mode == suspendAsync) { - inProgress := "Pause" - if mode == suspendAsync { - inProgress = "Suspend" - } - return nil, apierror.FailedPrecondition("MarkSuspending prerequisite not met for Actor: %s (a previous %s is in progress)", actorRef, inProgress) - } - markSkipped(ctx, "actor already SUSPENDING") - return actor, nil - } state := actor.GetStatus().GetState() - // Only a sync suspend takes a paused actor. - paused := mode == suspendSync && isPausedOriginSuspend(actor) // TODO(paused-suspend) - if state != ateapipb.ActorState_ACTOR_STATE_RUNNING && !paused { - if mode == suspendAsync { - return nil, apierror.FailedPrecondition("MarkSuspending prerequisite not met for Actor: %s (got: %v, want %s)", actorRef, state, ateapipb.ActorState_ACTOR_STATE_RUNNING) - } - return nil, apierror.FailedPrecondition("MarkSuspending prerequisite not met for Actor: %s (got: %v, want %s or %s with a node-local snapshot)", actorRef, state, ateapipb.ActorState_ACTOR_STATE_RUNNING, ateapipb.ActorState_ACTOR_STATE_SUSPENDED) + if state != ateapipb.ActorState_ACTOR_STATE_RUNNING && state != ateapipb.ActorState_ACTOR_STATE_SUSPENDING { + return nil, apierror.FailedPrecondition("MarkSuspending prerequisite not met for Actor: %s (got: %v, want %s)", actorRef, state, ateapipb.ActorState_ACTOR_STATE_RUNNING) } - // Golden actors are only suspended, never paused. - // TODO: Remove this check: the suspend workflow should not special-case golden actors. - if mode == suspendAsync && actorRef.Atespace == resources.GoldenActorAtespace { - return nil, apierror.FailedPrecondition("actors in atespace %q are golden actors, which cannot be paused", actorRef.Atespace) - } - // A paused actor's durable snapshot is an upload of its local checkpoint, - // so it keeps that checkpoint's name; a running actor's is a new one. - var snapshotName string - if paused { - // paused implies a snapshot with a locality, so this is never nil. - snapshotName = findLatestLocalSnapshot(actor.GetStatus()).GetUuid() - } else { - snapshotName = resources.NewSnapshotName() + if state == ateapipb.ActorState_ACTOR_STATE_SUSPENDING { + markSkipped(ctx, "actor already SUSPENDING") + return actor, nil } + snapshotName := resources.NewSnapshotName() // Fail here rather than at checkpoint time if the template's location // cannot produce a usable URI: nothing has been written yet. @@ -222,40 +178,21 @@ func (w *ActorWorkflow) ensureMarkedSuspending(ctx context.Context, actorRef res } storedActor, err := w.store.UpdateActor(ctx, actorRef, store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error { toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_SUSPENDING - if paused { - // Promote the current generation's local snapshot to a pending durable snapshot. - snap := findLatestLocalSnapshot(toUpdate.Status) - if snap == nil { - return fmt.Errorf("actor %s is paused but has no completed local snapshot", actorRef) - } - snap.ActorTemplateUid = actorTemplate.GetMetadata().GetUid() - snap.Fidelity = preferredFidelity(actorRef.Atespace, actorTemplate) - snap.DurableSnapshot = &ateapipb.SnapshotStorage{ - Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING, - Object: &ateapipb.ObjectSnapshot{SnapshotUri: uri.String()}, - } - return nil - } // Increment last_assigned_generation for the new Suspend request and create - // a pending snapshot for that generation: durable for a sync suspend, - // node-local with no locality until finalize for an async one. + // a snapshot for that generation with a pending durable copy. Both modes + // record where the snapshot is uploaded, so the mode can change on a retry. gen := toUpdate.Status.LastAssignedGeneration + 1 toUpdate.Status.LastAssignedGeneration = gen - fidelity := preferredFidelity(actorRef.Atespace, actorTemplate) - templateUID := actorTemplate.GetMetadata().GetUid() - snap := newLocalSnapshot(gen, fidelity, templateUID, snapshotName, "") - if mode == suspendSync { - snap = newDurableSnapshot( - gen, - ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, - fidelity, - templateUID, - uri.Name(), - uri.String(), - ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING, - ) - } + snap := newDurableSnapshot( + gen, + ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, + actorTemplate.GetSnapshotConfig().GetPreferredFidelity(), + actorTemplate.GetMetadata().GetUid(), + uri.Name(), + uri.String(), + ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING, + ) snap.SandboxRuntime = proto.CloneOf(toUpdate.GetStatus().GetWorkerAssignment().GetSandboxRuntime()) toUpdate.Status.Snapshots = append(toUpdate.Status.Snapshots, snap) return nil @@ -270,81 +207,42 @@ func (w *ActorWorkflow) ensureMarkedSuspending(ctx context.Context, actorRef res return storedActor, nil } -// preferredFidelity returns the fidelity a suspend snapshot is taken with. -// Golden actors always commit MEMORY regardless of the template's -// preferredFidelity: new actors borrow the golden snapshot and resume it -// with memory, so it must carry the guest memory and filesystem. -func preferredFidelity(atespace string, tmpl *ateapipb.ActorTemplate) ateapipb.SnapshotFidelity { - if atespace == resources.GoldenActorAtespace { - return ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY - } - return tmpl.GetSnapshotConfig().GetPreferredFidelity() -} - -// TODO(paused-suspend): remove with the paused -> suspended transition. -// isPausedOriginSuspend reports whether the suspend must upload a paused -// actor's node-local snapshot instead of checkpointing a running workload. -// A local snapshot alone does not mean paused-origin: resume restores from it -// without deleting its files, so a RUNNING actor resumed from pause still -// carries one until its next checkpoint. The nil worker assignment -// disambiguates — running-origin suspends keep their assignment until -// finalize, paused actors never have one. -func isPausedOriginSuspend(actor *ateapipb.Actor) bool { - if actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_SUSPENDED && latestSnapshotIsLocalOnly(actor.GetStatus()) { - return true - } - if actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_SUSPENDING && - actor.GetStatus().GetWorkerAssignment() == nil { - return findLatestLocalSnapshot(actor.GetStatus()) != nil - } - return false -} - // ensureAteletSuspended checkpoints the workload to the actor's persisted // in-progress snapshot location. A sync suspend has atelet upload it before -// returning; an async suspend has atelet keep it on the node. +// returning; an async suspend has atelet keep it on the node. It returns the +// snapshot state atelet reports once the checkpoint is taken, which finalize +// applies. // This is the atelet reentrancy seam (#372): // the request is keyed by the actor UID, the worker pod UID, and the // once-minted snapshot location, so a re-entered workflow re-sends the same // semantic request; once atelet's Checkpoint is idempotent on those keys this // step becomes fully reentrant with no changes here. -func (w *ActorWorkflow) ensureAteletSuspended(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate, mode suspendMode) (wireFidelity string, err error) { +func (w *ActorWorkflow) ensureAteletSuspended(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate, mode suspendMode) (wireFidelity string, state *ateletpb.ActorSnapshotState, err error) { ctx, done := stepSpan(ctx, "CallAteletSuspend") defer func() { err = done(err) }() assignment := actor.GetStatus().GetWorkerAssignment() if assignment == nil { // Missing active worker pod reference in SUSPENDING state indicates corrupted store state. - return "", crashActorAndFail(ctx, w.store, actorRef, ateattr.OperationSuspend, crashMessageWorkerAssignmentMissing) + return "", nil, crashActorAndFail(ctx, w.store, actorRef, ateattr.OperationSuspend, crashMessageWorkerAssignmentMissing) } ateletConn, err := w.dialer.DialForAteletOnNode(assignment.GetNodeName()) if err != nil { - return "", fmt.Errorf("while getting atelet conn for node %q: %w", assignment.GetNodeName(), err) + return "", nil, fmt.Errorf("while getting atelet conn for node %q: %w", assignment.GetNodeName(), err) } client := ateletpb.NewAteletClient(ateletConn) workloadSpec, err := workloadSpecFromActorTemplate(actorTemplate, actor, nil) if err != nil { - return "", err + return "", nil, err } // Checkpoint does not carry the sandbox config: atelet uses the version the // actor is currently running (recorded on-node at Run/Restore) and pins it // into the snapshot manifest. inProgressSnap := snapshotAtLatestGeneration(actor.GetStatus()) - fidelity := fidelityToAtelet(preferredFidelity(actor.GetMetadata().GetAtespace(), actorTemplate)) - // A sync suspend records its URI on the in-progress durable snapshot; - // an async one has no durable snapshot record yet, so derive the URI - // from the persisted snapshot UUID. - snapshotURI := inProgressSnap.GetDurableSnapshot().GetObject().GetSnapshotUri() - if mode == suspendAsync { - uri, err := newInProgressSnapshotURI(actorTemplate, actor, inProgressSnap.GetUuid()) - if err != nil { - return "", err - } - snapshotURI = uri.String() - } + fidelity := fidelityToAtelet(actorTemplate.GetSnapshotConfig().GetPreferredFidelity()) req := &ateletpb.CheckpointRequest{ WorkerPodUid: assignment.GetWorkerPodUid(), Atespace: actor.GetMetadata().GetAtespace(), @@ -357,7 +255,7 @@ func (w *ActorWorkflow) ensureAteletSuspended(ctx context.Context, actorRef reso Generation: inProgressSnap.GetGeneration(), Fidelity: fidelity, Object: &ateletpb.ObjectStorage{ - SnapshotUri: snapshotURI, + SnapshotUri: inProgressSnap.GetDurableSnapshot().GetObject().GetSnapshotUri(), }, }, UploadConfig: &ateletpb.UploadConfig{ @@ -367,66 +265,11 @@ func (w *ActorWorkflow) ensureAteletSuspended(ctx context.Context, actorRef reso } wireFidelity = ateattr.SnapshotFidelityValue(fidelity) - if _, err = client.Checkpoint(ctx, req); err != nil { - return wireFidelity, handleAteletError(ctx, w.store, actorRef, ateattr.OperationSuspend, "Checkpoint", false, err) - } - return wireFidelity, nil -} - -// TODO(paused-suspend): remove with the paused -> suspended transition. -// ensurePausedSnapshotUploaded suspends a paused actor by telling the atelet -// on the node holding the local pause snapshot to upload it to the actor's -// persisted in-progress snapshot location; no workload runs, so there is no -// worker to checkpoint. Retries re-send the same semantic request: the -// destination is minted once and the upload overwrites deterministic object -// names, with the remote manifest as the commit marker. -func (w *ActorWorkflow) ensurePausedSnapshotUploaded(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate) (wireFidelity string, err error) { - ctx, done := stepSpan(ctx, "PromoteSnapshot") - defer func() { err = done(err) }() - - localSnap := findLatestLocalSnapshot(actor.GetStatus()) - if localSnap == nil { - return "", crashActorAndFail(ctx, w.store, actorRef, ateattr.OperationSuspend, crashMessageLocalSnapshotMissing) - } - nodeName := actor.GetStatus().GetAssignedNode() - if nodeName == "" { - // Without the node the snapshot can never be found (mirrors an async - // FinalizeSuspended, which crashes rather than record an unknown node). - return "", crashActorAndFail(ctx, w.store, actorRef, ateattr.OperationSuspend, crashMessageLocalSnapshotNodeUnknown) - } - - ateletConn, err := w.dialer.DialForAteletOnNode(nodeName) + resp, err := client.Checkpoint(ctx, req) if err != nil { - // No atelet on the node is indistinguishable from an atelet restart or - // informer lag, and the snapshot bytes may still be on its disk: stay - // retryable rather than crash. - return "", fmt.Errorf("while getting atelet conn for node %q: %w", nodeName, err) - } - client := ateletpb.NewAteletClient(ateletConn) - - // The template's preferred fidelity, as a running-origin suspend commits. - fidelity := fidelityToAtelet(preferredFidelity(actor.GetMetadata().GetAtespace(), actorTemplate)) - req := &ateletpb.PromoteSnapshotRequest{ - Atespace: actor.GetMetadata().GetAtespace(), - ActorName: actor.GetMetadata().GetName(), - ActorUid: actor.GetMetadata().GetUid(), - ActorTemplateAtespace: actor.GetActorTemplate().GetAtespace(), - ActorTemplateName: actor.GetActorTemplate().GetName(), - Snapshot: &ateletpb.Snapshot{ - SnapshotUuid: localSnap.GetUuid(), - Generation: localSnap.GetGeneration(), - Fidelity: fidelity, - Object: &ateletpb.ObjectStorage{ - SnapshotUri: snapshotAtLatestGeneration(actor.GetStatus()).GetDurableSnapshot().GetObject().GetSnapshotUri(), - }, - }, + return wireFidelity, nil, handleAteletError(ctx, w.store, actorRef, ateattr.OperationSuspend, "Checkpoint", false, err) } - wireFidelity = ateattr.SnapshotFidelityValue(fidelity) - - if _, err = client.PromoteSnapshot(ctx, req); err != nil { - return wireFidelity, handleAteletError(ctx, w.store, actorRef, ateattr.OperationSuspend, "PromoteSnapshot", false, err) - } - return wireFidelity, nil + return wireFidelity, resp.GetState(), nil } // newInProgressSnapshotURI is where the snapshot an actor is currently taking is @@ -459,16 +302,19 @@ func (w *ActorWorkflow) ensureVolumesDetached(ctx context.Context, actor *ateapi // ensureSuspendedFinalized releases the actor's worker (only when it is still // owned by this actor), records the in-progress snapshot as the actor's -// durable snapshot (sync) or node-local snapshot (async), and commits -// SUSPENDED with the assignment cleared in a single update, then releases the -// durable snapshot that update replaced. +// durable snapshot (sync) or node-local snapshot with its PENDING durable copy +// (async), and commits SUSPENDED with the assignment cleared in a single +// update, then releases the durable snapshot that update replaced. // When the latest snapshot cannot be recorded, it commits CRASHED instead and // fails the step. // It re-reads the actor first so an out-of-band transition (e.g. the syncer // crashing the actor after its worker died) is not overwritten: an actor that // is no longer SUSPENDING is left as it is and the step fails with // FailedPrecondition. -func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef resources.ActorRef, mode suspendMode) (_ *ateapipb.Actor, err error) { +// ateletState is the state atelet returned from Checkpoint, applied in the +// same update. A state that conflicts with the record fails the step before it +// frees the worker. +func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef resources.ActorRef, mode suspendMode, ateletState *ateletpb.ActorSnapshotState) (_ *ateapipb.Actor, err error) { ctx, done := stepSpan(ctx, "FinalizeSuspended") defer func() { err = done(err) }() @@ -497,12 +343,7 @@ func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef r if state := latestActor.GetStatus().GetState(); state != ateapipb.ActorState_ACTOR_STATE_SUSPENDING { return nil, apierror.FailedPrecondition("FinalizeSuspended prerequisite not met for Actor: %s (got: %v, want %s)", actorRef, state, ateapipb.ActorState_ACTOR_STATE_SUSPENDING) } - switch mode { - case suspendSync: - if snapshotAtLatestGeneration(latestActor.GetStatus()).GetDurableSnapshot() == nil { - return nil, crashActorAndFail(ctx, w.store, actorRef, ateattr.OperationSuspend, crashMessageDurableSnapshotMissing) - } - case suspendAsync: + if mode == suspendAsync { // A local snapshot on an unknown node can never be found by the scheduler. if latestActor.GetStatus().GetAssignedNode() == "" { return nil, crashActorAndFail(ctx, w.store, actorRef, ateattr.OperationSuspend, crashMessageLocalSnapshotNodeUnknown) @@ -511,6 +352,20 @@ func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef r return nil, crashActorAndFail(ctx, w.store, actorRef, ateattr.OperationSuspend, crashMessageLocalSnapshotMissing) } } + // Both modes mark the durable copy PENDING up front. Without it a sync + // suspend has nothing to mark COMPLETED, and atelet's background upload of + // a pause's snapshot could never be recorded. + if snapshotAtLatestGeneration(latestActor.GetStatus()).GetDurableSnapshot() == nil { + return nil, crashActorAndFail(ctx, w.store, actorRef, ateattr.OperationSuspend, crashMessageDurableSnapshotMissing) + } + + // Apply atelet's state before freeing the worker, so a state that + // conflicts with the record fails the step while a retry can still call + // atelet again. + applied := proto.CloneOf(latestActor.GetStatus()) + if err := applyCheckpointState(ctx, applied, ateletState); err != nil { + return nil, err + } // 1. Free the worker (if it hasn't been freed yet) if latestActor.GetStatus().GetWorkerAssignment() != nil { @@ -533,6 +388,8 @@ func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef r t = time.Now() storedActor, err := w.store.UpdateActor(ctx, actorRef, store.PreconditionFrom(latestActor), func(toUpdate *ateapipb.Actor) error { toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_SUSPENDED + // The precondition makes toUpdate the version applied was built from. + toUpdate.Status.Snapshots = applied.GetSnapshots() snap := snapshotAtLatestGeneration(toUpdate.Status) switch mode { case suspendSync: @@ -560,31 +417,46 @@ func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef r } logActorStateChanged(ctx, storedActor, ateattr.OperationSuspend) - // 3. Release the external snapshot this suspend replaced. Best-effort: the + // 3. Release the external snapshots this suspend replaced. Best-effort: the // new snapshot is committed and the worker is gone, so failing the - // workflow here would only strand the actor. A failure leaves the old - // snapshot's objects in storage until the actor is deleted, which removes - // its whole prefix. + // workflow here would only strand the actor. t = time.Now() - replaced := findLatestDurableSnapshot(latestActor.GetStatus(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED).GetDurableSnapshot().GetObject().GetSnapshotUri() - current := findLatestDurableSnapshot(storedActor.GetStatus(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED).GetDurableSnapshot().GetObject().GetSnapshotUri() - releaseErr := w.releaseReplacedSnapshot(ctx, latestActor, replaced, current) + w.releaseDroppedSnapshots(ctx, latestActor, storedActor) dReleaseSnapshot = time.Since(t) - if releaseErr != nil { - slog.WarnContext(ctx, "Failed to release the external snapshot a suspend replaced; its objects are left in storage until the actor is deleted", - slog.Any("actor", actorRef), - slog.String("snapshot_uri", replaced), - slog.String("err", releaseErr.Error())) - } return storedActor, nil } -// releaseReplacedSnapshot releases previous, the URI of the external snapshot -// the actor held before this suspend, unless it is still current. actor is -// the record as it was before the suspend committed. +// releaseDroppedSnapshots releases each external snapshot that before records +// as COMPLETED and after doesn't. A failure is only logged, and leaves the +// snapshot's objects in storage until the actor is deleted, which removes its +// whole prefix. An upload atelet reports can make a newer snapshot COMPLETED +// outside any workflow, so the next one can drop more than one. +func (w *ActorWorkflow) releaseDroppedSnapshots(ctx context.Context, before, after *ateapipb.Actor) { + completed := ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + kept := map[string]bool{} + for _, snap := range after.GetStatus().GetSnapshots() { + kept[snap.GetDurableSnapshot().GetObject().GetSnapshotUri()] = true + } + for _, snap := range before.GetStatus().GetSnapshots() { + dropped := snap.GetDurableSnapshot().GetObject().GetSnapshotUri() + if snap.GetDurableSnapshot().GetStatus() != completed || kept[dropped] { + continue + } + if err := w.releaseReplacedSnapshot(ctx, before, dropped); err != nil { + slog.WarnContext(ctx, "Failed to release an external snapshot the actor no longer records; its objects are left in storage until the actor is deleted", + slog.Any("actor", resources.ActorRefFromActor(before)), + slog.String("snapshot_uri", dropped), + slog.String("err", err.Error())) + } + } +} + +// releaseReplacedSnapshot releases previous, the URI of an external snapshot +// the actor held before a suspend or revert dropped it. actor is the record as +// it was before that workflow committed. // An actor that borrowed its current snapshot from a tag releases nothing — // the snapshot lives under the tag's prefix, and the tag outlives the actor. -func (w *ActorWorkflow) releaseReplacedSnapshot(ctx context.Context, actor *ateapipb.Actor, previous, current string) (err error) { +func (w *ActorWorkflow) releaseReplacedSnapshot(ctx context.Context, actor *ateapipb.Actor, previous string) (err error) { ctx, done := stepSpan(ctx, "ReleaseReplacedSnapshot") defer func() { err = done(err) }() @@ -595,9 +467,6 @@ func (w *ActorWorkflow) releaseReplacedSnapshot(ctx context.Context, actor *atea case previous == "": markSkipped(ctx, "the actor held no external snapshot") return nil - case previous == current: - markSkipped(ctx, "the actor's external snapshot is unchanged") - return nil } uri, err := resources.ParseSnapshotURI(previous) if err != nil { diff --git a/cmd/ateapi/internal/controlapi/workflow_pause_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_async_test.go similarity index 55% rename from cmd/ateapi/internal/controlapi/workflow_pause_test.go rename to cmd/ateapi/internal/controlapi/workflow_suspend_async_test.go index fc6b9e3182..f6f57a3a2a 100644 --- a/cmd/ateapi/internal/controlapi/workflow_pause_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_async_test.go @@ -16,8 +16,6 @@ package controlapi import ( "context" - "fmt" - "strings" "testing" "github.com/google/go-cmp/cmp" @@ -32,7 +30,7 @@ import ( "google.golang.org/protobuf/testing/protocmp" ) -// TestPauseActor_Finalized_WorkerGone reproduces the scenario where the +// TestSuspendActorAsync_Finalized_WorkerGone reproduces the scenario where the // worker pod disappears from the DB during pause finalization, so the node it // ran on is unknown. // @@ -40,7 +38,7 @@ import ( // instead of left paused, since a local snapshot with an unknown node can // never be safely resumed. The pause fails with FailedPrecondition rather than // reporting the pause as done. -func TestPauseActor_Finalized_WorkerGone(t *testing.T) { +func TestSuspendActorAsync_Finalized_WorkerGone(t *testing.T) { ctx := context.Background() st := newTestPersistence(t) seedSuspendTemplate(t, ctx, st, nil) @@ -56,14 +54,14 @@ func TestPauseActor_Finalized_WorkerGone(t *testing.T) { WorkerAssignment: wireTestAssignment(), LastAssignedGeneration: 1, Snapshots: []*ateapipb.Snapshot{ - newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-snap-1", ""), + newInProgressSnapshot(t, 1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "local-snap-1"), }, }, }) // Intentionally NOT creating the worker in store, simulates worker already gone. - if _, err := w.PauseActor(ctx, actorRef); apierror.Code(err) != codes.FailedPrecondition { - t.Fatalf("PauseActor error = %v, want FailedPrecondition for the actor it crashed", err) + if _, err := w.SuspendActor(ctx, actorRef, suspendAsync); apierror.Code(err) != codes.FailedPrecondition { + t.Fatalf("Async SuspendActor error = %v, want FailedPrecondition for the actor it crashed", err) } if checkpoints, _ := atelet.calls(); checkpoints != 1 { t.Errorf("Checkpoint calls = %d, want 1 before finalize", checkpoints) @@ -99,11 +97,11 @@ func TestPauseActor_Finalized_WorkerGone(t *testing.T) { } } -// TestPauseActor_Finalized_AlreadyCrashed verifies that if the actor was +// TestSuspendActorAsync_Finalized_AlreadyCrashed verifies that if the actor was // crashed out-of-band while it checkpointed, and has no AssignedNode, the // pause's finalize preserves the existing Crash status and snapshots, emits no // duplicate crash log record, and still fails. -func TestPauseActor_Finalized_AlreadyCrashed(t *testing.T) { +func TestSuspendActorAsync_Finalized_AlreadyCrashed(t *testing.T) { ctx := context.Background() st := newTestPersistence(t) seedSuspendTemplate(t, ctx, st, nil) @@ -119,7 +117,7 @@ func TestPauseActor_Finalized_AlreadyCrashed(t *testing.T) { WorkerAssignment: wireTestAssignment(), LastAssignedGeneration: 1, Snapshots: []*ateapipb.Snapshot{ - newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-snap-1", ""), + newInProgressSnapshot(t, 1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "local-snap-1"), }, }, }) @@ -130,8 +128,8 @@ func TestPauseActor_Finalized_AlreadyCrashed(t *testing.T) { s.WorkerAssignment = nil }) - if _, err := w.PauseActor(ctx, actorRef); apierror.Code(err) != codes.FailedPrecondition { - t.Fatalf("PauseActor error = %v, want FailedPrecondition for a CRASHED actor", err) + if _, err := w.SuspendActor(ctx, actorRef, suspendAsync); apierror.Code(err) != codes.FailedPrecondition { + t.Fatalf("Async SuspendActor error = %v, want FailedPrecondition for a CRASHED actor", err) } got := mustReadActor(t, ctx, st, actorRef) if got.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_CRASHED { @@ -151,11 +149,11 @@ func TestPauseActor_Finalized_AlreadyCrashed(t *testing.T) { } } -// TestPauseActor_Finalized_KeepsCrashFromWorkerDelete verifies that a crash the +// TestSuspendActorAsync_Finalized_KeepsCrashFromWorkerDelete verifies that a crash the // DeleteWorker workflow commits while the pause's finalize is freeing the // worker is kept: the finalize update is pinned to the actor version it // checked, so it fails Aborted instead of committing SUSPENDED over the crash. -func TestPauseActor_Finalized_KeepsCrashFromWorkerDelete(t *testing.T) { +func TestSuspendActorAsync_Finalized_KeepsCrashFromWorkerDelete(t *testing.T) { ctx := context.Background() st := newTestPersistence(t) seedSuspendTemplate(t, ctx, st, nil) @@ -173,14 +171,14 @@ func TestPauseActor_Finalized_KeepsCrashFromWorkerDelete(t *testing.T) { seedWireWorker(t, ctx, st, assignmentOf(created)) w, _ := newWireCaptureWorkflow(t, workerDeletedDuringReleaseStore{Interface: st}) - _, err := w.PauseActor(ctx, actorRef) + _, err := w.SuspendActor(ctx, actorRef, suspendAsync) assertConcurrentCrashKept(t, ctx, st, actorRef, err, "suspend failed: "+crashMessageWorkerPodGone) } -// TestPauseActor_Finalized_RecordsFidelity verifies pause records the template's -// preferred_fidelity on the local Snapshot alongside Locality and leaves -// DurableSnapshot unset until the actor is suspended. -func TestPauseActor_Finalized_RecordsFidelity(t *testing.T) { +// TestSuspendActorAsync_Finalized_RecordsFidelity verifies pause records the template's +// preferred_fidelity on the local Snapshot alongside Locality, and keeps its +// durable copy PENDING for atelet to upload in the background. +func TestSuspendActorAsync_Finalized_RecordsFidelity(t *testing.T) { tests := []struct { name string preferred ateapipb.SnapshotFidelity @@ -227,9 +225,9 @@ func TestPauseActor_Finalized_RecordsFidelity(t *testing.T) { }) seedWireWorker(t, ctx, st, assignmentOf(created)) - got, err := w.PauseActor(ctx, actorRef) + got, err := w.SuspendActor(ctx, actorRef, suspendAsync) if err != nil { - t.Fatalf("PauseActor: %v", err) + t.Fatalf("SuspendActor: %v", err) } if got.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { @@ -242,8 +240,11 @@ func TestPauseActor_Finalized_RecordsFidelity(t *testing.T) { if localSnap.GetFidelity() != tc.want { t.Errorf("local Snapshot.Fidelity = %v, want %v", localSnap.GetFidelity(), tc.want) } - if localSnap.GetDurableSnapshot() != nil { - t.Errorf("local DurableSnapshot = %v, want nil", localSnap.GetDurableSnapshot()) + if got := localSnap.GetDurableSnapshot().GetStatus(); got != ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING { + t.Errorf("local DurableSnapshot.Status = %v, want PENDING", got) + } + if got := localSnap.GetDurableSnapshot().GetObject().GetSnapshotUri(); got == "" { + t.Error("local DurableSnapshot records no upload URI") } if localSnap.GetLocality() != "node-1" { t.Errorf("local Snapshot.Locality = %q, want %q", localSnap.GetLocality(), "node-1") @@ -274,218 +275,82 @@ func TestPauseActor_Finalized_RecordsFidelity(t *testing.T) { } } -// TestPauseActor_Finalized_MissingLocalSnapshotClearsAssignment verifies that if the -// latest generation snapshot entry is missing when the pause finalizes, the -// actor transitions to CRASHED and clears WorkerAssignment after releasing -// the worker rather than getting stuck SUSPENDING, and the pause fails. -func TestPauseActor_Finalized_MissingLocalSnapshotClearsAssignment(t *testing.T) { - ctx := context.Background() - st := newTestPersistence(t) - seedSuspendTemplate(t, ctx, st, nil) - w, atelet := newWireCaptureWorkflow(t, st) - actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"} - records := crashRecords(t) - - created := storetest.MustCreateActor(t, ctx, st, &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, - ActorTemplate: suspendTestTemplateRef(), - Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, - AssignedNode: "node-1", - WorkerAssignment: wireTestAssignment(), - LastAssignedGeneration: 1, - Snapshots: []*ateapipb.Snapshot{ - newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-snap-1", ""), - }, - }, - }) - seedWireWorker(t, ctx, st, assignmentOf(created)) - updateDuringCheckpoint(t, atelet, st, actorRef, func(s *ateapipb.ActorStatus) { - s.Snapshots = nil - }) - - if _, err := w.PauseActor(ctx, actorRef); apierror.Code(err) != codes.FailedPrecondition { - t.Fatalf("PauseActor error = %v, want FailedPrecondition for the actor it crashed", err) - } - got := mustReadActor(t, ctx, st, actorRef) - if got.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_CRASHED { - t.Errorf("state = %v, want CRASHED", got.GetStatus().GetState()) - } - if msg, want := got.GetStatus().GetCrash().GetMessage(), "suspend failed: "+crashMessageLocalSnapshotMissing; msg != want { - t.Errorf("crash message = %q, want %q", msg, want) - } - if got.GetStatus().GetWorkerAssignment() != nil { - t.Errorf("WorkerAssignment = %v, want nil", got.GetStatus().GetWorkerAssignment()) - } - if len(*records) != 1 { - t.Fatalf("got %d crash records, want 1", len(*records)) - } - if got := (*records)[0].attrs[string(ateattr.ActorOperationNameKey)]; got != ateattr.OperationSuspend { - t.Errorf("crash record %s = %q, want %q", ateattr.ActorOperationNameKey, got, ateattr.OperationSuspend) - } -} - -// TestPauseActorWorkflow_RejectedAndIdempotentPaths covers the two -// short-circuit paths of the pause workflow: rejection of the pause edge for -// a non-RUNNING actor and the idempotent fast-forward for a paused one. Both -// are SUSPENDED: a node-local latest snapshot is what tells them apart. -func TestPauseActorWorkflow_RejectedAndIdempotentPaths(t *testing.T) { +// TestSuspendActorAsync_Finalized_MissingSnapshotClearsAssignment verifies that if the +// latest generation snapshot entry, or its PENDING durable copy, is missing +// when the pause finalizes, the actor transitions to CRASHED and clears +// WorkerAssignment after releasing the worker rather than getting stuck +// SUSPENDING, and the pause fails. +func TestSuspendActorAsync_Finalized_MissingSnapshotClearsAssignment(t *testing.T) { tests := []struct { - name string - seedState ateapipb.ActorState - seedOpts []func(*ateapipb.Actor) - // wantErr true means PauseActor must fail with FailedPrecondition. - wantErr bool - // wantState is the stored state after the call. - wantState ateapipb.ActorState + name string + // clobber rewrites the record while atelet checkpoints. + clobber func(s *ateapipb.ActorStatus) + wantCrashMessage string }{ { - // Pausing a SUSPENDED actor is rejected by the pause's state gate - // and the actor's state is left untouched. - name: "not running rejected", - seedState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - wantErr: true, - wantState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, + name: "no snapshot at the latest generation", + clobber: func(s *ateapipb.ActorStatus) { s.Snapshots = nil }, + wantCrashMessage: "suspend failed: " + crashMessageLocalSnapshotMissing, }, { - // Pausing a paused actor succeeds idempotently via IsComplete - // fast-forward without calling atelet. - name: "already paused succeeds", - seedState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - seedOpts: []func(*ateapipb.Actor){withPausedShape("node-1")}, - wantState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, + name: "no durable copy of the latest snapshot", + clobber: func(s *ateapipb.ActorStatus) { + for _, snap := range s.Snapshots { + snap.DurableSnapshot = nil + } + }, + wantCrashMessage: "suspend failed: " + crashMessageDurableSnapshotMissing, }, } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { ctx := context.Background() - st, cleanup := storetest.SetupTestStore(t) - defer cleanup() - w := newTestActorWorkflow(t, st, "ns", "tmpl1") + st := newTestPersistence(t) + seedSuspendTemplate(t, ctx, st, nil) + w, atelet := newWireCaptureWorkflow(t, st) + actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"} + records := crashRecords(t) - seedWorkflowActor(t, ctx, st, resources.ActorRef{Atespace: "team-a", Name: "id1"}, "ns", "tmpl1", tc.seedState, tc.seedOpts...) + created := storetest.MustCreateActor(t, ctx, st, &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, + ActorTemplate: suspendTestTemplateRef(), + Status: &ateapipb.ActorStatus{ + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, + AssignedNode: "node-1", + WorkerAssignment: wireTestAssignment(), + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{ + newInProgressSnapshot(t, 1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "local-snap-1"), + }, + }, + }) + seedWireWorker(t, ctx, st, assignmentOf(created)) + updateDuringCheckpoint(t, atelet, st, actorRef, tt.clobber) - actor, err := w.PauseActor(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}) - if tc.wantErr { - if got := apierror.Code(err); got != codes.FailedPrecondition { - t.Fatalf("apierror.Code(err) = %v, want %v (err: %v)", got, codes.FailedPrecondition, err) - } - } else { - if err != nil { - t.Fatalf("PauseActor failed: %v", err) - } - if actor.GetStatus().GetState() != tc.wantState { - t.Errorf("returned state = %v, want %v", actor.GetStatus().GetState(), tc.wantState) - } + if _, err := w.SuspendActor(ctx, actorRef, suspendAsync); apierror.Code(err) != codes.FailedPrecondition { + t.Fatalf("Async SuspendActor error = %v, want FailedPrecondition for the actor it crashed", err) } - - got, err := st.GetActor(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}) - if err != nil { - t.Fatalf("GetActor failed: %v", err) + got := mustReadActor(t, ctx, st, actorRef) + if got.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_CRASHED { + t.Errorf("state = %v, want CRASHED", got.GetStatus().GetState()) } - if got.GetStatus().GetState() != tc.wantState { - t.Errorf("stored state = %v, want %v", got.GetStatus().GetState(), tc.wantState) + if msg := got.GetStatus().GetCrash().GetMessage(); msg != tt.wantCrashMessage { + t.Errorf("crash message = %q, want %q", msg, tt.wantCrashMessage) + } + if got.GetStatus().GetWorkerAssignment() != nil { + t.Errorf("WorkerAssignment = %v, want nil", got.GetStatus().GetWorkerAssignment()) + } + if len(*records) != 1 { + t.Fatalf("got %d crash records, want 1", len(*records)) + } + if got := (*records)[0].attrs[string(ateattr.ActorOperationNameKey)]; got != ateattr.OperationSuspend { + t.Errorf("crash record %s = %q, want %q", ateattr.ActorOperationNameKey, got, ateattr.OperationSuspend) } }) } } -// TestPauseActor_MarkedPausing_StateMatrix verifies the pause edge's state gating against -// every actor state: RUNNING takes the edge, everything else is rejected with -// FailedPrecondition and left untouched. SUSPENDED is rejected whatever its -// snapshots short of the paused shape, which fast-forwards (see -// TestPauseActorWorkflow_RejectedAndIdempotentPaths). SUSPENDING without a -// pending node-local snapshot is a suspend in progress, which owns the actor; -// a pause in progress is covered by TestPauseActor_MarkedPausing_ReentrySkipsPauseInProgress. -func TestPauseActor_MarkedPausing_StateMatrix(t *testing.T) { - ctx := context.Background() - st := newTestPersistence(t) - seedSuspendTemplate(t, ctx, st, nil) - w := newMarkOnlyWorkflow(st) - - for _, seedState := range allActorStates { - actorRef := resources.ActorRef{Atespace: "team-a", Name: fmt.Sprintf("id%d", seedState)} - seedWorkflowActor(t, ctx, st, actorRef, "ns", "tmpl1", seedState, func(a *ateapipb.Actor) { - a.Status.WorkerAssignment = wireTestAssignment() - }) - - _, err := w.PauseActor(ctx, actorRef) - stored := mustReadActor(t, ctx, st, actorRef) - if seedState == ateapipb.ActorState_ACTOR_STATE_RUNNING { - assertStoppedAtDial(t, err) - if got := stored.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDING { - t.Errorf("state %v: stored state = %v, want SUSPENDING", seedState, got) - } - if !latestSnapshotIsPendingLocal(stored.GetStatus()) { - t.Errorf("state %v: snapshots = %v, want a pending local snapshot at the latest generation", seedState, stored.GetStatus().GetSnapshots()) - } - continue - } - if got := apierror.Code(err); got != codes.FailedPrecondition { - t.Errorf("state %v: code = %v, want FailedPrecondition (err: %v)", seedState, got, err) - } - // Only a sync suspend accepts a paused actor, so a pause must not offer it. - if strings.Contains(err.Error(), "node-local snapshot") { - t.Errorf("state %v: err = %v, want no mention of a node-local snapshot", seedState, err) - } - if got := stored.GetStatus().GetState(); got != seedState { - t.Errorf("state %v: stored state = %v, want it untouched", seedState, got) - } - } -} - -// TestPauseActor_MarkedPausing_ReentrySkipsPauseInProgress verifies a re-entered pause -// recognizes its own mark, a SUSPENDING actor whose latest generation is a -// pending node-local snapshot, and checkpoints under that snapshot without -// minting another, while a running-origin suspend in progress (pending -// durable snapshot) is rejected before atelet is called. -func TestPauseActor_MarkedPausing_ReentrySkipsPauseInProgress(t *testing.T) { - ctx := context.Background() - st := newTestPersistence(t) - seedSuspendTemplate(t, ctx, st, nil) - w, atelet := newWireCaptureWorkflow(t, st) - actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} - - seedWorkflowActor(t, ctx, st, actorRef, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_SUSPENDING, withPauseInProgress(), func(a *ateapipb.Actor) { - a.Status.AssignedNode = "node-1" - a.Status.WorkerAssignment = wireTestAssignment() - }) - paused, err := w.PauseActor(ctx, actorRef) - if err != nil { - t.Fatalf("PauseActor on a pause in progress: %v", err) - } - if got := atelet.checkpointRequest().GetSnapshot().GetSnapshotUuid(); got != "pause-snap" { - t.Errorf("CheckpointRequest.Snapshot.SnapshotUuid = %q, want the persisted %q", got, "pause-snap") - } - if got := paused.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { - t.Errorf("state = %v, want SUSPENDED", got) - } - if got := paused.GetStatus().GetLastAssignedGeneration(); got != 1 { - t.Errorf("last_assigned_generation = %d, want 1 (no new snapshot minted on re-entry)", got) - } - if got := len(paused.GetStatus().GetSnapshots()); got != 1 { - t.Errorf("len(snapshots) = %d, want 1", got) - } - - suspendingRef := resources.ActorRef{Atespace: actorRef.Atespace, Name: "id2"} - seedWorkflowActor(t, ctx, st, suspendingRef, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_SUSPENDING, func(a *ateapipb.Actor) { - a.Status.AssignedNode = "node-1" - a.Status.WorkerAssignment = wireTestAssignment() - a.Status.LastAssignedGeneration = 1 - a.Status.Snapshots = []*ateapipb.Snapshot{ - newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snap", "gs://snapshots/s", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), - } - }) - _, err = w.PauseActor(ctx, suspendingRef) - if got := apierror.Code(err); got != codes.FailedPrecondition { - t.Errorf("PauseActor on a suspend in progress: code = %v, want FailedPrecondition (err: %v)", got, err) - } - if checkpoints, _ := atelet.calls(); checkpoints != 1 { - t.Errorf("Checkpoint calls = %d, want 1 (none for the suspend in progress)", checkpoints) - } -} - -func TestPauseActor_AteletPaused_DialFailureLeavesActorRetryable(t *testing.T) { +func TestSuspendActorAsync_AteletPaused_DialFailureLeavesActorRetryable(t *testing.T) { tests := []struct { name string prevSnapshot string @@ -510,7 +375,7 @@ func TestPauseActor_AteletPaused_DialFailureLeavesActorRetryable(t *testing.T) { if tt.prevSnapshot != "" { snapshots = append(snapshots, newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "prev-snapshot", tt.prevSnapshot, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED)) } - snapshots = append(snapshots, newLocalSnapshot(2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "actor-1-never-written", "")) + snapshots = append(snapshots, newInProgressSnapshot(t, 2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "actor-1-never-written")) actor := &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor-1"}, @@ -530,7 +395,7 @@ func TestPauseActor_AteletPaused_DialFailureLeavesActorRetryable(t *testing.T) { storetest.MustCreateActor(t, ctx, persistence, actor) w := newMarkOnlyWorkflow(persistence) - _, err := w.PauseActor(ctx, resources.ActorRef{Atespace: "team-a", Name: "actor-1"}) + _, err := w.SuspendActor(ctx, resources.ActorRef{Atespace: "team-a", Name: "actor-1"}, suspendAsync) assertStoppedAtDial(t, err) stored := mustReadActor(t, ctx, persistence, resources.ActorRef{Atespace: "team-a", Name: "actor-1"}) @@ -548,19 +413,19 @@ func TestPauseActor_AteletPaused_DialFailureLeavesActorRetryable(t *testing.T) { } } -// TestPauseActor_CrashesWhenPausingActorMissingWorkerPod verifies that a +// TestSuspendActorAsync_CrashesWhenPausingActorMissingWorkerPod verifies that a // pausing actor with no worker pod recorded is moved to CRASHED by the // corrupted-assignment check before atelet is called, and the pause fails // with FailedPrecondition. -func TestPauseActor_CrashesWhenPausingActorMissingWorkerPod(t *testing.T) { +func TestSuspendActorAsync_CrashesWhenPausingActorMissingWorkerPod(t *testing.T) { ctx := context.Background() st, cleanup := storetest.SetupTestStore(t) defer cleanup() w := newTestActorWorkflow(t, st, "ns", "tmpl1") - seedWorkflowActor(t, ctx, st, resources.ActorRef{Atespace: "team-a", Name: "id1"}, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_SUSPENDING, withPauseInProgress()) + seedWorkflowActor(t, ctx, st, resources.ActorRef{Atespace: "team-a", Name: "id1"}, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_SUSPENDING, withPauseInProgress(t)) - _, err := w.PauseActor(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}) + _, err := w.SuspendActor(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, suspendAsync) if got := apierror.Code(err); got != codes.FailedPrecondition { t.Fatalf("apierror.Code(err) = %v, want %v (err: %v)", got, codes.FailedPrecondition, err) } @@ -577,32 +442,10 @@ func TestPauseActor_CrashesWhenPausingActorMissingWorkerPod(t *testing.T) { } } -// TestPauseActor_MarkedPausing_GoldenAtespaceRejected verifies golden actors cannot be -// paused: by design they can only be suspended (committed). -func TestPauseActor_MarkedPausing_GoldenAtespaceRejected(t *testing.T) { - ctx := context.Background() - st := newTestPersistence(t) - seedSuspendTemplate(t, ctx, st, nil) - w := newMarkOnlyWorkflow(st) - - actorRef := resources.ActorRef{Atespace: resources.GoldenActorAtespace, Name: "golden-1"} - seedWorkflowActor(t, ctx, st, actorRef, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_RUNNING, func(a *ateapipb.Actor) { - a.Status.AssignedNode = "node-1" - a.Status.WorkerAssignment = wireTestAssignment() - }) - _, err := w.PauseActor(ctx, actorRef) - if got := apierror.Code(err); got != codes.FailedPrecondition { - t.Fatalf("apierror.Code = %v (err %v), want FailedPrecondition", got, err) - } - if got := mustReadActor(t, ctx, st, actorRef).GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RUNNING { - t.Errorf("stored state = %v, want RUNNING untouched", got) - } -} - -// TestPauseActor_AteletPaused_ForwardsPreferredFidelity verifies that local pause +// TestSuspendActorAsync_AteletPaused_ForwardsPreferredFidelity verifies that local pause // checkpoints forward the template's PreferredFidelity to atelet, and label the // pause's lifecycle-op metric with it. -func TestPauseActor_AteletPaused_ForwardsPreferredFidelity(t *testing.T) { +func TestSuspendActorAsync_AteletPaused_ForwardsPreferredFidelity(t *testing.T) { tests := []struct { preferred ateapipb.SnapshotFidelity want ateletpb.SnapshotFidelity @@ -616,13 +459,13 @@ func TestPauseActor_AteletPaused_ForwardsPreferredFidelity(t *testing.T) { t.Run(tc.preferred.String(), func(t *testing.T) { ctx := context.Background() persistence := newTestPersistence(t) - tmpl := seedSuspendTemplate(t, ctx, persistence, &ateapipb.SnapshotConfig{PreferredFidelity: tc.preferred}) + seedSuspendTemplate(t, ctx, persistence, &ateapipb.SnapshotConfig{PreferredFidelity: tc.preferred}) w, atelet := newWireCaptureWorkflow(t, persistence) inst, reader := newTestInstruments(t) w.instruments = inst actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"} - created := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ + storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, ActorTemplate: suspendTestTemplateRef(), Status: &ateapipb.ActorStatus{ @@ -631,13 +474,13 @@ func TestPauseActor_AteletPaused_ForwardsPreferredFidelity(t *testing.T) { WorkerAssignment: wireTestAssignment(), LastAssignedGeneration: 1, Snapshots: []*ateapipb.Snapshot{ - newLocalSnapshot(1, tc.preferred, "", "local-snap-1", ""), + newInProgressSnapshot(t, 1, tc.preferred, "local-snap-1"), }, }, }) - if _, err := w.PauseActor(ctx, actorRef); err != nil { - t.Fatalf("PauseActor: %v", err) + if _, err := w.SuspendActor(ctx, actorRef, suspendAsync); err != nil { + t.Fatalf("SuspendActor: %v", err) } req := atelet.checkpointRequest() if req == nil { @@ -652,7 +495,8 @@ func TestPauseActor_AteletPaused_ForwardsPreferredFidelity(t *testing.T) { if got := req.GetSnapshot().GetGeneration(); got != 1 { t.Errorf("CheckpointRequest.Snapshot.Generation = %d, want 1, the in-progress snapshot's", got) } - wantURI := mustActorSnapshotURI(t, tmpl, created, "local-snap-1").String() + // The mark recorded the upload URI; the checkpoint goes there. + wantURI := someActorSnapshotURI(t, testStorageLocation, "team-a", "local-snap-1") if got := req.GetSnapshot().GetObject().GetSnapshotUri(); got != wantURI { t.Errorf("CheckpointRequest.Snapshot.Object.SnapshotUri = %q, want %q", got, wantURI) } diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go index 9f8a6c0a74..721be71052 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go @@ -62,7 +62,7 @@ func TestSuspendActor_MarkedSuspending_SnapshotURI(t *testing.T) { }, }) w := newMarkOnlyWorkflow(persistence) - _, err := w.SuspendActor(ctx, actorRef) + _, err := w.SuspendActor(ctx, actorRef, suspendSync) assertStoppedAtDial(t, err) marked := mustReadActor(t, ctx, persistence, actorRef) @@ -84,229 +84,177 @@ func TestSuspendActor_MarkedSuspending_SnapshotURI(t *testing.T) { } // TestSuspendActor_MarkedSuspending_ReentryKeepsPersistedSnapshotLocation verifies a -// re-entered workflow does not mint a second snapshot location: the location -// persisted by the first attempt stays authoritative, both on the wire and in -// the snapshot the suspend commits. +// re-entered workflow, in either mode, does not mint a second snapshot +// location: the location persisted by the first attempt stays authoritative, +// both on the wire and in the snapshot the suspend commits. Both modes mark +// the same shape, so the retry's mode may differ from the first attempt's. func TestSuspendActor_MarkedSuspending_ReentryKeepsPersistedSnapshotLocation(t *testing.T) { const firstAttempt = "gs://bucket/root/atespaces/team-a/actors/actor-uid/snapshots/first-attempt" + pending := ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + completed := ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED - ctx := context.Background() - persistence := newTestPersistence(t) - seedSuspendTemplate(t, ctx, persistence, &ateapipb.SnapshotConfig{StorageLocation: testStorageLocation}) - w, atelet := newWireCaptureWorkflow(t, persistence) - actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"} - storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, - ActorTemplate: suspendTestTemplateRef(), - Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, - AssignedNode: "node-1", - WorkerAssignment: wireTestAssignment(), - LastAssignedGeneration: 1, - Snapshots: []*ateapipb.Snapshot{ - newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "first-attempt", firstAttempt, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), - }, - }, - }) - suspended, err := w.SuspendActor(ctx, actorRef) - if err != nil { - t.Fatalf("SuspendActor: %v", err) - } - if got := atelet.checkpointRequest().GetSnapshot().GetObject().GetSnapshotUri(); got != firstAttempt { - t.Errorf("CheckpointRequest.Snapshot.Object.SnapshotUri = %q, want the first attempt's location", got) - } - if got := suspended.GetStatus().GetLastAssignedGeneration(); got != 1 { - t.Errorf("last_assigned_generation = %d, want 1 (no new snapshot minted on re-entry)", got) - } - gotSnap := findLatestDurableSnapshot(suspended.GetStatus(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) - if got := gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri(); got != firstAttempt { - t.Errorf("committed durable snapshot URI = %q, want the first attempt's location", got) - } -} - -// TestSuspendActorWorkflow_RejectedAndIdempotentPaths covers the two -// short-circuit paths of the suspend workflow: rejection of the suspend edge -// for a non-RUNNING actor and the idempotent fast-forward for a SUSPENDED one. -func TestSuspendActorWorkflow_RejectedAndIdempotentPaths(t *testing.T) { tests := []struct { - name string - seedState ateapipb.ActorState - // wantErr true means SuspendActor must fail with FailedPrecondition. - wantErr bool - // wantState is the stored state after the call. - wantState ateapipb.ActorState + name string + mode suspendMode + wantUploadMode ateletpb.UploadMode + wantLocality string + wantDurable ateapipb.SnapshotStorageStatus + wantNode string }{ - { - // Suspending a SUSPENDED actor succeeds idempotently via - // IsComplete fast-forward without calling atelet. - name: "newly created suspended succeeds", - seedState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - wantState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - }, + {"sync", suspendSync, ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE, "", completed, ""}, + {"async", suspendAsync, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED, "node-1", pending, "node-1"}, } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { ctx := context.Background() - st, cleanup := storetest.SetupTestStore(t) - defer cleanup() - w := newTestActorWorkflow(t, st, "ns", "tmpl1") - - seedWorkflowActor(t, ctx, st, resources.ActorRef{Atespace: "team-a", Name: "id1"}, "ns", "tmpl1", tc.seedState) - - actor, err := w.SuspendActor(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}) - if tc.wantErr { - if got := apierror.Code(err); got != codes.FailedPrecondition { - t.Fatalf("apierror.Code(err) = %v, want %v (err: %v)", got, codes.FailedPrecondition, err) - } - } else { - if err != nil { - t.Fatalf("SuspendActor failed: %v", err) - } - if actor.GetStatus().GetState() != tc.wantState { - t.Errorf("returned state = %v, want %v", actor.GetStatus().GetState(), tc.wantState) - } - } - - got, err := st.GetActor(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}) + persistence := newTestPersistence(t) + seedSuspendTemplate(t, ctx, persistence, &ateapipb.SnapshotConfig{StorageLocation: testStorageLocation}) + w, atelet := newWireCaptureWorkflow(t, persistence) + actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"} + storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, + ActorTemplate: suspendTestTemplateRef(), + Status: &ateapipb.ActorStatus{ + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, + AssignedNode: "node-1", + WorkerAssignment: wireTestAssignment(), + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "first-attempt", firstAttempt, pending), + }, + }, + }) + suspended, err := w.SuspendActor(ctx, actorRef, tt.mode) if err != nil { - t.Fatalf("GetActor failed: %v", err) + t.Fatalf("SuspendActor: %v", err) } - if got.GetStatus().GetState() != tc.wantState { - t.Errorf("stored state = %v, want %v", got.GetStatus().GetState(), tc.wantState) + req := atelet.checkpointRequest() + if got := req.GetSnapshot().GetObject().GetSnapshotUri(); got != firstAttempt { + t.Errorf("CheckpointRequest.Snapshot.Object.SnapshotUri = %q, want the first attempt's location", got) } - }) - } -} - -// TestSuspendActor_MarkedSuspending_StateMatrix verifies the suspend edge's state gating -// against every actor state: RUNNING takes the edge (checkpoint the -// workload), SUSPENDING skips the mark (a previous attempt already marked the -// actor) and goes on to atelet, everything else is rejected with -// FailedPrecondition and left untouched. SUSPENDED without a node-local -// snapshot fast-forwards (see TestSuspendActorWorkflow_RejectedAndIdempotentPaths); -// SUSPENDED with one is the paused shape, covered by -// TestSuspendActor_MarkedSuspending_PausedPromotesLocalSnapshot. -func TestSuspendActor_MarkedSuspending_StateMatrix(t *testing.T) { - ctx := context.Background() - persistence := newTestPersistence(t) - seedSuspendTemplate(t, ctx, persistence, &ateapipb.SnapshotConfig{StorageLocation: "gs://snapshots"}) - w := newMarkOnlyWorkflow(persistence) - - for _, seedState := range allActorStates { - actorRef := resources.ActorRef{Atespace: "team-a", Name: fmt.Sprintf("id%d", seedState)} - seedWorkflowActor(t, ctx, persistence, actorRef, "ns", "tmpl1", seedState, func(a *ateapipb.Actor) { - a.Status.WorkerAssignment = wireTestAssignment() - }) - - _, err := w.SuspendActor(ctx, actorRef) - stored := mustReadActor(t, ctx, persistence, actorRef) - switch seedState { - case ateapipb.ActorState_ACTOR_STATE_RUNNING, ateapipb.ActorState_ACTOR_STATE_SUSPENDING: - assertStoppedAtDial(t, err) - if got := stored.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDING { - t.Errorf("state %v: stored state = %v, want SUSPENDING", seedState, got) + if got := req.GetUploadConfig().GetMode(); got != tt.wantUploadMode { + t.Errorf("CheckpointRequest.UploadConfig.Mode = %v, want %v", got, tt.wantUploadMode) } - case ateapipb.ActorState_ACTOR_STATE_SUSPENDED: - if err != nil { - t.Errorf("state %v: SuspendActor = %v, want the fast-forward", seedState, err) + if got := suspended.GetStatus().GetLastAssignedGeneration(); got != 1 { + t.Errorf("last_assigned_generation = %d, want 1 (no new snapshot minted on re-entry)", got) } - if got := stored.GetStatus().GetState(); got != seedState { - t.Errorf("state %v: stored state = %v, want it untouched", seedState, got) + gotSnap := snapshotAtLatestGeneration(suspended.GetStatus()) + if got := gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri(); got != firstAttempt { + t.Errorf("committed durable snapshot URI = %q, want the first attempt's location", got) } - default: - if got := apierror.Code(err); got != codes.FailedPrecondition { - t.Errorf("state %v: code = %v, want FailedPrecondition (err: %v)", seedState, got, err) + if got := gotSnap.GetDurableSnapshot().GetStatus(); got != tt.wantDurable { + t.Errorf("durable copy status = %v, want %v", got, tt.wantDurable) } - if got := stored.GetStatus().GetState(); got != seedState { - t.Errorf("state %v: stored state = %v, want it untouched", seedState, got) + if got := gotSnap.GetLocality(); got != tt.wantLocality { + t.Errorf("Snapshot.Locality = %q, want %q", got, tt.wantLocality) } - } + if got := suspended.GetStatus().GetAssignedNode(); got != tt.wantNode { + t.Errorf("AssignedNode = %q, want %q", got, tt.wantNode) + } + }) } } -// TestSuspendActor_MarkedSuspending_RejectsPauseInProgress verifies a suspend does not adopt a -// SUSPENDING actor whose latest generation is a pending node-local snapshot: -// that actor is mid-pause, and the pause owns it until it is paused. -func TestSuspendActor_MarkedSuspending_RejectsPauseInProgress(t *testing.T) { - ctx := context.Background() - persistence := newTestPersistence(t) - seedSuspendTemplate(t, ctx, persistence, &ateapipb.SnapshotConfig{StorageLocation: "gs://snapshots"}) - w := newMarkOnlyWorkflow(persistence) - actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} - - seedWorkflowActor(t, ctx, persistence, actorRef, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_SUSPENDING, withPauseInProgress(), func(a *ateapipb.Actor) { - a.Status.WorkerAssignment = wireTestAssignment() - }) - - _, err := w.SuspendActor(ctx, actorRef) - if got := apierror.Code(err); got != codes.FailedPrecondition { - t.Errorf("SuspendActor on a pause in progress: code = %v, want FailedPrecondition (err: %v)", got, err) +// TestSuspendActorWorkflow_SuspendedReturnsAsIs verifies that SuspendActor, in +// either mode, returns a SUSPENDED actor as it is, wherever its snapshot is. +// It never promotes a node-local snapshot: the stored record, durable copies, +// locality, and assigned node included, is left exactly as seeded. +func TestSuspendActorWorkflow_SuspendedReturnsAsIs(t *testing.T) { + durableURI := someActorSnapshotURI(t, "gs://snapshots", "team-a", "durable-snap") + withDurable := func(a *ateapipb.Actor) { + a.Status.LastAssignedGeneration = 1 + a.Status.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "durable-snap", durableURI, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } } - stored := mustReadActor(t, ctx, persistence, actorRef) - if got := stored.GetStatus().GetLastAssignedGeneration(); got != 1 { - t.Errorf("last_assigned_generation = %d, want 1 (the pause's snapshot untouched)", got) + + tests := []struct { + name string + seedOpts []func(*ateapipb.Actor) + }{ + {name: "newly created, no snapshot"}, + {name: "durable snapshot", seedOpts: []func(*ateapipb.Actor){withDurable}}, + {name: "paused, upload pending", seedOpts: []func(*ateapipb.Actor){withPausedShape(t, "node-1")}}, + {name: "paused, uploaded", seedOpts: []func(*ateapipb.Actor){withUploadedPausedShape(t, "node-1")}}, } - if !latestSnapshotIsPendingLocal(stored.GetStatus()) { - t.Errorf("snapshots = %v, want the pause's pending local snapshot untouched", stored.GetStatus().GetSnapshots()) + for _, tc := range tests { + for _, mode := range suspendModes { + t.Run(tc.name+"/"+mode.name, func(t *testing.T) { + ctx := context.Background() + st, cleanup := storetest.SetupTestStore(t) + defer cleanup() + w := newTestActorWorkflow(t, st, "ns", "tmpl1") + actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} + + seedWorkflowActor(t, ctx, st, actorRef, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_SUSPENDED, tc.seedOpts...) + seeded := mustReadActor(t, ctx, st, actorRef) + + returned, err := w.SuspendActor(ctx, actorRef, mode.suspendMode) + if err != nil { + t.Fatalf("SuspendActor: %v", err) + } + if diff := cmp.Diff(seeded, returned, protocmp.Transform()); diff != "" { + t.Errorf("returned actor mismatch (-seeded +returned):\n%s", diff) + } + if diff := cmp.Diff(seeded, mustReadActor(t, ctx, st, actorRef), protocmp.Transform()); diff != "" { + t.Errorf("stored actor changed (-seeded +stored):\n%s", diff) + } + }) + } } } -// TestSuspendActor_MarkedSuspending_PausedPromotesLocalSnapshot verifies the suspend edge from -// a paused actor (SUSPENDED whose latest snapshot is node-local): it takes the -// edge and promotes that snapshot in place to a pending durable one, keeping -// its generation, uuid, and sandbox runtime stamp, with the upload destination -// derived from the uuid. -func TestSuspendActor_MarkedSuspending_PausedPromotesLocalSnapshot(t *testing.T) { - ctx := context.Background() - persistence := newTestPersistence(t) - seedSuspendTemplate(t, ctx, persistence, &ateapipb.SnapshotConfig{StorageLocation: "gs://snapshots"}) - w := newMarkOnlyWorkflow(persistence) +// TestSuspendActor_MarkedSuspending_StateMatrix verifies the suspend edge's +// state gating against every actor state, in either mode: RUNNING takes the +// edge and marks a snapshot with a PENDING durable copy, SUSPENDING skips the +// mark (a previous attempt already marked the actor) and goes on to atelet, +// SUSPENDED is returned as it is (see TestSuspendActorWorkflow_SuspendedReturnsAsIs), +// and everything else is rejected with FailedPrecondition and left untouched. +func TestSuspendActor_MarkedSuspending_StateMatrix(t *testing.T) { + for _, mode := range suspendModes { + t.Run(mode.name, func(t *testing.T) { + ctx := context.Background() + persistence := newTestPersistence(t) + seedSuspendTemplate(t, ctx, persistence, &ateapipb.SnapshotConfig{StorageLocation: "gs://snapshots"}) + w := newMarkOnlyWorkflow(persistence) - actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} - localSnap := newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snap-1", "node-1") - localSnap.SandboxRuntime = testSandboxRuntime() - actor := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, - ActorTemplate: suspendTestTemplateRef(), - Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - AssignedNode: "node-1", - LastAssignedGeneration: 1, - Snapshots: []*ateapipb.Snapshot{localSnap}, - }, - }) + for _, seedState := range allActorStates { + actorRef := resources.ActorRef{Atespace: "team-a", Name: fmt.Sprintf("id%d", seedState)} + seedWorkflowActor(t, ctx, persistence, actorRef, "ns", "tmpl1", seedState, func(a *ateapipb.Actor) { + a.Status.WorkerAssignment = wireTestAssignment() + }) - // The upload's dial fails, so the actor stays as the mark left it. - _, err := w.SuspendActor(ctx, actorRef) - assertStoppedAtDial(t, err) - marked := mustReadActor(t, ctx, persistence, actorRef) - if got := marked.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDING { - t.Errorf("state = %v, want SUSPENDING", got) - } - wantURI, err := resources.NewActorSnapshotURI("gs://snapshots", actorRef.Atespace, actor.GetMetadata().GetUid(), "snap-1") - if err != nil { - t.Fatalf("NewActorSnapshotURI: %v", err) - } - pending := findLatestDurableSnapshot(marked.GetStatus(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) - gotURI := pending.GetDurableSnapshot().GetObject().GetSnapshotUri() - if gotURI != wantURI.String() { - t.Errorf("paused-origin in-progress SnapshotUri = %q, want %q (derived from local snapshot uuid)", gotURI, wantURI.String()) - } - // The upload promotes the pause snapshot in place rather than minting a - // new generation. - if pending.GetGeneration() != 1 || pending.GetUuid() != "snap-1" { - t.Errorf("paused-origin pending snapshot = generation %d, uuid %q; want the pause snapshot (generation 1, uuid %q)", pending.GetGeneration(), pending.GetUuid(), "snap-1") - } - if pending.GetLocality() != "node-1" { - t.Errorf("pending snapshot locality = %q, want node-1 kept until the upload finishes", pending.GetLocality()) - } - // The durable copy is the pause checkpoint uploaded, so it keeps the - // runtime stamp the pause recorded. - if diff := cmp.Diff(testSandboxRuntime(), pending.GetSandboxRuntime(), protocmp.Transform()); diff != "" { - t.Errorf("paused-origin pending Snapshot.SandboxRuntime mismatch (-want +got):\n%s", diff) - } - if got := marked.GetStatus().GetLastAssignedGeneration(); got != 1 { - t.Errorf("paused-origin LastAssignedGeneration = %d, want 1", got) + _, err := w.SuspendActor(ctx, actorRef, mode.suspendMode) + stored := mustReadActor(t, ctx, persistence, actorRef) + switch seedState { + case ateapipb.ActorState_ACTOR_STATE_RUNNING, ateapipb.ActorState_ACTOR_STATE_SUSPENDING: + assertStoppedAtDial(t, err) + if got := stored.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDING { + t.Errorf("state %v: stored state = %v, want SUSPENDING", seedState, got) + } + if seedState == ateapipb.ActorState_ACTOR_STATE_RUNNING { + latest := snapshotAtLatestGeneration(stored.GetStatus()) + if latest.GetDurableSnapshot().GetStatus() != ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING || latest.GetLocality() != "" { + t.Errorf("state %v: latest snapshot = %v, want a PENDING durable copy and no locality", seedState, latest) + } + } + case ateapipb.ActorState_ACTOR_STATE_SUSPENDED: + if err != nil { + t.Errorf("state %v: SuspendActor = %v, want it returned as it is", seedState, err) + } + if got := stored.GetStatus().GetState(); got != seedState { + t.Errorf("state %v: stored state = %v, want it untouched", seedState, got) + } + default: + if got := apierror.Code(err); got != codes.FailedPrecondition { + t.Errorf("state %v: code = %v, want FailedPrecondition (err: %v)", seedState, got, err) + } + if got := stored.GetStatus().GetState(); got != seedState { + t.Errorf("state %v: stored state = %v, want it untouched", seedState, got) + } + } + } + }) } } @@ -322,7 +270,7 @@ func TestSuspendActor_CrashesWhenSuspendingActorMissingWorkerPod(t *testing.T) { seedWorkflowActor(t, ctx, st, resources.ActorRef{Atespace: "team-a", Name: "id1"}, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_SUSPENDING) - _, err := w.SuspendActor(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}) + _, err := w.SuspendActor(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, suspendSync) if got := apierror.Code(err); got != codes.FailedPrecondition { t.Fatalf("apierror.Code(err) = %v, want %v (err: %v)", got, codes.FailedPrecondition, err) } @@ -401,7 +349,7 @@ func TestSuspendActor_AteletSuspended_DialFailureLeavesActorRetryable(t *testing storetest.MustCreateActor(t, ctx, persistence, actor) w := newMarkOnlyWorkflow(persistence) - _, err := w.SuspendActor(ctx, resources.ActorRef{Atespace: "team-a", Name: "actor-1"}) + _, err := w.SuspendActor(ctx, resources.ActorRef{Atespace: "team-a", Name: "actor-1"}, suspendSync) assertStoppedAtDial(t, err) // A dial failure is transient from this workflow's point of view: it @@ -453,7 +401,7 @@ func TestSuspendActor_AteletSuspended_WireRequest(t *testing.T) { }, }) - if _, err := w.SuspendActor(ctx, actorRef); err != nil { + if _, err := w.SuspendActor(ctx, actorRef, suspendSync); err != nil { t.Fatalf("SuspendActor: %v", err) } req := atelet.checkpointRequest() @@ -477,79 +425,6 @@ func TestSuspendActor_AteletSuspended_WireRequest(t *testing.T) { assertLifecycleFidelity(t, reader, ateattr.SnapshotFidelityMemory) } -// TestSuspendActor_Finalized_NoAssignment verifies finalization runs even -// when the actor has no worker assignment: the external snapshot must be -// recorded on the actor and the actor moved to SUSPENDED rather than silently -// left SUSPENDING. This is the shape a paused-origin suspend (#791) produces — -// a paused actor has no worker — and the regression test for finalization -// previously living inside the worker-freeing branch. -func TestSuspendActor_Finalized_NoAssignment(t *testing.T) { - ctx := context.Background() - persistence := newTestPersistence(t) - seedSuspendTemplate(t, ctx, persistence, &ateapipb.SnapshotConfig{StorageLocation: testStorageLocation}) - w, atelet := newWireCaptureWorkflow(t, persistence) - inst, reader := newTestInstruments(t) - w.instruments = inst - actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"} - - snapshotURI := someActorSnapshotURI(t, testStorageLocation, "team-a", "2026-01-01t00-00-00z-abc") - snap := newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-uid-1", "actor-1-pause-snapshot", snapshotURI, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) - snap.Locality = "node-1" - storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, - ActorTemplate: suspendTestTemplateRef(), - Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, - AssignedNode: "node-1", - LastAssignedGeneration: 1, - Snapshots: []*ateapipb.Snapshot{snap}, - }, - }) - - stored, err := w.SuspendActor(ctx, actorRef) - if err != nil { - t.Fatalf("SuspendActor: %v", err) - } - - // With no worker there is no workload to checkpoint: the suspend uploads - // the node-local snapshot instead. - if checkpoints, promotes := atelet.calls(); checkpoints != 0 || promotes != 1 { - t.Errorf("atelet calls = %d Checkpoint, %d PromoteSnapshot; want 0 and 1", checkpoints, promotes) - } - promote := atelet.promoteRequest() - if got := promote.GetSnapshot().GetSnapshotUuid(); got != "actor-1-pause-snapshot" { - t.Errorf("PromoteSnapshotRequest.Snapshot.SnapshotUuid = %q, want %q", got, "actor-1-pause-snapshot") - } - if got := promote.GetSnapshot().GetObject().GetSnapshotUri(); got != snapshotURI { - t.Errorf("PromoteSnapshotRequest.Snapshot.Object.SnapshotUri = %q, want %q", got, snapshotURI) - } - // The template's preferred fidelity, which the store defaults to MEMORY, - // labels the suspend's lifecycle-op metric. - assertLifecycleFidelity(t, reader, ateattr.SnapshotFidelityMemory) - - if stored.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { - t.Errorf("state = %v, want SUSPENDED", stored.GetStatus().GetState()) - } - gotSnap := findLatestDurableSnapshot(stored.GetStatus(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) - if got := gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri(); got != snapshotURI { - t.Errorf("SnapshotUri = %q, want %q", got, snapshotURI) - } - // The snapshot carries the template it was captured under, so a later - // repoint can tell that its guest state no longer matches. - if got := gotSnap.GetActorTemplateUid(); got != "tmpl-uid-1" { - t.Errorf("Snapshot.ActorTemplateUid = %q, want %q", got, "tmpl-uid-1") - } - if got := findLatestDurableSnapshot(stored.GetStatus(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING); got != nil { - t.Errorf("in-progress durable snapshot = %v, want cleared", got) - } - if local := findLatestLocalSnapshot(stored.GetStatus()); local != nil { - t.Errorf("LocalSnapshot = %v, want cleared", local) - } - if got := stored.GetStatus().GetAssignedNode(); got != "" { - t.Errorf("AssignedNode = %q, want cleared", got) - } -} - // TestSuspendActor_Finalized_ReleasesReplacedSnapshot verifies which external // snapshot a suspend collects. The actor's previous one goes away, because // nothing else can name it — unless the actor was only borrowing it from a tag, @@ -614,7 +489,7 @@ func TestSuspendActor_Finalized_ReleasesReplacedSnapshot(t *testing.T) { } }) - stored, err := w.SuspendActor(ctx, actorRef) + stored, err := w.SuspendActor(ctx, actorRef, suspendSync) if err != nil { t.Fatalf("SuspendActor: %v", err) } @@ -634,6 +509,90 @@ func TestSuspendActor_Finalized_ReleasesReplacedSnapshot(t *testing.T) { } } +// TestSuspendActor_Finalized_PauseReleasesReplacedSnapshot verifies that a +// pause keeping the previous pause's upload collects the external snapshot +// that upload replaced, and keeps the upload. The upload is recorded from the +// pause's Checkpoint state, or before the pause from atelet's report. +func TestSuspendActor_Finalized_PauseReleasesReplacedSnapshot(t *testing.T) { + const node = "node-1" + memory := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY + completed := ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + pending := ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + for _, tc := range []struct { + name string + // reported records the upload before the pause, as atelet's report + // does. Otherwise only the pause's Checkpoint state carries it. + reported bool + }{ + {name: "recorded from the pause's Checkpoint state"}, + {name: "recorded from atelet's report before the pause", reported: true}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + persistence := newTestPersistence(t) + template := seedSubstrateTemplate(t, ctx, persistence, "sub-tmpl") + w, atelet, objects := newWireFinalizeWorkflow(t, persistence) + + actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"} + actor := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: "team-a", Name: "sub-tmpl"}, + Status: &ateapipb.ActorStatus{ + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, + AssignedNode: node, + WorkerAssignment: wireTestAssignment(), + }, + }) + + // The actor suspended to old, paused to snap-2 and resumed from + // it, and is now pausing to snap-3. atelet has uploaded snap-2. + old := mustActorSnapshotURI(t, template, actor, "old") + paused := mustActorSnapshotURI(t, template, actor, "snap-2") + pausing := mustActorSnapshotURI(t, template, actor, "snap-3") + objects.PutSnapshot(t, old, "manifest.json") + objects.PutSnapshot(t, paused, "manifest.json") + mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { + previousStatus := pending + if tc.reported { + previousStatus = completed + } + previous := newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, memory, "", paused.Name(), paused.String(), previousStatus) + previous.Locality = node + s.LastAssignedGeneration = 3 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, memory, "", old.Name(), old.String(), completed), + previous, + newDurableSnapshot(3, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, memory, "", pausing.Name(), pausing.String(), pending), + } + }) + atelet.checkpointState = &ateletpb.ActorSnapshotState{LatestDurable: &ateletpb.SnapshotReport{Snapshot: &ateletpb.Snapshot{ + SnapshotUuid: "snap-2", + Generation: 2, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateletpb.ObjectStorage{SnapshotUri: paused.String()}, + }}} + + stored, err := w.SuspendActor(ctx, actorRef, suspendAsync) + if err != nil { + t.Fatalf("SuspendActor: %v", err) + } + want := []snapshotOutcome{{generation: 2, fidelity: memory, durable: completed}, {generation: 3, fidelity: memory, locality: node, durable: pending}} + if diff := cmp.Diff(want, describeSnapshots(stored.GetStatus()), cmp.AllowUnexported(snapshotOutcome{})); diff != "" { + t.Errorf("snapshots mismatch (-want +got):\n%s", diff) + } + if got := findLatestDurableSnapshot(stored.GetStatus(), completed).GetDurableSnapshot().GetObject().GetSnapshotUri(); got != paused.String() { + t.Errorf("external snapshot = %q, want the previous pause's upload, %q", got, paused) + } + if len(objects.Snapshot(t, old)) != 0 { + t.Error("the external snapshot the recorded upload replaced was not released") + } + if len(objects.Snapshot(t, paused)) == 0 { + t.Error("the recorded upload was released") + } + }) + } +} + // errObjectStore stands in for object storage being unreachable. var errObjectStore = errors.New("object storage is unavailable") @@ -681,7 +640,7 @@ func TestSuspendActor_Finalized_CommitsDespiteObjectStoreFailure(t *testing.T) { }) objects.OnDelete = func(string, string) error { return errObjectStore } - stored, err := w.SuspendActor(ctx, actorRef) + stored, err := w.SuspendActor(ctx, actorRef, suspendSync) if err != nil { t.Fatalf("SuspendActor: %v", err) } @@ -738,7 +697,7 @@ func TestSuspendActor_Finalized_KeepsReplacedSnapshotOnConflict(t *testing.T) { } }) - if _, err := w.SuspendActor(ctx, actorRef); apierror.Code(err) != codes.Aborted { + if _, err := w.SuspendActor(ctx, actorRef, suspendSync); apierror.Code(err) != codes.Aborted { t.Fatalf("SuspendActor = %v, want code Aborted", err) } // Every actor update conflicts, so pin that the Aborted came from finalize @@ -787,7 +746,7 @@ func TestSuspendActor_Finalized_KeepsCrashFromWorkerDelete(t *testing.T) { seedWireWorker(t, ctx, persistence, assignmentOf(created)) w, _ := newWireCaptureWorkflow(t, workerDeletedDuringReleaseStore{Interface: persistence}) - _, err := w.SuspendActor(ctx, actorRef) + _, err := w.SuspendActor(ctx, actorRef, suspendSync) assertConcurrentCrashKept(t, ctx, persistence, actorRef, err, "suspend failed: "+crashMessageWorkerPodGone) } @@ -850,7 +809,7 @@ func TestSuspendActor_Finalized_ReleasesOnlyOwnWorker(t *testing.T) { ActorUid: uid, }) - if _, err := w.SuspendActor(ctx, resources.ActorRef{Atespace: "team-a", Name: "shared"}); err != nil { + if _, err := w.SuspendActor(ctx, resources.ActorRef{Atespace: "team-a", Name: "shared"}, suspendSync); err != nil { t.Fatalf("SuspendActor: %v", err) } @@ -879,183 +838,6 @@ func TestSuspendActor_Finalized_ReleasesOnlyOwnWorker(t *testing.T) { } } -// TestPreferredFidelity verifies golden actors always commit Full — new -// actors resume the golden snapshot Full, so the template's preferredFidelity must not -// thin it down to a data-only capture. -func TestPreferredFidelity(t *testing.T) { - tmpl := func(fidelity ateapipb.SnapshotFidelity) *ateapipb.ActorTemplate { - return &ateapipb.ActorTemplate{ - SnapshotConfig: &ateapipb.SnapshotConfig{PreferredFidelity: fidelity}, - } - } - fullScope := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY - dataScope := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES - tests := []struct { - name string - atespace string - fidelity ateapipb.SnapshotFidelity - want ateapipb.SnapshotFidelity - }{ - {"golden actor ignores Data fidelity", resources.GoldenActorAtespace, dataScope, fullScope}, - {"golden actor keeps Full fidelity", resources.GoldenActorAtespace, fullScope, fullScope}, - {"regular actor uses Data fidelity", "team-a", dataScope, dataScope}, - {"regular actor uses Full fidelity", "team-a", fullScope, fullScope}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - if got := preferredFidelity(tc.atespace, tmpl(tc.fidelity)); got != tc.want { - t.Errorf("preferredFidelity(%q, fidelity=%s) = %s, want %s", tc.atespace, tc.fidelity, got, tc.want) - } - }) - } -} - -// TestIsPausedOriginSuspend pins the paused-origin discriminator: a local -// snapshot alone must not select the paused path, because a RUNNING actor -// resumed from pause still carries one; only the paused shape (SUSPENDED with -// a node-local latest snapshot), or SUSPENDING with no worker assignment, -// means the suspend uploads a local snapshot. -func TestIsPausedOriginSuspend(t *testing.T) { - assignment := &ateapipb.WorkerAssignment{WorkerNamespace: "ns", WorkerPool: "pool", WorkerPod: "pod-1"} - localSnaps := []*ateapipb.Snapshot{ - newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snap", "node1"), - } - tests := []struct { - name string - actor *ateapipb.Actor - want bool - }{ - {"paused actor", &ateapipb.Actor{Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, AssignedNode: "node1", LastAssignedGeneration: 1, Snapshots: localSnaps}}, true}, - {"suspended actor without a local snapshot", &ateapipb.Actor{Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}}, false}, - {"suspending retry of a paused-origin suspend", &ateapipb.Actor{Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, AssignedNode: "node1", Snapshots: localSnaps}}, true}, - {"running actor resumed from pause", &ateapipb.Actor{Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_RUNNING, Snapshots: localSnaps, WorkerAssignment: assignment}}, false}, - {"suspending retry of a running-origin suspend after a resume from pause", &ateapipb.Actor{Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, Snapshots: localSnaps, WorkerAssignment: assignment}}, false}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - if got := isPausedOriginSuspend(tc.actor); got != tc.want { - t.Errorf("isPausedOriginSuspend = %t, want %t", got, tc.want) - } - }) - } -} - -// TestSuspendActor_PausedSnapshotUploaded_Preconditions covers the paused -// branch's failure handling: a lost node record crashes the actor (the -// snapshot can never be found), while an unreachable atelet stays retryable -// (the bytes are likely still on the node's disk). Both actors are SUSPENDING -// with no worker, so the mark step skips and the suspend takes the paused -// branch. -func TestSuspendActor_PausedSnapshotUploaded_Preconditions(t *testing.T) { - t.Run("no node recorded crashes", func(t *testing.T) { - ctx := context.Background() - persistence := newTestPersistence(t) - seedSuspendTemplate(t, ctx, persistence, &ateapipb.SnapshotConfig{StorageLocation: "gs://snapshots"}) - w := newMarkOnlyWorkflow(persistence) - actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"} - - storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, - ActorTemplate: suspendTestTemplateRef(), - Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, - LastAssignedGeneration: 1, - Snapshots: []*ateapipb.Snapshot{ - newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snap", "node1"), - }, - }, - }) - - _, err := w.SuspendActor(ctx, actorRef) - if got := apierror.Code(err); got != codes.FailedPrecondition { - t.Fatalf("apierror.Code(err) = %v, want %v for a missing node record (err: %v)", got, codes.FailedPrecondition, err) - } - - stored := mustReadActor(t, ctx, persistence, actorRef) - if stored.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_CRASHED { - t.Errorf("state = %v, want CRASHED", stored.GetStatus().GetState()) - } - if msg, want := stored.GetStatus().GetCrash().GetMessage(), "suspend failed: "+crashMessageLocalSnapshotNodeUnknown; msg != want { - t.Errorf("crash message = %q, want %q", msg, want) - } - }) - - t.Run("no atelet on node stays retryable", func(t *testing.T) { - ctx := context.Background() - persistence := newTestPersistence(t) - seedSuspendTemplate(t, ctx, persistence, &ateapipb.SnapshotConfig{StorageLocation: "gs://snapshots"}) - w := newMarkOnlyWorkflow(persistence) - actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"} - - snap := newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snap", someActorSnapshotURI(t, testStorageLocation, "team-a", "snap-dest"), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) - snap.Locality = "node1" - storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, - ActorTemplate: suspendTestTemplateRef(), - Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, - AssignedNode: "node1", - LastAssignedGeneration: 1, - Snapshots: []*ateapipb.Snapshot{snap}, - }, - }) - - _, err := w.SuspendActor(ctx, actorRef) - if !errors.Is(err, ErrNoAteletOnNode) { - t.Fatalf("SuspendActor = %v, want ErrNoAteletOnNode", err) - } - - stored := mustReadActor(t, ctx, persistence, actorRef) - if stored.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDING { - t.Errorf("state = %v, want SUSPENDING (retryable, not crashed)", stored.GetStatus().GetState()) - } - }) -} - -// TestSuspendActor_PausedSnapshotUploaded_WireRequest verifies the -// PromoteSnapshot request names the paused actor's local snapshot, with its -// generation, and the destination the suspend recorded for it. -func TestSuspendActor_PausedSnapshotUploaded_WireRequest(t *testing.T) { - ctx := context.Background() - persistence := newTestPersistence(t) - seedSuspendTemplate(t, ctx, persistence, &ateapipb.SnapshotConfig{StorageLocation: testStorageLocation, PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY}) - w, atelet := newWireCaptureWorkflow(t, persistence) - - actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"} - destURI := someActorSnapshotURI(t, testStorageLocation, "team-a", "snap-dest") - snap := newDurableSnapshot(3, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "pause-snap", destURI, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) - snap.Locality = "node-1" - // A suspend from paused in progress: SUSPENDING with no worker, so the - // suspend skips its mark step and promotes the local snapshot. - storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, - ActorTemplate: suspendTestTemplateRef(), - Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, - AssignedNode: "node-1", - LastAssignedGeneration: 3, - Snapshots: []*ateapipb.Snapshot{snap}, - }, - }) - - if _, err := w.SuspendActor(ctx, actorRef); err != nil { - t.Fatalf("SuspendActor: %v", err) - } - req := atelet.promoteRequest() - if req == nil { - t.Fatal("expected PromoteSnapshot RPC to be called") - } - want := &ateletpb.Snapshot{ - SnapshotUuid: "pause-snap", - Generation: 3, - Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, - Object: &ateletpb.ObjectStorage{SnapshotUri: destURI}, - } - if got := req.GetSnapshot(); !proto.Equal(got, want) { - t.Errorf("PromoteSnapshotRequest.Snapshot = %v, want %v", got, want) - } -} - // TestSuspendActor_Finalized_MissingDurableSnapshotClearsAssignment verifies that if the // latest generation snapshot entry is missing when the suspend finalizes, the // actor transitions to CRASHED and clears WorkerAssignment after releasing the @@ -1086,7 +868,7 @@ func TestSuspendActor_Finalized_MissingDurableSnapshotClearsAssignment(t *testin s.Snapshots = nil }) - if _, err := w.SuspendActor(ctx, actorRef); apierror.Code(err) != codes.FailedPrecondition { + if _, err := w.SuspendActor(ctx, actorRef, suspendSync); apierror.Code(err) != codes.FailedPrecondition { t.Fatalf("SuspendActor error = %v, want FailedPrecondition for the actor it crashed", err) } got, err := st.GetActor(ctx, actorRef) @@ -1111,11 +893,31 @@ func TestSuspendActor_Finalized_MissingDurableSnapshotClearsAssignment(t *testin // snapshot is durable before it returns, and async, whose snapshot stays // node-local. var suspendModes = []struct { - name string - run func(*ActorWorkflow, context.Context, resources.ActorRef) (*ateapipb.Actor, error) + name string + suspendMode suspendMode }{ - {"sync", (*ActorWorkflow).SuspendActor}, - {"async", (*ActorWorkflow).PauseActor}, + {"sync", suspendSync}, + {"async", suspendAsync}, +} + +func TestSuspendModeFromOptions(t *testing.T) { + tests := []struct { + name string + opts *ateapipb.SuspendOptions + want suspendMode + }{ + {"no options", nil, suspendAsync}, + {"mode unspecified", &ateapipb.SuspendOptions{}, suspendAsync}, + {"synchronous", &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, suspendSync}, + {"asynchronous", &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, suspendAsync}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := suspendModeFromOptions(tt.opts); got != tt.want { + t.Errorf("suspendModeFromOptions(%v) = %v, want %v", tt.opts, got, tt.want) + } + }) + } } // suspendOutcome is what one suspend workflow does to a seeded actor. Rows @@ -1152,9 +954,9 @@ type snapshotOutcome struct { } // TestSuspendAndPause_CharacteristicMatrix pins, for each actor shape either -// workflow can be called on, what SuspendActor (sync) and PauseActor (async) -// do: the code returned, the atelet RPC sent, the record left behind, and -// whether the actor's worker was released. +// mode can be called on, what a sync and an async SuspendActor do: the code +// returned, the atelet RPC sent, the record left behind, and whether the +// actor's worker was released. func TestSuspendAndPause_CharacteristicMatrix(t *testing.T) { const node = "node-1" volumes := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES @@ -1163,19 +965,15 @@ func TestSuspendAndPause_CharacteristicMatrix(t *testing.T) { return newDurableSnapshot(gen, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, volumes, "", name, someActorSnapshotURI(t, testStorageLocation, "team-a", name), status) } local := func(gen int32, locality string) *ateapipb.Snapshot { - return newLocalSnapshot(gen, volumes, "", fmt.Sprintf("snap-%d", gen), locality) + return newPausedSnapshot(t, gen, volumes, "", fmt.Sprintf("snap-%d", gen), locality) } completed := ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED pending := ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING - noDurable := ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED - promoting := durable(1, pending) - promoting.Locality = node ateletVolumes := ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES immediate := ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE checkpointSync := ateletCall{checkpoints: 1, fidelity: ateletVolumes, uploadMode: immediate, object: true} checkpointAsync := ateletCall{checkpoints: 1, fidelity: ateletVolumes, uploadMode: ateletpb.UploadMode_UPLOAD_MODE_DEFERRED, object: true} - promote := ateletCall{promotes: 1, fidelity: ateletVolumes, object: true} // rejected is a FailedPrecondition outcome: no atelet call, and the // record left as want describes it. rejected := func(want suspendOutcome) suspendOutcome { @@ -1195,7 +993,7 @@ func TestSuspendAndPause_CharacteristicMatrix(t *testing.T) { name: "RUNNING", status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_RUNNING, AssignedNode: node, WorkerAssignment: wireTestAssignment()}, sync: suspendOutcome{code: codes.OK, atelet: checkpointSync, state: suspended, generation: 1, snapshots: []snapshotOutcome{{1, volumes, "", completed}}}, - async: suspendOutcome{code: codes.OK, atelet: checkpointAsync, state: suspended, assignedNode: node, generation: 1, snapshots: []snapshotOutcome{{1, volumes, node, noDurable}}}, + async: suspendOutcome{code: codes.OK, atelet: checkpointAsync, state: suspended, assignedNode: node, generation: 1, snapshots: []snapshotOutcome{{1, volumes, node, pending}}}, }, { name: "RUNNING resumed from pause", @@ -1204,7 +1002,7 @@ func TestSuspendAndPause_CharacteristicMatrix(t *testing.T) { LastAssignedGeneration: 1, Snapshots: []*ateapipb.Snapshot{local(1, node)}, }, sync: suspendOutcome{code: codes.OK, atelet: checkpointSync, state: suspended, generation: 2, snapshots: []snapshotOutcome{{2, volumes, "", completed}}}, - async: suspendOutcome{code: codes.OK, atelet: checkpointAsync, state: suspended, assignedNode: node, generation: 2, snapshots: []snapshotOutcome{{2, volumes, node, noDurable}}}, + async: suspendOutcome{code: codes.OK, atelet: checkpointAsync, state: suspended, assignedNode: node, generation: 2, snapshots: []snapshotOutcome{{2, volumes, node, pending}}}, }, { name: "RUNNING resumed from a durable snapshot", @@ -1213,46 +1011,30 @@ func TestSuspendAndPause_CharacteristicMatrix(t *testing.T) { LastAssignedGeneration: 1, Snapshots: []*ateapipb.Snapshot{durable(1, completed)}, }, sync: suspendOutcome{code: codes.OK, atelet: checkpointSync, state: suspended, generation: 2, snapshots: []snapshotOutcome{{2, volumes, "", completed}}}, - async: suspendOutcome{code: codes.OK, atelet: checkpointAsync, state: suspended, assignedNode: node, generation: 2, snapshots: []snapshotOutcome{{1, volumes, "", completed}, {2, volumes, node, noDurable}}}, + async: suspendOutcome{code: codes.OK, atelet: checkpointAsync, state: suspended, assignedNode: node, generation: 2, snapshots: []snapshotOutcome{{1, volumes, "", completed}, {2, volumes, node, pending}}}, }, { name: "SUSPENDED with a durable snapshot", status: &ateapipb.ActorStatus{State: suspended, LastAssignedGeneration: 1, Snapshots: []*ateapipb.Snapshot{durable(1, completed)}}, sync: suspendOutcome{code: codes.OK, state: suspended, generation: 1, snapshots: []snapshotOutcome{{1, volumes, "", completed}}}, - async: rejected(suspendOutcome{state: suspended, generation: 1, snapshots: []snapshotOutcome{{1, volumes, "", completed}}}), + async: suspendOutcome{code: codes.OK, state: suspended, generation: 1, snapshots: []snapshotOutcome{{1, volumes, "", completed}}}, }, { + // Neither mode promotes the node-local snapshot. name: "paused", status: &ateapipb.ActorStatus{State: suspended, AssignedNode: node, LastAssignedGeneration: 1, Snapshots: []*ateapipb.Snapshot{local(1, node)}}, - sync: suspendOutcome{code: codes.OK, atelet: promote, state: suspended, generation: 1, snapshots: []snapshotOutcome{{1, volumes, "", completed}}}, - async: suspendOutcome{code: codes.OK, state: suspended, assignedNode: node, generation: 1, snapshots: []snapshotOutcome{{1, volumes, node, noDurable}}}, + sync: suspendOutcome{code: codes.OK, state: suspended, assignedNode: node, generation: 1, snapshots: []snapshotOutcome{{1, volumes, node, pending}}}, + async: suspendOutcome{code: codes.OK, state: suspended, assignedNode: node, generation: 1, snapshots: []snapshotOutcome{{1, volumes, node, pending}}}, }, { - name: "suspend from RUNNING in progress", + // Both modes mark this shape, and a retry finishes in its own mode. + name: "suspend in progress", status: &ateapipb.ActorStatus{ State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, AssignedNode: node, WorkerAssignment: wireTestAssignment(), LastAssignedGeneration: 1, Snapshots: []*ateapipb.Snapshot{durable(1, pending)}, }, sync: suspendOutcome{code: codes.OK, atelet: checkpointSync, state: suspended, generation: 1, snapshots: []snapshotOutcome{{1, volumes, "", completed}}}, - async: rejected(suspendOutcome{state: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, assignedNode: node, workerHeld: true, generation: 1, snapshots: []snapshotOutcome{{1, volumes, "", pending}}}), - }, - { - name: "suspend from paused in progress", - status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, AssignedNode: node, - LastAssignedGeneration: 1, Snapshots: []*ateapipb.Snapshot{promoting}, - }, - sync: suspendOutcome{code: codes.OK, atelet: promote, state: suspended, generation: 1, snapshots: []snapshotOutcome{{1, volumes, "", completed}}}, - async: rejected(suspendOutcome{state: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, assignedNode: node, generation: 1, snapshots: []snapshotOutcome{{1, volumes, node, pending}}}), - }, - { - name: "pause in progress", - status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, AssignedNode: node, WorkerAssignment: wireTestAssignment(), - LastAssignedGeneration: 1, Snapshots: []*ateapipb.Snapshot{local(1, "")}, - }, - sync: rejected(suspendOutcome{state: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, assignedNode: node, workerHeld: true, generation: 1, snapshots: []snapshotOutcome{{1, volumes, "", noDurable}}}), - async: suspendOutcome{code: codes.OK, atelet: checkpointAsync, state: suspended, assignedNode: node, generation: 1, snapshots: []snapshotOutcome{{1, volumes, node, noDurable}}}, + async: suspendOutcome{code: codes.OK, atelet: checkpointAsync, state: suspended, assignedNode: node, generation: 1, snapshots: []snapshotOutcome{{1, volumes, node, pending}}}, }, { name: "RESUMING", @@ -1272,15 +1054,6 @@ func TestSuspendAndPause_CharacteristicMatrix(t *testing.T) { sync: rejected(suspendOutcome{state: ateapipb.ActorState_ACTOR_STATE_DELETING}), async: rejected(suspendOutcome{state: ateapipb.ActorState_ACTOR_STATE_DELETING}), }, - { - // A golden actor commits MEMORY whatever the template prefers, and - // cannot be paused at all. - name: "golden RUNNING", - atespace: resources.GoldenActorAtespace, - status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_RUNNING, AssignedNode: node, WorkerAssignment: wireTestAssignment()}, - sync: suspendOutcome{code: codes.OK, atelet: ateletCall{checkpoints: 1, fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, uploadMode: immediate, object: true}, state: suspended, generation: 1, snapshots: []snapshotOutcome{{1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", completed}}}, - async: rejected(suspendOutcome{state: ateapipb.ActorState_ACTOR_STATE_RUNNING, assignedNode: node, workerHeld: true}), - }, } for i, tc := range tests { @@ -1313,7 +1086,7 @@ func TestSuspendAndPause_CharacteristicMatrix(t *testing.T) { seedWireWorker(t, ctx, persistence, assignment) w, atelet := newWireCaptureWorkflow(t, persistence) - _, err := mode.run(w, ctx, actorRef) + _, err := w.SuspendActor(ctx, actorRef, mode.suspendMode) stored := mustReadActor(t, ctx, persistence, actorRef) got := suspendOutcome{ code: apierror.Code(err), @@ -1367,7 +1140,7 @@ func TestSuspendAndPause_FinalizeWithoutAssignment(t *testing.T) { s.WorkerAssignment = nil }) - returned, err := mode.run(w, ctx, actorRef) + returned, err := w.SuspendActor(ctx, actorRef, mode.suspendMode) if got := apierror.Code(err); got != codes.FailedPrecondition { t.Fatalf("%s: code = %v (err %v), want FailedPrecondition", mode.name, got, err) } @@ -1406,7 +1179,7 @@ func TestSuspendAndPause_InvalidStorageLocationRejected(t *testing.T) { }, }) - if _, err := mode.run(w, ctx, actorRef); err == nil { + if _, err := w.SuspendActor(ctx, actorRef, mode.suspendMode); err == nil { t.Fatalf("%s: expected error for invalid storage location, got nil", mode.name) } stored := mustReadActor(t, ctx, persistence, actorRef) @@ -1480,3 +1253,160 @@ func assertRequestNamesLatestSnapshot(t *testing.T, atelet *capturingAtelet, tmp t.Errorf("request Snapshot.Object.SnapshotUri = %q, want %q", got, wantURI) } } + +// TestSuspendActor_AppliesCheckpointState verifies finalize applies the +// snapshot state atelet returns from Checkpoint, with the report rules. A state +// that conflicts with the record fails the suspend, and leaves the actor +// SUSPENDING with its worker. +func TestSuspendActor_AppliesCheckpointState(t *testing.T) { + const node = "node-1" + memory := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY + pending := ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + completed := ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + durable := func(gen int32, status ateapipb.SnapshotStorageStatus) *ateapipb.Snapshot { + name := fmt.Sprintf("snap-%d", gen) + return newDurableSnapshot(gen, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, memory, "", name, someActorSnapshotURI(t, testStorageLocation, "team-a", name), status) + } + report := func(gen int32, uri string, fidelity ateletpb.SnapshotFidelity) *ateletpb.SnapshotReport { + return &ateletpb.SnapshotReport{Snapshot: &ateletpb.Snapshot{ + SnapshotUuid: fmt.Sprintf("snap-%d", gen), + Generation: gen, + Fidelity: fidelity, + Object: &ateletpb.ObjectStorage{SnapshotUri: uri}, + }} + } + uriOf := func(gen int32) string { + return someActorSnapshotURI(t, testStorageLocation, "team-a", fmt.Sprintf("snap-%d", gen)) + } + + tests := []struct { + name string + mode suspendMode + // status is the RUNNING actor's status before the suspend. + status *ateapipb.ActorStatus + // state builds what Checkpoint returns, from the request it got. + state func(req *ateletpb.CheckpointRequest) *ateletpb.ActorSnapshotState + wantSnapshots []snapshotOutcome + // wantCode, when set, is the error the suspend fails with. + wantCode codes.Code + }{{ + // Async suspend can't reach this shape. It starts only from RUNNING, and + // a sync suspend in progress must be reverted before the actor runs + // again. The case pins that finalize applies the state before it prunes. + name: "async: an older upload atelet finished is kept as COMPLETED", + mode: suspendAsync, + status: &ateapipb.ActorStatus{LastAssignedGeneration: 1, Snapshots: []*ateapipb.Snapshot{durable(1, pending)}}, + state: func(req *ateletpb.CheckpointRequest) *ateletpb.ActorSnapshotState { + return &ateletpb.ActorSnapshotState{ + LatestLocal: &ateletpb.SnapshotReport{Snapshot: req.GetSnapshot()}, + LatestDurable: report(1, uriOf(1), ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY), + } + }, + wantSnapshots: []snapshotOutcome{{generation: 1, fidelity: memory, durable: completed}, {generation: 2, fidelity: memory, locality: node, durable: pending}}, + }, { + // A RUNNING actor resumed from a pause keeps that pause's snapshot, + // with its PENDING durable copy. atelet uploads it in the background, + // and the next pause records that. + name: "async: the previous pause's upload atelet finished is recorded as COMPLETED", + mode: suspendAsync, + status: &ateapipb.ActorStatus{LastAssignedGeneration: 1, Snapshots: []*ateapipb.Snapshot{newPausedSnapshot(t, 1, memory, "", "snap-1", node)}}, + state: func(req *ateletpb.CheckpointRequest) *ateletpb.ActorSnapshotState { + return &ateletpb.ActorSnapshotState{ + LatestLocal: &ateletpb.SnapshotReport{Snapshot: req.GetSnapshot()}, + LatestDurable: report(1, uriOf(1), ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY), + } + }, + wantSnapshots: []snapshotOutcome{{generation: 1, fidelity: memory, durable: completed}, {generation: 2, fidelity: memory, locality: node, durable: pending}}, + }, { + name: "async: without a state, the PENDING upload is pruned", + mode: suspendAsync, + status: &ateapipb.ActorStatus{LastAssignedGeneration: 1, Snapshots: []*ateapipb.Snapshot{durable(1, pending)}}, + state: func(*ateletpb.CheckpointRequest) *ateletpb.ActorSnapshotState { + return nil + }, + wantSnapshots: []snapshotOutcome{{generation: 2, fidelity: memory, locality: node, durable: pending}}, + }, { + name: "sync: the fidelity atelet reports is recorded", + mode: suspendSync, + state: func(req *ateletpb.CheckpointRequest) *ateletpb.ActorSnapshotState { + snap := proto.CloneOf(req.GetSnapshot()) + snap.Fidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS + return &ateletpb.ActorSnapshotState{LatestDurable: &ateletpb.SnapshotReport{Snapshot: snap}} + }, + wantSnapshots: []snapshotOutcome{{generation: 1, fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, durable: completed}}, + }, { + name: "sync: a state that conflicts with the record fails the suspend", + mode: suspendSync, + state: func(req *ateletpb.CheckpointRequest) *ateletpb.ActorSnapshotState { + snap := proto.CloneOf(req.GetSnapshot()) + snap.Fidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS + snap.Object = &ateletpb.ObjectStorage{SnapshotUri: uriOf(9)} + return &ateletpb.ActorSnapshotState{LatestDurable: &ateletpb.SnapshotReport{Snapshot: snap}} + }, + wantCode: codes.InvalidArgument, + }, { + name: "async: a state that conflicts with the record fails the suspend", + mode: suspendAsync, + status: &ateapipb.ActorStatus{LastAssignedGeneration: 1, Snapshots: []*ateapipb.Snapshot{durable(1, pending)}}, + state: func(req *ateletpb.CheckpointRequest) *ateletpb.ActorSnapshotState { + return &ateletpb.ActorSnapshotState{ + LatestLocal: &ateletpb.SnapshotReport{Snapshot: req.GetSnapshot()}, + LatestDurable: report(1, uriOf(9), ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY), + } + }, + wantCode: codes.InvalidArgument, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + persistence := newTestPersistence(t) + seedSuspendTemplate(t, ctx, persistence, &ateapipb.SnapshotConfig{StorageLocation: testStorageLocation, PreferredFidelity: memory}) + actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"} + status := proto.CloneOf(tt.status) + if status == nil { + status = &ateapipb.ActorStatus{} + } + status.State = ateapipb.ActorState_ACTOR_STATE_RUNNING + status.AssignedNode = node + status.WorkerAssignment = wireTestAssignment() + created := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, + ActorTemplate: suspendTestTemplateRef(), + Status: status, + }) + seedWireWorker(t, ctx, persistence, assignmentOf(created)) + w, atelet := newWireCaptureWorkflow(t, persistence) + atelet.onCheckpoint = func(context.Context) error { + st := tt.state(atelet.checkpointRequest()) + atelet.mu.Lock() + atelet.checkpointState = st + atelet.mu.Unlock() + return nil + } + + _, err := w.SuspendActor(ctx, actorRef, tt.mode) + stored := mustReadActor(t, ctx, persistence, actorRef) + if tt.wantCode != codes.OK { + if got := apierror.Code(err); got != tt.wantCode { + t.Fatalf("suspend = %v, want code %v", err, tt.wantCode) + } + if got := stored.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDING { + t.Errorf("state = %v, want SUSPENDING", got) + } + if firstAssignment(t, persistence, wireTestAssignment().GetWorker().GetName()) == nil { + t.Error("worker released, want it still holding the actor so a retry can call atelet") + } + return + } + if err != nil { + t.Fatalf("suspend: %v", err) + } + if got := stored.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { + t.Errorf("state = %v, want SUSPENDED", got) + } + if diff := cmp.Diff(tt.wantSnapshots, describeSnapshots(stored.GetStatus()), cmp.AllowUnexported(snapshotOutcome{})); diff != "" { + t.Errorf("snapshots mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/cmd/ateapi/internal/controlapi/workflow_tag.go b/cmd/ateapi/internal/controlapi/workflow_tag.go index 690d52bc84..e880e9c090 100644 --- a/cmd/ateapi/internal/controlapi/workflow_tag.go +++ b/cmd/ateapi/internal/controlapi/workflow_tag.go @@ -223,8 +223,9 @@ func (w *ActorWorkflow) loadActorForTag(ctx context.Context, actorRef resources. return nil, nil, apierror.FailedPrecondition("Actor %s holds no external snapshot to tag", actorRef) } // The actor's latest snapshot is local only. It must be uploaded before it can be tagged. + // TODO: Wait for atelet's background upload, or upload it here, instead of failing. if latestSnapshotIsLocalOnly(actor.GetStatus()) { - return nil, nil, apierror.FailedPrecondition("Actor %s holds no external snapshot of its latest state to tag: suspend it first", actorRef) + return nil, nil, apierror.FailedPrecondition("Actor %s holds no external snapshot of its latest state to tag: its latest snapshot is not uploaded yet", actorRef) } // Every way an Actor comes to hold an external snapshot records the // template its guest state was built under: a suspend through diff --git a/cmd/ateapi/internal/controlapi/workflow_tag_test.go b/cmd/ateapi/internal/controlapi/workflow_tag_test.go index 425bd24bd4..25e0d301af 100644 --- a/cmd/ateapi/internal/controlapi/workflow_tag_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_tag_test.go @@ -225,7 +225,7 @@ func TestTagActorSnapshot_Preconditions(t *testing.T) { } if tt.paused { s.LastAssignedGeneration = 2 - s.Snapshots = append(s.Snapshots, newLocalSnapshot(2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmplUID, "paused", "node-1")) + s.Snapshots = append(s.Snapshots, newPausedSnapshot(t, 2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmplUID, "paused", "node-1")) } }) } diff --git a/cmd/ateapi/internal/controlapi/workflow_template_delete.go b/cmd/ateapi/internal/controlapi/workflow_template_delete.go index 553ee0585a..acdb37b0aa 100644 --- a/cmd/ateapi/internal/controlapi/workflow_template_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_template_delete.go @@ -53,12 +53,12 @@ func (w *ActorWorkflow) DeleteActorTemplate(ctx context.Context, templateRef res return nil, apierror.Aborted("concurrent update conflict, please retry") } - // Both are named after the template's uid, in the reserved atespace. + // Both are named after the template's UID, in the template's atespace. goldenName := tmpl.GetMetadata().GetUid() - if err := w.ensureGoldenActorDeleted(ctx, resources.ActorRef{Atespace: resources.GoldenActorAtespace, Name: goldenName}); err != nil { + if err := w.ensureGoldenActorDeleted(ctx, resources.ActorRef{Atespace: templateRef.Atespace, Name: goldenName}); err != nil { return nil, err } - if err := w.ensureGoldenTagDeleted(ctx, resources.TagRef{Atespace: resources.GoldenActorAtespace, Name: goldenName}); err != nil { + if err := w.ensureGoldenTagDeleted(ctx, resources.TagRef{Atespace: templateRef.Atespace, Name: goldenName}); err != nil { return nil, err } return w.finalizeTemplateDeleted(ctx, templateRef, precondition) diff --git a/cmd/ateapi/internal/controlapi/workflow_test.go b/cmd/ateapi/internal/controlapi/workflow_test.go index b40faa7a06..87a09d79fd 100644 --- a/cmd/ateapi/internal/controlapi/workflow_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_test.go @@ -49,7 +49,7 @@ func TestActorStateChangeRecords(t *testing.T) { name: "suspend marks the actor suspending", seedState: ateapipb.ActorState_ACTOR_STATE_RUNNING, transition: func(t *testing.T, w *ActorWorkflow, ref resources.ActorRef, actor *ateapipb.Actor, _ *ateapipb.ActorTemplate) { - markOnly(t, w, ref, actor, w.SuspendActor) + markOnly(t, w, ref, actor, suspendSync) }, wantOp: ateattr.OperationSuspend, wantState: ateattr.ActorStateSuspending, @@ -59,7 +59,7 @@ func TestActorStateChangeRecords(t *testing.T) { name: "pause marks the actor suspending", seedState: ateapipb.ActorState_ACTOR_STATE_RUNNING, transition: func(t *testing.T, w *ActorWorkflow, ref resources.ActorRef, actor *ateapipb.Actor, _ *ateapipb.ActorTemplate) { - markOnly(t, w, ref, actor, w.PauseActor) + markOnly(t, w, ref, actor, suspendAsync) }, wantOp: ateattr.OperationSuspend, wantState: ateattr.ActorStateSuspending, @@ -69,15 +69,15 @@ func TestActorStateChangeRecords(t *testing.T) { name: "finalize pause marks the actor suspended", seedState: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, seedOpts: []func(*ateapipb.Actor){ - withPauseInProgress(), + withPauseInProgress(t), func(a *ateapipb.Actor) { a.Status.AssignedNode = "node-1" a.Status.WorkerAssignment = wireTestAssignment() }, }, transition: func(t *testing.T, w *ActorWorkflow, ref resources.ActorRef, _ *ateapipb.Actor, _ *ateapipb.ActorTemplate) { - if _, err := w.PauseActor(context.Background(), ref); err != nil { - t.Fatalf("PauseActor: %v", err) + if _, err := w.SuspendActor(context.Background(), ref, suspendAsync); err != nil { + t.Fatalf("SuspendActor: %v", err) } }, wantOp: ateattr.OperationSuspend, @@ -171,7 +171,7 @@ func TestActorStateChangeRecords(t *testing.T) { // markOnly runs a suspend or pause on a RUNNING actor only as far as its mark // step: the actor gets a worker to checkpoint on, but the atelet dial fails, // so the SUSPENDING mark is the only transition the workflow commits. -func markOnly(t *testing.T, w *ActorWorkflow, ref resources.ActorRef, actor *ateapipb.Actor, run func(context.Context, resources.ActorRef) (*ateapipb.Actor, error)) { +func markOnly(t *testing.T, w *ActorWorkflow, ref resources.ActorRef, actor *ateapipb.Actor, mode suspendMode) { t.Helper() ctx := context.Background() if _, err := w.store.UpdateActor(ctx, ref, store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error { @@ -181,7 +181,7 @@ func markOnly(t *testing.T, w *ActorWorkflow, ref resources.ActorRef, actor *ate t.Fatalf("UpdateActor: %v", err) } w.dialer = newDanglingDialer() - _, err := run(ctx, ref) + _, err := w.SuspendActor(ctx, ref, mode) assertStoppedAtDial(t, err) } diff --git a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go index 000565e178..4e35ff9891 100644 --- a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go @@ -55,14 +55,10 @@ func newTestActorWorkflow(t *testing.T, st store.Interface, tmplAtespace, tmplNa SnapshotConfig: &ateapipb.SnapshotConfig{ StorageLocation: "gs://snapshots", }, - SandboxConfig: &ateapipb.SandboxConfig{ - SandboxClass: "gvisor", - ConfigName: "gvisor", - }, }); err != nil && !errors.Is(err, store.ErrAlreadyExists) { t.Fatalf("create test ActorTemplate: %v", err) } - return NewActorWorkflow(st, nil, nil, nil, nil, nil, "", nil, time.Minute, objectstoreplugintest.ControlClient(objectstoretest.New())) + return NewActorWorkflow(st, nil, nil, nil, nil, nil, nil, "", nil, time.Minute, objectstoreplugintest.ControlClient(objectstoretest.New())) } // newFinalizeWorkflow builds an ActorWorkflow over persistence with an @@ -163,27 +159,57 @@ func seedWorkflowActor(t *testing.T, ctx context.Context, st store.Interface, ac } // withPausedShape seeds the shape a pause leaves behind: a node-local -// snapshot at the latest generation, pinned to node. Pair it with -// ACTOR_STATE_SUSPENDED to seed a paused actor. -func withPausedShape(node string) func(*ateapipb.Actor) { +// snapshot at the latest generation with a PENDING durable copy, pinned to +// node. Pair it with ACTOR_STATE_SUSPENDED to seed a paused actor. +func withPausedShape(t *testing.T, node string) func(*ateapipb.Actor) { + t.Helper() + snap := newPausedSnapshot(t, 1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "pause-snap", node) return func(a *ateapipb.Actor) { a.Status.AssignedNode = node a.Status.LastAssignedGeneration = 1 - a.Status.Snapshots = []*ateapipb.Snapshot{ - newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "pause-snap", node), - } + a.Status.Snapshots = []*ateapipb.Snapshot{proto.CloneOf(snap)} + } +} + +// withUploadedPausedShape seeds a paused actor whose snapshot atelet has since +// uploaded: the paused shape, with its durable copy COMPLETED. +func withUploadedPausedShape(t *testing.T, node string) func(*ateapipb.Actor) { + t.Helper() + paused := withPausedShape(t, node) + return func(a *ateapipb.Actor) { + paused(a) + a.Status.Snapshots[0].DurableSnapshot.Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED } } -// withPauseInProgress seeds the shape a pause has between marking the actor -// SUSPENDING and finalizing it: a node-local snapshot at the latest generation -// that has no locality yet. Pair it with ACTOR_STATE_SUSPENDING. -func withPauseInProgress() func(*ateapipb.Actor) { +// newInProgressSnapshot builds the snapshot a suspend of either mode marks at +// generation gen: a PENDING durable copy at a URI named uuid, and no locality +// yet. +func newInProgressSnapshot(t *testing.T, gen int32, fidelity ateapipb.SnapshotFidelity, uuid string) *ateapipb.Snapshot { + t.Helper() + return newDurableSnapshot(gen, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, fidelity, "", uuid, someActorSnapshotURI(t, testStorageLocation, "team-a", uuid), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) +} + +// newPausedSnapshot builds the snapshot a pause leaves on node locality at +// generation gen: a node-local checkpoint whose PENDING durable copy atelet +// uploads in the background. With no locality it is the in-progress mark. +func newPausedSnapshot(t *testing.T, gen int32, fidelity ateapipb.SnapshotFidelity, templateUID, uuid, locality string) *ateapipb.Snapshot { + t.Helper() + snap := newInProgressSnapshot(t, gen, fidelity, uuid) + snap.ActorTemplateUid = templateUID + snap.Locality = locality + return snap +} + +// withPauseInProgress seeds the shape a suspend of either mode has between +// marking the actor SUSPENDING and finalizing it: a snapshot at the latest +// generation with a PENDING durable copy and no locality yet. Pair it with +// ACTOR_STATE_SUSPENDING. +func withPauseInProgress(t *testing.T) func(*ateapipb.Actor) { + snap := newInProgressSnapshot(t, 1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "pause-snap") return func(a *ateapipb.Actor) { a.Status.LastAssignedGeneration = 1 - a.Status.Snapshots = []*ateapipb.Snapshot{ - newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "pause-snap", ""), - } + a.Status.Snapshots = []*ateapipb.Snapshot{proto.CloneOf(snap)} } } @@ -245,10 +271,6 @@ func seedSuspendTemplate(t *testing.T, ctx context.Context, st store.Interface, created, err := st.CreateActorTemplate(ctx, &ateapipb.ActorTemplate{ Metadata: &ateapipb.ResourceMetadata{Atespace: ref.GetAtespace(), Name: ref.GetName()}, SnapshotConfig: snapshotConfig, - SandboxConfig: &ateapipb.SandboxConfig{ - SandboxClass: "gvisor", - ConfigName: "gvisor", - }, }) if err != nil { t.Fatalf("CreateActorTemplate: %v", err) diff --git a/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go index 0bd5037174..9db03ecf27 100644 --- a/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go @@ -107,13 +107,10 @@ func TestDeleteWorkerWorkflow_ReleasesBoundActor(t *testing.T) { wf, persistence := newWorkerDeleteWorkflow(t) seedAPIWorker(t, ctx, persistence, validWorker(apiWorkerName)) actor := seedAPIActor(t, ctx, persistence, ateapipb.ActorState_ACTOR_STATE_RUNNING, func(a *ateapipb.Actor) { - // Both in-progress checkpoints are set so the assertion covers the - // shared crash path, which cannot know which workflow was in flight. - a.Status.LastAssignedGeneration = 3 + a.Status.LastAssignedGeneration = 2 a.Status.Snapshots = []*ateapipb.Snapshot{ newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "last", someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "last"), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), - newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "partial-snapshot", someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "partial-snapshot"), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), - newLocalSnapshot(3, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "partial-local-snapshot", ""), + newInProgressSnapshot(t, 2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "partial-snapshot"), } }) assignAPIWorker(t, ctx, persistence, apiWorkerName, actor.GetMetadata().GetUid()) @@ -135,13 +132,8 @@ func TestDeleteWorkerWorkflow_ReleasesBoundActor(t *testing.T) { if got.GetStatus().GetWorkerAssignment() != nil { t.Errorf("actor worker assignment = %v, want it cleared", got.GetStatus().GetWorkerAssignment()) } - // In-progress checkpoints are kept so delete or revert can clean them up - // while assigned_node or the object prefix is still recorded. - if gotSnap := snapshotAtLatestGeneration(got.GetStatus()); gotSnap.GetUuid() != "partial-local-snapshot" { - t.Errorf("in-progress local checkpoint = %q, want %q preserved", gotSnap.GetUuid(), "partial-local-snapshot") - } - // The durable one is kept: it names the prefix whatever atelet already - // uploaded lives under, which delete or revert needs to collect it. + // The in-progress checkpoint is kept: it names the prefix whatever atelet + // already uploaded lives under, which delete or revert needs to collect it. gotInProgressSnap := findLatestDurableSnapshot(got.GetStatus(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) if want := someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "partial-snapshot"); gotInProgressSnap.GetDurableSnapshot().GetObject().GetSnapshotUri() != want { t.Errorf("in-progress external checkpoint not preserved: %v", got.GetStatus()) diff --git a/cmd/ateapi/internal/controlapi/workload_spec_test.go b/cmd/ateapi/internal/controlapi/workload_spec_test.go index 35032f5c2f..925a614201 100644 --- a/cmd/ateapi/internal/controlapi/workload_spec_test.go +++ b/cmd/ateapi/internal/controlapi/workload_spec_test.go @@ -607,8 +607,7 @@ func TestToAteletResources(t *testing.T) { // ate-api-server. func TestWorkloadSpecFromActorTemplatePropagatesResources(t *testing.T) { got, err := workloadSpecFromActorTemplate(&ateapipb.ActorTemplate{ - Metadata: &ateapipb.ResourceMetadata{Atespace: "agent-ns", Name: "tmpl-limits"}, - SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "microvm"}, + Metadata: &ateapipb.ResourceMetadata{Atespace: "agent-ns", Name: "tmpl-limits"}, Containers: []*ateapipb.Container{ { Name: "limited", diff --git a/cmd/ateapi/internal/scheduling/scheduling.go b/cmd/ateapi/internal/scheduling/scheduling.go index 070d2bceb2..3bee872b17 100644 --- a/cmd/ateapi/internal/scheduling/scheduling.go +++ b/cmd/ateapi/internal/scheduling/scheduling.go @@ -30,8 +30,12 @@ import ( // Constraints describes what a worker must satisfy to host an actor. type Constraints struct { - // SandboxClass must equal the worker's sandbox class. Snapshots are not - // portable across sandbox classes, so this is never relaxed. + // SandboxClass, when set, is the sandbox class of the snapshot the actor + // restores from, and the worker's must equal it. A snapshot's files, its + // durable volumes included, are written by one class's worker and read + // back by the same, so unlike RequiredRuntime this holds for every + // fidelity and is never relaxed. Empty for an actor with no snapshot, + // which boots on whichever class its selectors pick. SandboxClass string // RequiredRuntime, when it names a runtime, restricts placement to @@ -221,7 +225,10 @@ func quantitiesUtilization(capQ, usedQ resources.Quantities) float64 { } func (s *scheduler) Applies(worker *ateapipb.Worker, constraints Constraints) bool { - if worker.GetSandboxClass() != constraints.SandboxClass { + if worker.GetSandboxClass() == "" { + return false + } + if constraints.SandboxClass != "" && worker.GetSandboxClass() != constraints.SandboxClass { return false } diff --git a/cmd/ateapi/internal/scheduling/scheduling_test.go b/cmd/ateapi/internal/scheduling/scheduling_test.go index d94e50d5f9..c42920eced 100644 --- a/cmd/ateapi/internal/scheduling/scheduling_test.go +++ b/cmd/ateapi/internal/scheduling/scheduling_test.go @@ -44,15 +44,15 @@ func TestSchedule(t *testing.T) { worker("w-busy-2", "gvisor", "node-a", tierTwo, assigned("demo", "other-2")), worker("w-free", "gvisor", "node-a", tierTwo), }, - constraints: Constraints{SandboxClass: "gvisor"}, + constraints: Constraints{}, wantPod: "w-free", }, { - name: "sandbox class is a hard constraint", + name: "worker with no sandbox class is never scheduled", fleet: fleet{ - worker("w-vm", "microvm", "node-a", tierTwo), + worker("w-unclassed", "", "node-a", tierTwo), }, - constraints: Constraints{SandboxClass: "gvisor"}, + constraints: Constraints{}, }, { name: "template selector filters", @@ -60,7 +60,7 @@ func TestSchedule(t *testing.T) { worker("w-1", "gvisor", "node-a", tierTwo), worker("w-2", "gvisor", "node-a", tierOne), }, - constraints: Constraints{SandboxClass: "gvisor", + constraints: Constraints{ TemplateSelector: labels.SelectorFromSet(labels.Set{"tier": "1"})}, wantPod: "w-2", }, @@ -70,7 +70,7 @@ func TestSchedule(t *testing.T) { worker("w-1", "gvisor", "node-a", map[string]string{"tier": "1", "worload": "a"}), worker("w-2", "gvisor", "node-b", map[string]string{"tier": "1", "workload": "b"}), }, - constraints: Constraints{SandboxClass: "gvisor", + constraints: Constraints{ TemplateSelector: labels.SelectorFromSet(labels.Set{"tier": "1"}), ActorSelector: labels.SelectorFromSet(labels.Set{"workload": "b"})}, wantPod: "w-2", @@ -81,7 +81,7 @@ func TestSchedule(t *testing.T) { worker("w-a", "gvisor", "node-a", tierTwo), worker("w-b", "gvisor", "node-b", tierTwo), }, - constraints: Constraints{SandboxClass: "gvisor", RequiredNodes: []string{"node-b"}}, + constraints: Constraints{RequiredNodes: []string{"node-b"}}, wantPod: "w-b", }, { @@ -89,7 +89,7 @@ func TestSchedule(t *testing.T) { fleet: fleet{ worker("w-1", "gvisor", "node-a", nil), }, - constraints: Constraints{SandboxClass: "gvisor"}, + constraints: Constraints{}, wantPod: "w-1", }, { @@ -97,21 +97,21 @@ func TestSchedule(t *testing.T) { fleet: fleet{ worker("w-busy", "gvisor", "node-a", tierTwo, assigned("demo", "a")), }, - constraints: Constraints{SandboxClass: "gvisor"}, + constraints: Constraints{}, }, { name: "draining workers never scheduled", fleet: fleet{ worker("w-draining", "gvisor", "node-a", tierTwo, withState(ateapipb.WorkerState_WORKER_STATE_DRAINING)), }, - constraints: Constraints{SandboxClass: "gvisor"}, + constraints: Constraints{}, }, { name: "unspecified workers never scheduled", fleet: fleet{ worker("w-unspecified", "gvisor", "node-a", tierTwo, withState(ateapipb.WorkerState_WORKER_STATE_UNSPECIFIED)), }, - constraints: Constraints{SandboxClass: "gvisor"}, + constraints: Constraints{}, }, { name: "picks active worker over draining one", @@ -119,7 +119,7 @@ func TestSchedule(t *testing.T) { worker("w-draining", "gvisor", "node-a", tierTwo, withState(ateapipb.WorkerState_WORKER_STATE_DRAINING)), worker("w-active", "gvisor", "node-a", tierTwo), }, - constraints: Constraints{SandboxClass: "gvisor"}, + constraints: Constraints{}, wantPod: "w-active", }, { @@ -128,7 +128,7 @@ func TestSchedule(t *testing.T) { worker("w-small", "gvisor", "node-a", tierTwo, withCapacity(1000, 8<<30)), worker("w-big", "gvisor", "node-a", tierTwo, withCapacity(4000, 8<<30)), }, - constraints: Constraints{SandboxClass: "gvisor", Limits: resources.CPUMemory(2000, 0)}, + constraints: Constraints{Limits: resources.CPUMemory(2000, 0)}, wantPod: "w-big", }, { @@ -137,7 +137,7 @@ func TestSchedule(t *testing.T) { worker("w-small", "gvisor", "node-a", tierTwo, withCapacity(4000, 1<<30)), worker("w-big", "gvisor", "node-a", tierTwo, withCapacity(4000, 4<<30)), }, - constraints: Constraints{SandboxClass: "gvisor", Limits: resources.CPUMemory(0, 2<<30)}, + constraints: Constraints{Limits: resources.CPUMemory(0, 2<<30)}, wantPod: "w-big", }, { @@ -145,7 +145,7 @@ func TestSchedule(t *testing.T) { fleet: fleet{ worker("w-small", "gvisor", "node-a", tierTwo, withCapacity(1000, 1<<30)), }, - constraints: Constraints{SandboxClass: "gvisor", Limits: resources.CPUMemory(2000, 0)}, + constraints: Constraints{Limits: resources.CPUMemory(2000, 0)}, }, { // A Worker reports everything it has, so one that has reported no @@ -154,7 +154,7 @@ func TestSchedule(t *testing.T) { fleet: fleet{ worker("w-unknown", "gvisor", "node-a", tierTwo), }, - constraints: Constraints{SandboxClass: "gvisor", Limits: resources.CPUMemory(2000, 2<<30)}, + constraints: Constraints{Limits: resources.CPUMemory(2000, 2<<30)}, }, { // An Actor that declares nothing still fits: it asks for no @@ -163,7 +163,7 @@ func TestSchedule(t *testing.T) { fleet: fleet{ worker("w-unknown", "gvisor", "node-a", tierTwo), }, - constraints: Constraints{SandboxClass: "gvisor"}, + constraints: Constraints{}, wantPod: "w-unknown", }, { @@ -171,13 +171,13 @@ func TestSchedule(t *testing.T) { fleet: fleet{ worker("w-tiny", "gvisor", "node-a", tierTwo, withCapacity(100, 1<<20)), }, - constraints: Constraints{SandboxClass: "gvisor"}, + constraints: Constraints{}, wantPod: "w-tiny", }, { name: "empty fleet", fleet: fleet{}, - constraints: Constraints{SandboxClass: "gvisor"}, + constraints: Constraints{}, }, { // A worker that has not said it can hold more admits one, so this is @@ -186,14 +186,14 @@ func TestSchedule(t *testing.T) { fleet: fleet{ worker("w-busy", "gvisor", "node-a", tierTwo, assigned("demo", "other")), }, - constraints: Constraints{SandboxClass: "gvisor"}, + constraints: Constraints{}, }, { name: "a worker below its actor ceiling still has room", fleet: fleet{ worker("w-two", "gvisor", "node-a", tierTwo, withMaxActors(2), assigned("demo", "other")), }, - constraints: Constraints{SandboxClass: "gvisor"}, + constraints: Constraints{}, wantPod: "w-two", }, { @@ -202,7 +202,7 @@ func TestSchedule(t *testing.T) { worker("w-two", "gvisor", "node-a", tierTwo, withMaxActors(2), assigned("demo", "a"), assigned("demo", "b")), }, - constraints: Constraints{SandboxClass: "gvisor"}, + constraints: Constraints{}, }, { // Placement is against what is left, not against the whole capacity: @@ -212,7 +212,7 @@ func TestSchedule(t *testing.T) { worker("w-half", "gvisor", "node-a", tierTwo, withCapacity(4000, 8<<30), withMaxActors(4), assignedFor("demo", "other", resources.CPUMemory(3000, 4<<30))), }, - constraints: Constraints{SandboxClass: "gvisor", Limits: resources.CPUMemory(2000, 0)}, + constraints: Constraints{Limits: resources.CPUMemory(2000, 0)}, }, { name: "what the residents left over is still placeable", @@ -220,7 +220,7 @@ func TestSchedule(t *testing.T) { worker("w-half", "gvisor", "node-a", tierTwo, withCapacity(4000, 8<<30), withMaxActors(4), assignedFor("demo", "other", resources.CPUMemory(3000, 4<<30))), }, - constraints: Constraints{SandboxClass: "gvisor", Limits: resources.CPUMemory(1000, 4<<30)}, + constraints: Constraints{Limits: resources.CPUMemory(1000, 4<<30)}, wantPod: "w-half", }, { @@ -229,7 +229,7 @@ func TestSchedule(t *testing.T) { worker("w-busy", "gvisor", "node-a", tierTwo, withMaxActors(4), assigned("demo", "a"), assigned("demo", "b")), worker("w-idle", "gvisor", "node-b", tierTwo, withMaxActors(4)), }, - constraints: Constraints{SandboxClass: "gvisor"}, + constraints: Constraints{}, wantPod: "w-idle", }, { @@ -238,7 +238,7 @@ func TestSchedule(t *testing.T) { worker("w-idle", "gvisor", "node-a", tierTwo, withMaxActors(4)), worker("w-busy", "gvisor", "node-b", tierTwo, withMaxActors(4), assigned("demo", "a")), }, - constraints: Constraints{SandboxClass: "gvisor"}, + constraints: Constraints{}, wantPod: "w-idle", }, { @@ -248,7 +248,7 @@ func TestSchedule(t *testing.T) { worker("w-small", "gvisor", "node-a", tierTwo, withMaxActors(2), assigned("demo", "a")), worker("w-large", "gvisor", "node-b", tierTwo, withMaxActors(16), assigned("demo", "b"), assigned("demo", "c")), }, - constraints: Constraints{SandboxClass: "gvisor"}, + constraints: Constraints{}, wantPod: "w-large", }, { @@ -259,7 +259,7 @@ func TestSchedule(t *testing.T) { worker("w-light", "gvisor", "node-b", tierTwo, withCapacity(4000, 8<<30), withMaxActors(4), assignedFor("demo", "b", resources.CPUMemory(1000, 1<<30))), }, - constraints: Constraints{SandboxClass: "gvisor", Limits: resources.CPUMemory(500, 1<<30)}, + constraints: Constraints{Limits: resources.CPUMemory(500, 1<<30)}, wantPod: "w-light", }, { @@ -275,7 +275,7 @@ func TestSchedule(t *testing.T) { assigned("demo", "b2"), assigned("demo", "b3"), assigned("demo", "b4"), assigned("demo", "b5"), assigned("demo", "b6")), }, - constraints: Constraints{SandboxClass: "gvisor", Limits: resources.CPUMemory(100, 512<<20)}, + constraints: Constraints{Limits: resources.CPUMemory(100, 512<<20)}, wantPod: "w-cool", }, { @@ -285,7 +285,7 @@ func TestSchedule(t *testing.T) { worker("w-known", "gvisor", "node-b", tierTwo, withCapacity(4000, 8<<30), withMaxActors(4), assignedFor("demo", "b", resources.CPUMemory(1000, 2<<30))), }, - constraints: Constraints{SandboxClass: "gvisor"}, + constraints: Constraints{}, wantPod: "w-known", }, { @@ -301,7 +301,7 @@ func TestSchedule(t *testing.T) { worker("w-known", "gvisor", "node-b", tierTwo, withCapacity(4000, 8<<30), withMaxActors(4), assignedFor("demo", "b", resources.CPUMemory(1000, 2<<30))), }, - constraints: Constraints{SandboxClass: "gvisor"}, + constraints: Constraints{}, wantPod: "w-known", }, { @@ -313,7 +313,7 @@ func TestSchedule(t *testing.T) { worker("w-large", "gvisor", "node-b", tierTwo, withMaxActors(8), assigned("demo", "b"), assigned("demo", "c"), assigned("demo", "d"), assigned("demo", "e")), }, - constraints: Constraints{SandboxClass: "gvisor"}, + constraints: Constraints{}, wantPod: "w-large", }, { @@ -323,7 +323,7 @@ func TestSchedule(t *testing.T) { worker("w-busy-big", "gvisor", "node-b", tierTwo, withCapacity(4000, 8<<30), withMaxActors(4), assignedFor("demo", "a", resources.CPUMemory(1000, 2<<30))), }, - constraints: Constraints{SandboxClass: "gvisor", Limits: resources.CPUMemory(1000, 2<<30)}, + constraints: Constraints{Limits: resources.CPUMemory(1000, 2<<30)}, wantPod: "w-busy-big", }, { @@ -332,7 +332,7 @@ func TestSchedule(t *testing.T) { worker("w-arm", "gvisor", "node-a", tierTwo, withRuntimes(sandboxRuntime("gvisor", "arm64"))), worker("w-amd", "gvisor", "node-b", tierTwo, withRuntimes(sandboxRuntime("gvisor", "amd64"))), }, - constraints: Constraints{SandboxClass: "gvisor", RequiredRuntime: hardware.NewRequirement(sandboxRuntime("gvisor", "amd64"))}, + constraints: Constraints{RequiredRuntime: hardware.NewRequirement(sandboxRuntime("gvisor", "amd64"))}, wantPod: "w-amd", }, { @@ -341,7 +341,7 @@ func TestSchedule(t *testing.T) { worker("w-upgraded", "gvisor", "node-a", tierTwo, withRuntimes(sandboxRuntime("gvisor", "amd64", "new"), sandboxRuntime("gvisor", "amd64", "old"))), }, - constraints: Constraints{SandboxClass: "gvisor", RequiredRuntime: hardware.NewRequirement(sandboxRuntime("gvisor", "amd64", "old"))}, + constraints: Constraints{RequiredRuntime: hardware.NewRequirement(sandboxRuntime("gvisor", "amd64", "old"))}, wantPod: "w-upgraded", }, { @@ -349,7 +349,7 @@ func TestSchedule(t *testing.T) { fleet: fleet{ worker("w-arm", "gvisor", "node-a", tierTwo, withRuntimes(sandboxRuntime("gvisor", "arm64"))), }, - constraints: Constraints{SandboxClass: "gvisor", RequiredRuntime: hardware.NewRequirement(sandboxRuntime("gvisor", "amd64"))}, + constraints: Constraints{RequiredRuntime: hardware.NewRequirement(sandboxRuntime("gvisor", "amd64"))}, }, } @@ -388,44 +388,62 @@ func TestApplies(t *testing.T) { { name: "satisfies all constraints", worker: worker("w", "gvisor", "node-a", map[string]string{"tier": "1", "workload": "a"}), - constraints: Constraints{SandboxClass: "gvisor", TemplateSelector: tierSel, RequiredNodes: []string{"node-a"}, + constraints: Constraints{TemplateSelector: tierSel, RequiredNodes: []string{"node-a"}, ActorSelector: workloadSel}, want: true, }, { name: "assignment is ignored", worker: worker("w", "gvisor", "node-a", nil, assigned("demo", "someone-else")), - constraints: Constraints{SandboxClass: "gvisor"}, + constraints: Constraints{}, want: true, }, { - name: "class mismatch", + name: "worker with no sandbox class", + worker: worker("w", "", "node-a", nil), + constraints: Constraints{}, + want: false, + }, + { + name: "snapshot class pins the worker class", worker: worker("w", "microvm", "node-a", nil), constraints: Constraints{SandboxClass: "gvisor"}, want: false, }, + { + name: "snapshot class matches without a runtime requirement", + worker: worker("w", "gvisor", "node-a", nil), + constraints: Constraints{SandboxClass: "gvisor"}, + want: true, + }, + { + name: "no snapshot class accepts any class", + worker: worker("w", "microvm", "node-a", nil), + constraints: Constraints{}, + want: true, + }, { name: "template selector mismatch", worker: worker("w", "gvisor", "node-a", map[string]string{"tier": "2"}), - constraints: Constraints{SandboxClass: "gvisor", TemplateSelector: tierSel}, + constraints: Constraints{TemplateSelector: tierSel}, want: false, }, { name: "actor selector mismatch", worker: worker("w", "gvisor", "node-a", map[string]string{"workload": "b"}), - constraints: Constraints{SandboxClass: "gvisor", ActorSelector: workloadSel}, + constraints: Constraints{ActorSelector: workloadSel}, want: false, }, { name: "node restriction excludes", worker: worker("w", "gvisor", "node-b", nil), - constraints: Constraints{SandboxClass: "gvisor", RequiredNodes: []string{"node-a"}}, + constraints: Constraints{RequiredNodes: []string{"node-a"}}, want: false, }, { name: "AND of template and actor selectors match", worker: worker("w", "gvisor", "node-a", map[string]string{"tier": "1", "workload": "a"}), - constraints: Constraints{SandboxClass: "gvisor", + constraints: Constraints{ TemplateSelector: tierSel, ActorSelector: workloadSel}, want: true, @@ -433,7 +451,7 @@ func TestApplies(t *testing.T) { { name: "AND of two selectors, one fails", worker: worker("w", "gvisor", "node-a", map[string]string{"tier": "1", "workload": "b"}), - constraints: Constraints{SandboxClass: "gvisor", + constraints: Constraints{ TemplateSelector: tierSel, ActorSelector: workloadSel}, want: false, @@ -441,38 +459,38 @@ func TestApplies(t *testing.T) { { name: "skips draining worker", worker: worker("w", "gvisor", "node-a", nil, withState(ateapipb.WorkerState_WORKER_STATE_DRAINING)), - constraints: Constraints{SandboxClass: "gvisor"}, + constraints: Constraints{}, want: false, }, { name: "skips unavailable worker", worker: worker("w", "gvisor", "node-a", nil, withState(ateapipb.WorkerState_WORKER_STATE_UNAVAILABLE)), - constraints: Constraints{SandboxClass: "gvisor"}, + constraints: Constraints{}, want: false, }, { name: "skips unspecified worker", worker: worker("w", "gvisor", "node-a", nil, withState(ateapipb.WorkerState_WORKER_STATE_UNSPECIFIED)), - constraints: Constraints{SandboxClass: "gvisor"}, + constraints: Constraints{}, want: false, }, { name: "matches required runtime via default runtime", worker: worker("w", "gvisor", "node-a", nil, withRuntimes(sandboxRuntime("gvisor", "amd64", "old"))), - constraints: Constraints{SandboxClass: "gvisor", RequiredRuntime: hardware.NewRequirement(sandboxRuntime("gvisor", "amd64", "old"))}, + constraints: Constraints{RequiredRuntime: hardware.NewRequirement(sandboxRuntime("gvisor", "amd64", "old"))}, want: true, }, { name: "matches required runtime via restorable runtimes", worker: worker("w", "gvisor", "node-a", nil, withRuntimes(sandboxRuntime("gvisor", "amd64", "new"), sandboxRuntime("gvisor", "amd64", "old"))), - constraints: Constraints{SandboxClass: "gvisor", RequiredRuntime: hardware.NewRequirement(sandboxRuntime("gvisor", "amd64", "old"))}, + constraints: Constraints{RequiredRuntime: hardware.NewRequirement(sandboxRuntime("gvisor", "amd64", "old"))}, want: true, }, { name: "skips worker when required runtime does not match", worker: worker("w", "gvisor", "node-a", nil, withRuntimes(sandboxRuntime("gvisor", "arm64"))), - constraints: Constraints{SandboxClass: "gvisor", RequiredRuntime: hardware.NewRequirement(sandboxRuntime("gvisor", "amd64"))}, + constraints: Constraints{RequiredRuntime: hardware.NewRequirement(sandboxRuntime("gvisor", "amd64"))}, want: false, }, } @@ -506,10 +524,10 @@ func worker(pod, class, node string, lbls map[string]string, opts ...func(*ateap return w } -func sandboxRuntime(class, arch string, assetHash ...string) *ateapipb.SandboxRuntime { +func sandboxRuntime(class, arch string, bundleHash ...string) *ateapipb.SandboxRuntime { attrs := []*ateapipb.AttributeEntry{{Key: "architecture", Value: arch}} - if len(assetHash) > 0 && assetHash[0] != "" { - attrs = append(attrs, &ateapipb.AttributeEntry{Key: "gvisor_asset_hash", Value: assetHash[0]}) + if len(bundleHash) > 0 && bundleHash[0] != "" { + attrs = append(attrs, &ateapipb.AttributeEntry{Key: "sandbox_bundle_hash", Value: bundleHash[0]}) } return &ateapipb.SandboxRuntime{ SandboxClass: class, @@ -581,7 +599,7 @@ func withCapacity(cpuMilli, memBytes int64) func(*ateapipb.Worker) { // because the actor it is asking about filled the worker up. func TestAppliesIgnoresRoom(t *testing.T) { full := worker("w-full", "gvisor", "node-a", nil, withMaxActors(1), assigned("demo", "resident")) - constraints := Constraints{SandboxClass: "gvisor"} + constraints := Constraints{} s := New(fleet{full}) if !s.Applies(full, constraints) { @@ -601,7 +619,7 @@ func TestSchedulePowerOfTwoChoicesTieBreaking(t *testing.T) { worker("w-1", "gvisor", "node-b", nil, withMaxActors(4), assigned("demo", "b")), worker("w-2", "gvisor", "node-c", nil, withMaxActors(4), assigned("demo", "c")), } - constraints := Constraints{SandboxClass: "gvisor"} + constraints := Constraints{} // On equal utilization, Schedule must preserve the first sampled candidate so // every tied worker remains reachable via the random source. @@ -644,7 +662,7 @@ func TestSchedulePowerOfTwoChoicesShiftedSecondSample(t *testing.T) { worker("w-2", "gvisor", "node-c", nil, withMaxActors(4)), } s := New(f, WithIntn(seqIntn(0, 1))) - got, err := s.Schedule(context.Background(), Constraints{SandboxClass: "gvisor"}) + got, err := s.Schedule(context.Background(), Constraints{}) if err != nil { t.Fatalf("Schedule() error = %v", err) } @@ -656,8 +674,7 @@ func TestSchedulePowerOfTwoChoicesShiftedSecondSample(t *testing.T) { func TestScheduleMalformedLimitsIsNotNoCapacity(t *testing.T) { f := fleet{worker("w-1", "gvisor", "node-a", nil)} constraints := Constraints{ - SandboxClass: "gvisor", - Limits: &ateapipb.Resources{Limits: []*ateapipb.Limits{{Name: resources.ResourceCPU, Quantity: "not-a-quantity"}}}, + Limits: &ateapipb.Resources{Limits: []*ateapipb.Limits{{Name: resources.ResourceCPU, Quantity: "not-a-quantity"}}}, } _, err := New(f, WithIntn(firstIntn)).Schedule(context.Background(), constraints) if err == nil { diff --git a/cmd/ateapi/internal/store/atepg/common_test.go b/cmd/ateapi/internal/store/atepg/common_test.go index accb18c49f..acf24efd93 100644 --- a/cmd/ateapi/internal/store/atepg/common_test.go +++ b/cmd/ateapi/internal/store/atepg/common_test.go @@ -56,10 +56,11 @@ func setTestPolicyManager(t *testing.T, p *Persistence) { } t.Cleanup(fgaServer.Close) _, policyManager, err := authz.New(t.Context(), p.pool, fgaServer, nil, authz.SystemPrincipals{ - ControllerSPIFFEID: installdefaults.ControllerSPIFFEID(installdefaults.SystemNamespace), - EgressGatewaySPIFFEID: installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace), - IngressRouterSPIFFEID: installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), - AteletSPIFFEID: installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), + ControllerSPIFFEID: installdefaults.ControllerSPIFFEID(installdefaults.SystemNamespace), + EgressGatewaySPIFFEID: installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace), + IngressRouterSPIFFEID: installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), + AteletSPIFFEID: installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), + APIServerSidecarSPIFFEID: installdefaults.APIServerSPIFFEID(installdefaults.SystemNamespace), }) if err != nil { t.Fatalf("authz.New failed: %v", err) diff --git a/cmd/ateapi/internal/store/storetest/storetest.go b/cmd/ateapi/internal/store/storetest/storetest.go index 3385bb4112..3ed744c6a7 100644 --- a/cmd/ateapi/internal/store/storetest/storetest.go +++ b/cmd/ateapi/internal/store/storetest/storetest.go @@ -100,10 +100,11 @@ func SetupPostgresPersistence(t *testing.T) *atepg.Persistence { // initialize authz.New without testing system component roles. func DefaultSystemPrincipals() authz.SystemPrincipals { return authz.SystemPrincipals{ - ControllerSPIFFEID: installdefaults.ControllerSPIFFEID(installdefaults.SystemNamespace), - EgressGatewaySPIFFEID: installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace), - IngressRouterSPIFFEID: installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), - AteletSPIFFEID: installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), + ControllerSPIFFEID: installdefaults.ControllerSPIFFEID(installdefaults.SystemNamespace), + EgressGatewaySPIFFEID: installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace), + IngressRouterSPIFFEID: installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), + AteletSPIFFEID: installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), + APIServerSidecarSPIFFEID: installdefaults.APIServerSPIFFEID(installdefaults.SystemNamespace), } } diff --git a/cmd/ateapi/internal/workerservice/certificate_test.go b/cmd/ateapi/internal/workerservice/certificate_test.go index afcbb1ea08..357aa7f9a0 100644 --- a/cmd/ateapi/internal/workerservice/certificate_test.go +++ b/cmd/ateapi/internal/workerservice/certificate_test.go @@ -373,9 +373,9 @@ func TestMintWorkerActorCertificate_Errors(t *testing.T) { want: codes.NotFound, }, { - name: "paused actor being suspended on this node", + name: "paused actor with its upload pending on this node", ctx: authed, - status: pausedStatus(ateapipb.ActorState_ACTOR_STATE_SUSPENDING, "node-1"), + status: uploadPendingStatus("node-1"), assignments: released, want: codes.NotFound, }, diff --git a/cmd/ateapi/internal/workerservice/jwt_test.go b/cmd/ateapi/internal/workerservice/jwt_test.go index e3cb3e47dc..c27ad58871 100644 --- a/cmd/ateapi/internal/workerservice/jwt_test.go +++ b/cmd/ateapi/internal/workerservice/jwt_test.go @@ -125,6 +125,18 @@ func pausedStatus(state ateapipb.ActorState, node string) *ateapipb.ActorStatus } } +// uploadPendingStatus is how a paused actor looks until atelet's background +// upload of its local snapshot is recorded: SUSPENDED, no worker, and the +// latest generation's durable copy PENDING. +func uploadPendingStatus(node string) *ateapipb.ActorStatus { + status := pausedStatus(ateapipb.ActorState_ACTOR_STATE_SUSPENDED, node) + status.Snapshots[0].DurableSnapshot = &ateapipb.SnapshotStorage{ + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: "gs://snaps/root/atespaces/team-a/actors/uid/snap-1"}, + } + return status +} + func TestMintWorkerActorJwt_Errors(t *testing.T) { st, actor := newFakeStoreWithActor("team-a", "my-actor", "3b9f1e77-2c4d-4a80-91be-6d5c8f0a7e21") s := New(st, &fakeSuspender{}, testAteletSPIFFEID, nil, testActorJWTIssuer, newTestJWTPool(t)) @@ -328,20 +340,41 @@ func TestMintWorkerActorJwt_Errors(t *testing.T) { want: codes.NotFound, }, { - // assigned_node without a local-only snapshot is not a paused actor, - // whatever left it set. - name: "suspended actor with a stale assigned_node", + // atelet uploads the paused actor's checkpoint in the background + // from the node that holds it, with no worker assigned. + name: "paused actor with its upload pending on this node", ctx: authed, - status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, AssignedNode: "node-1"}, - want: codes.NotFound, + status: uploadPendingStatus("node-1"), + want: codes.OK, }, { - // Suspending a paused actor uploads its checkpoint from the node that - // holds it, with no worker assigned. - name: "paused actor being suspended on this node", + name: "paused actor with its upload pending on another node", ctx: authed, - status: pausedStatus(ateapipb.ActorState_ACTOR_STATE_SUSPENDING, "node-1"), - want: codes.OK, + status: uploadPendingStatus("node-2"), + want: codes.NotFound, + }, + { + // The node still holds the snapshot's files until its detach report + // clears the locality, so it still acts for the actor. + name: "paused actor whose snapshot is uploaded, local copy kept", + ctx: authed, + status: func() *ateapipb.ActorStatus { + status := uploadPendingStatus("node-1") + status.Snapshots[0].DurableSnapshot.Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + return status + }(), + want: codes.OK, + }, + { + // Only a SUSPENDED actor is hosted by a node with no worker. + name: "suspending actor with no worker", + ctx: authed, + status: func() *ateapipb.ActorStatus { + status := uploadPendingStatus("node-1") + status.State = ateapipb.ActorState_ACTOR_STATE_SUSPENDING + return status + }(), + want: codes.NotFound, }, { // A crash clears the worker but leaves assigned_node on the last diff --git a/cmd/ateapi/internal/workerservice/register_test.go b/cmd/ateapi/internal/workerservice/register_test.go index 2dec5e845d..2906277774 100644 --- a/cmd/ateapi/internal/workerservice/register_test.go +++ b/cmd/ateapi/internal/workerservice/register_test.go @@ -293,15 +293,15 @@ func TestRegisterWorker_SortsAttributes(t *testing.T) { return got.GetWorker() } - first := register(runtimeWith("cpu_features", "architecture"), runtimeWith("gvisor_asset_hash", "architecture")) + first := register(runtimeWith("cpu_features", "architecture"), runtimeWith("sandbox_bundle_hash", "architecture")) if diff := cmp.Diff(runtimeWith("architecture", "cpu_features"), first.GetStatus().GetDefaultRuntime(), protocmp.Transform()); diff != "" { t.Errorf("default_runtime not stored sorted by key (-want +got):\n%s", diff) } - if diff := cmp.Diff([]*ateapipb.SandboxRuntime{runtimeWith("architecture", "gvisor_asset_hash")}, first.GetStatus().GetRestorableRuntimes(), protocmp.Transform()); diff != "" { + if diff := cmp.Diff([]*ateapipb.SandboxRuntime{runtimeWith("architecture", "sandbox_bundle_hash")}, first.GetStatus().GetRestorableRuntimes(), protocmp.Transform()); diff != "" { t.Errorf("restorable_runtimes not stored sorted by key (-want +got):\n%s", diff) } - again := register(runtimeWith("architecture", "cpu_features"), runtimeWith("architecture", "gvisor_asset_hash")) + again := register(runtimeWith("architecture", "cpu_features"), runtimeWith("architecture", "sandbox_bundle_hash")) if got, want := again.GetMetadata().GetVersion(), first.GetMetadata().GetVersion(); got != want { t.Errorf("version = %d after a reordered repeat, want %d unchanged", got, want) } diff --git a/cmd/ateapi/internal/workerservice/suspend.go b/cmd/ateapi/internal/workerservice/suspend.go index 35d1a4c3f7..c59d0d9622 100644 --- a/cmd/ateapi/internal/workerservice/suspend.go +++ b/cmd/ateapi/internal/workerservice/suspend.go @@ -39,16 +39,6 @@ func (s *Server) RequestActorSuspend(ctx context.Context, req *ateapipb.RequestA return nil, resources.ToAPIError(errs) } - // A golden actor may be suspended, but only by the template controller: - // the suspend is what commits its snapshot, and the controller runs it once - // the warmup window it set has elapsed. A golden actor boots and waits, so - // it looks idle from the moment it starts, and honoring its own request - // would commit the snapshot before that window ends -- leaving every actor - // restored from the template to start from a workload that never warmed up. - if req.GetActor().GetAtespace() == resources.GoldenActorAtespace { - return nil, apierror.FailedPrecondition("actors in atespace %q are golden actors, which cannot request their own suspend", resources.GoldenActorAtespace) - } - workerName := req.GetWorker().GetName() actorRef := resources.ActorRefFromObjectRef(req.GetActor()) diff --git a/cmd/ateapi/internal/workerservice/suspend_test.go b/cmd/ateapi/internal/workerservice/suspend_test.go index 83866ec88b..426a406274 100644 --- a/cmd/ateapi/internal/workerservice/suspend_test.go +++ b/cmd/ateapi/internal/workerservice/suspend_test.go @@ -26,7 +26,6 @@ import ( "go.ate.dev/substrate/cmd/ateapi/internal/store" "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" "go.ate.dev/substrate/internal/apierror" - "go.ate.dev/substrate/internal/resources" "go.ate.dev/substrate/pkg/proto/ateapipb" ) @@ -209,48 +208,6 @@ func TestRequestActorSuspend_Errors(t *testing.T) { } } -// A golden actor is committed by the template controller once the warmup it -// timed has elapsed, and the suspend is that commit. The actor idles from the -// moment it boots, so it would ask immediately and snapshot a workload that -// never warmed up. Control.SuspendActor still serves the controller; only the -// Worker's request for one is refused. -func TestRequestActorSuspend_GoldenActorCannotSelfSuspend(t *testing.T) { - st, cleanup := storetest.SetupTestStore(t) - defer cleanup() - seedReportedWorker(t, st, testNode, &ateapipb.WorkerResources{Actors: 1}) - golden := storetest.MustCreateActor(t, context.Background(), st, &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: resources.GoldenActorAtespace, Name: suspendActor}, - Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_RUNNING, - WorkerAssignment: &ateapipb.WorkerAssignment{ - Worker: &ateapipb.ObjectRef{Name: testWorkerName}, - WorkerNamespace: "ate-system", - WorkerPool: "pool-1", - WorkerPod: "worker-pod-1", - WorkerPodUid: testWorkerName, - }, - }, - }) - suspender := &fakeSuspender{} - s := New(st, suspender, testAteletSPIFFEID, nil, "", nil) - - // Everything else about the request is in order: the Worker really does - // host this actor, so being golden is the only reason it is refused. - _, err := s.RequestActorSuspend( - ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)), - &ateapipb.RequestActorSuspendRequest{ - Worker: &ateapipb.ObjectRef{Name: testWorkerName}, - Actor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: suspendActor}, - ActorUid: golden.GetMetadata().GetUid(), - }) - if got := apierror.Code(err); got != codes.FailedPrecondition { - t.Fatalf("code = %v (err %v), want FailedPrecondition", got, err) - } - if len(suspender.calls) != 0 { - t.Errorf("golden actor request still reached the suspend: %v", suspender.calls) - } -} - // The suspend is a proposal, and the control plane's refusal is the answer: a // request that loses a race to a resume, pause, or delete comes back as what // the workflow said, not as something the worker has to interpret. diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index ee8343dc5f..5fec04b9db 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -36,12 +36,12 @@ import ( "go.ate.dev/substrate/cmd/ateapi/internal/store/atepg" "go.ate.dev/substrate/cmd/ateapi/internal/workercache" "go.ate.dev/substrate/cmd/ateapi/internal/workerservice" + "go.ate.dev/substrate/internal/actoridjwt" "go.ate.dev/substrate/internal/ateinterceptors" "go.ate.dev/substrate/internal/credbundle" "go.ate.dev/substrate/internal/installdefaults" "go.ate.dev/substrate/internal/localca" "go.ate.dev/substrate/internal/localjwtauthority" - "go.ate.dev/substrate/internal/oidcdiscovery" "go.ate.dev/substrate/internal/oidcjwt" "go.ate.dev/substrate/internal/pluginsocket" "go.ate.dev/substrate/internal/preview" @@ -92,12 +92,13 @@ var ( authzBootstrapOwners = pflag.StringSlice("authz-bootstrap-owners", nil, "Principal IDs that are always global owners while listed, independent of the stored global AccessPolicy. Removing an ID revokes its access on restart. At least one is required when --experimental-enable-authz is set.") // These SPIFFE IDs hold the matching system component role when they call // over mTLS. - controllerSPIFFEID = pflag.String("controller-spiffe-id", installdefaults.ControllerSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of ate-controller. Holds the controller authz role, which manages workers.") - egressGatewaySPIFFEID = pflag.String("egress-gateway-spiffe-id", installdefaults.EgressSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of atenet-egress. Holds the egress_gateway authz role, which reads and lists actors and reads their egress policies.") - ingressRouterSPIFFEID = pflag.String("ingress-router-spiffe-id", installdefaults.RouterSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of atenet-router. Holds the ingress_router authz role, which resumes and lists actors.") + controllerSPIFFEID = pflag.String("controller-spiffe-id", installdefaults.ControllerSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of ate-controller. Holds the controller authz role, which manages workers.") + egressGatewaySPIFFEID = pflag.String("egress-gateway-spiffe-id", installdefaults.EgressSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of atenet-egress. Holds the egress_gateway authz role, which reads and lists actors and reads their egress policies.") + ingressRouterSPIFFEID = pflag.String("ingress-router-spiffe-id", installdefaults.RouterSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of atenet-router. Holds the ingress_router authz role, which resumes and lists actors.") + apiServerSidecarSPIFFEID = pflag.String("api-server-sidecar-spiffe-id", installdefaults.APIServerSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of the ate-api-server pod, which its sidecars share. Holds the api_server_sidecar role, which reads actors so the objectstore control plugin can scope their storage credentials.") actorIDJWTPoolFile = pflag.String("actor-id-jwt-pool", "", "The file that contains the serialized JWT authority pool for signing actor JWTs") - actorJWTIssuer = pflag.String("actor-jwt-issuer", "", "Issuer URL placed in the iss claim of actor JWTs. Relying parties fetch /.well-known/openid-configuration to verify them. Must be https with no query or fragment. Required.") + actorJWTIssuer = pflag.String("actor-jwt-issuer", "", "Issuer URL placed in the iss claim of actor JWTs. Relying parties fetch /.well-known/openid-configuration to verify them. Must be https://, with no path. Required.") defaultEgressGatewayAddress = pflag.String("default-egress-gateway-address", "", "Default address (host:port) of the egress PEP that each actor's atunnel dials. Sent on every atelet Run and Restore, so it takes effect at the actor's next activation. Empty leaves actors with no TCP egress.") actorIDCAPoolFile = pflag.String("actor-id-ca-pool", "", "The file that contains the CA pool for signing actor JWTs") @@ -251,10 +252,11 @@ func main() { } defer fgaServer.Close() authorizer, policyManager, err := authz.New(shutdownCtx, pool, fgaServer, *authzBootstrapOwners, authz.SystemPrincipals{ - ControllerSPIFFEID: *controllerSPIFFEID, - EgressGatewaySPIFFEID: *egressGatewaySPIFFEID, - IngressRouterSPIFFEID: *ingressRouterSPIFFEID, - AteletSPIFFEID: ateletSPIFFEID, + ControllerSPIFFEID: *controllerSPIFFEID, + EgressGatewaySPIFFEID: *egressGatewaySPIFFEID, + IngressRouterSPIFFEID: *ingressRouterSPIFFEID, + AteletSPIFFEID: ateletSPIFFEID, + APIServerSidecarSPIFFEID: *apiServerSidecarSPIFFEID, }) if err != nil { serverboot.Fatal(ctx, "Failed to initialize OpenFGA authz", err) @@ -341,6 +343,7 @@ func main() { controlSrv := controlapi.NewRPCService( persistence, workerCache, + workerPoolLister, sandboxConfigLister, csiDriverConfigLister, storageClassLister, @@ -689,5 +692,5 @@ func validateActorJWTIssuer(issuer string) error { if issuer == "" { return errors.New("required") } - return oidcdiscovery.ValidateIssuer(issuer) + return actoridjwt.ValidateIssuer(issuer) } diff --git a/cmd/ateapi/main_test.go b/cmd/ateapi/main_test.go index 1231be9215..b66a7c234f 100644 --- a/cmd/ateapi/main_test.go +++ b/cmd/ateapi/main_test.go @@ -105,7 +105,7 @@ func TestValidateActorJWTIssuer(t *testing.T) { {issuer: "", wantErr: true}, {issuer: "http://idp.example.com", wantErr: true}, {issuer: "https://idp.ate-system.svc"}, - {issuer: "https://idp.example.com/prod/"}, + {issuer: "https://idp.example.com/prod", wantErr: true}, } { err := validateActorJWTIssuer(tt.issuer) if gotErr := err != nil; gotErr != tt.wantErr { diff --git a/cmd/atelet/deviceplugin.go b/cmd/atelet/deviceplugin.go index c63c42da47..a5bd8abda5 100644 --- a/cmd/atelet/deviceplugin.go +++ b/cmd/atelet/deviceplugin.go @@ -30,27 +30,6 @@ import ( // exist; workers are handed the real host paths, which kubelet resolves. const hostDevRoot = "/host/dev" -// microvmNodeCapable reports whether this node can host micro-VM workers: -// cloud-hypervisor needs /dev/kvm (VmCreate fails with EPERM without it), and -// worker pods request the matching extended resource, so they only schedule to -// nodes where the device exists. Device presence is therefore the earliest -// reliable eligibility signal — known at atelet startup, before any WorkerPool -// schedules here. -// -// TODO(https://github.com/agent-substrate/substrate/pull/1207): /dev/kvm is -// not the only micro-VM hypervisor device; once /dev/mshv support lands, an -// mshv-only node would be wrongly reported incapable here. Treat presence of -// any micro-VM hypervisor device in SandboxDevices as capability, not KVM -// alone. -func microvmNodeCapable(devRoot string) bool { - for _, d := range deviceplugin.SandboxDevices { - if d.ResourceName == deviceplugin.ResourceKVM { - return d.Present(devRoot) - } - } - return false -} - // startDevicePlugins advertises the sandbox host devices present on this node to // kubelet as extended resources, in the background for the lifetime of ctx. This // is what lets a worker be granted /dev/kvm without running privileged. atelet diff --git a/cmd/atelet/internal/apivalidation/validate.go b/cmd/atelet/internal/apivalidation/validate.go index bc67bff8dd..4e73e734b2 100644 --- a/cmd/atelet/internal/apivalidation/validate.go +++ b/cmd/atelet/internal/apivalidation/validate.go @@ -16,7 +16,6 @@ package apivalidation import ( "context" - "fmt" "go.ate.dev/substrate/internal/proto/ateletpb" "go.ate.dev/substrate/internal/resources" @@ -105,16 +104,6 @@ func ValidateCustom_AssetFile_Sha256(_ context.Context, _ operation.Operation, f return nil } -// ValidateCustom_PromoteSnapshotRequest_Atespace rejects the golden atespace: -// golden actors are never paused, so a paused checkpoint is never promoted to -// a golden snapshot. -func ValidateCustom_PromoteSnapshotRequest_Atespace(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { - if *value == resources.GoldenActorAtespace { - return field.ErrorList{field.Forbidden(fldPath, fmt.Sprintf("atespace %q holds golden actors, which are never paused", *value))} - } - return nil -} - func ValidateCustom_ObjectStorage_SnapshotUri(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { return validateSnapshotURI(fldPath, *value) } diff --git a/cmd/atelet/internal/apivalidation/validation_test.go b/cmd/atelet/internal/apivalidation/validation_test.go index 88d5d17949..1978ec860c 100644 --- a/cmd/atelet/internal/apivalidation/validation_test.go +++ b/cmd/atelet/internal/apivalidation/validation_test.go @@ -21,7 +21,6 @@ import ( "testing" "go.ate.dev/substrate/internal/proto/ateletpb" - "go.ate.dev/substrate/internal/resources" "k8s.io/apimachinery/pkg/api/operation" "k8s.io/apimachinery/pkg/util/validation/field" ) @@ -392,6 +391,18 @@ func TestValidateRunRequest(t *testing.T) { r.CpuMilli = 0 r.MemoryBytes = 0 }), + }, { + name: "valid sandbox_runtime", + obj: valid(func(r *ateletpb.RunRequest) { + r.SandboxRuntime = runtimeWithAttributes(1) + }), + }, { + name: "sandbox_runtime errors surface at their path", + obj: valid(func(r *ateletpb.RunRequest) { + r.SandboxRuntime = runtimeWithAttributes(1) + r.SandboxRuntime.SandboxClass = "kata" + }), + want: field.ErrorList{field.NotSupported[string](field.NewPath("sandbox_runtime", "sandbox_class"), nil, nil)}, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -835,6 +846,18 @@ func TestValidateRestoreRequest(t *testing.T) { r.SandboxAssets = validSandboxAssets(func(a *ateletpb.SandboxAssets) { a.SandboxClass = "kata" }) }), want: field.ErrorList{field.NotSupported[string](field.NewPath("sandbox_assets", "sandbox_class"), nil, nil)}, + }, { + name: "valid sandbox_runtime", + obj: valid(func(r *ateletpb.RestoreRequest) { + r.SandboxRuntime = runtimeWithAttributes(1) + }), + }, { + name: "sandbox_runtime errors surface at their path", + obj: valid(func(r *ateletpb.RestoreRequest) { + r.SandboxRuntime = runtimeWithAttributes(1) + r.SandboxRuntime.SandboxClass = "kata" + }), + want: field.ErrorList{field.NotSupported[string](field.NewPath("sandbox_runtime", "sandbox_class"), nil, nil)}, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -886,10 +909,6 @@ func TestValidatePromoteSnapshotRequest(t *testing.T) { name: "invalid atespace: path escape", obj: valid(func(r *ateletpb.PromoteSnapshotRequest) { r.Atespace = "../escape" }), want: field.ErrorList{field.Invalid(field.NewPath("atespace"), nil, "").WithOrigin("format=k8s-short-name")}, - }, { - name: "golden atespace", - obj: valid(func(r *ateletpb.PromoteSnapshotRequest) { r.Atespace = resources.GoldenActorAtespace }), - want: field.ErrorList{field.Forbidden(field.NewPath("atespace"), "")}, }, { name: "invalid actor_name: uppercase", obj: valid(func(r *ateletpb.PromoteSnapshotRequest) { r.ActorName = "UPPER" }), diff --git a/cmd/atelet/internal/apivalidation/zz_generated.validation.go b/cmd/atelet/internal/apivalidation/zz_generated.validation.go index 10cc29c62e..d69ca8cd5c 100644 --- a/cmd/atelet/internal/apivalidation/zz_generated.validation.go +++ b/cmd/atelet/internal/apivalidation/zz_generated.validation.go @@ -1966,10 +1966,6 @@ func Validate_PromoteSnapshotRequest( if earlyReturn { return // do not proceed } - // custom validation - if e := ValidateCustom_PromoteSnapshotRequest_Atespace(ctx, op, fldPath, obj, oldObj); len(e) != 0 { - errs = append(errs, e...) - } if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { errs = append(errs, e...) } @@ -2886,6 +2882,36 @@ func Validate_RestoreRequest( errs = append(errs, fn(fldPath.Child("sandbox_assets"), obj.SandboxAssets, oldVal, oldObj != nil)...) } + { // field ateletpb.RestoreRequest.SandboxRuntime + fn := func( + fldPath *field.Path, + obj, oldObj *ateletpb.SandboxRuntime, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_SandboxRuntime(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.RestoreRequest) *ateletpb.SandboxRuntime { + return oldObj.SandboxRuntime + }) + errs = append(errs, fn(fldPath.Child("sandbox_runtime"), obj.SandboxRuntime, oldVal, oldObj != nil)...) + } + return errs } @@ -3242,6 +3268,36 @@ func Validate_RunRequest( errs = append(errs, fn(fldPath.Child("memory_bytes"), &obj.MemoryBytes, oldVal, oldObj != nil)...) } + { // field ateletpb.RunRequest.SandboxRuntime + fn := func( + fldPath *field.Path, + obj, oldObj *ateletpb.SandboxRuntime, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_SandboxRuntime(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.RunRequest) *ateletpb.SandboxRuntime { + return oldObj.SandboxRuntime + }) + errs = append(errs, fn(fldPath.Child("sandbox_runtime"), obj.SandboxRuntime, oldVal, oldObj != nil)...) + } + return errs } diff --git a/cmd/atelet/internal/ateletpath/ateletpath.go b/cmd/atelet/internal/ateletpath/ateletpath.go index 88cdc56429..dc0dbec4b3 100644 --- a/cmd/atelet/internal/ateletpath/ateletpath.go +++ b/cmd/atelet/internal/ateletpath/ateletpath.go @@ -47,6 +47,14 @@ var ( // default version. Wiped at startup. Under BasePath so the links the // cache serves into it are on the cache's mount. SandboxAssetScratchDir = filepath.Join(nodepath.BasePath, "sandbox-asset-scratch") + + // SandboxRuntimeIndexFile holds every sandbox runtime a worker on this + // node has advertised, by the runtime's hash, with the asset record the + // hash was computed from. atelet rewrites it before a worker's own file + // can let the worker register a new hash, and loads it at startup, so a + // runtime registered before a restart resolves the moment atelet serves + // again. + SandboxRuntimeIndexFile = filepath.Join(nodepath.BasePath, "sandbox-runtime-index.json") ) func ActorPath(actorUID string) string { diff --git a/cmd/atelet/internal/atomicfile/atomicfile.go b/cmd/atelet/internal/atomicfile/atomicfile.go index d9c71c70cd..e42b3b3bc0 100644 --- a/cmd/atelet/internal/atomicfile/atomicfile.go +++ b/cmd/atelet/internal/atomicfile/atomicfile.go @@ -24,10 +24,20 @@ import ( // renaming it over the target so readers never observe a truncated or // partially written value. // -// TODO: Write does not fsync, so after a node crash path can be empty or still -// hold its old contents. Investigate an opt-in sync for callers whose files -// must survive a crash; other callers don't need it. +// Write does not fsync, so after a node crash path can be empty or still +// hold its old contents; WriteSync is for a file that must survive one. func Write(path string, data []byte, perm os.FileMode) error { + return write(path, data, perm, false) +} + +// WriteSync is Write for a file that must survive a node crash: the data is +// synced to disk before the rename, and the directory after it, so once +// WriteSync returns the file is durable with exactly this content. +func WriteSync(path string, data []byte, perm os.FileMode) error { + return write(path, data, perm, true) +} + +func write(path string, data []byte, perm os.FileMode, sync bool) error { f, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".tmp-*") if err != nil { return err @@ -42,8 +52,25 @@ func Write(path string, data []byte, perm os.FileMode) error { f.Close() return err } + if sync { + if err := f.Sync(); err != nil { + f.Close() + return err + } + } if err := f.Close(); err != nil { return err } - return os.Rename(f.Name(), path) + if err := os.Rename(f.Name(), path); err != nil { + return err + } + if !sync { + return nil + } + dir, err := os.Open(filepath.Dir(path)) + if err != nil { + return err + } + defer dir.Close() + return dir.Sync() } diff --git a/cmd/atelet/internal/atomicfile/atomicfile_test.go b/cmd/atelet/internal/atomicfile/atomicfile_test.go index 94d2576352..92c8447590 100644 --- a/cmd/atelet/internal/atomicfile/atomicfile_test.go +++ b/cmd/atelet/internal/atomicfile/atomicfile_test.go @@ -67,3 +67,27 @@ func TestWrite(t *testing.T) { } }) } + +// TestWriteSync covers the durable variant: same contract as Write, the +// file holds exactly the data afterwards and no temp file is left. +func TestWriteSync(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "runtime.json") + if err := WriteSync(path, []byte("v1"), 0o600); err != nil { + t.Fatalf("WriteSync: %v", err) + } + if err := WriteSync(path, []byte("v2"), 0o600); err != nil { + t.Fatalf("WriteSync over an existing file: %v", err) + } + got, err := os.ReadFile(path) + if err != nil || string(got) != "v2" { + t.Fatalf("ReadFile = %q, %v; want v2", got, err) + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 { + t.Errorf("dir holds %d entries after WriteSync, want only the file", len(entries)) + } +} diff --git a/cmd/atelet/internal/snapshottracker/actor.go b/cmd/atelet/internal/snapshottracker/actor.go index b7be0e9c36..13c5a5d6ba 100644 --- a/cmd/atelet/internal/snapshottracker/actor.go +++ b/cmd/atelet/internal/snapshottracker/actor.go @@ -18,7 +18,9 @@ import ( "encoding/json" "time" + "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" @@ -32,8 +34,13 @@ type SnapshotRef struct { // on this node. Zero when this atelet did not take it. CommitTime time.Time // UploadTime is when this atelet committed the snapshot's manifest to - // object storage. Zero unless it did. + // object storage. Zero unless it did. It is only valid for durable + // snapshots, so LatestLocal never carries it. UploadTime time.Time + // Actor is the actor the snapshot belongs to, so a background upload of + // the snapshot can act for the actor the way an RPC for it would. The + // Checkpoint or PromoteSnapshot that commits the snapshot sets it. + Actor resources.ActorRef } // snapshotRefJSON is a SnapshotRef as the snapshot state file stores it. @@ -42,6 +49,8 @@ type snapshotRefJSON struct { Snapshot json.RawMessage `json:"snapshot"` CommitTime time.Time `json:"commitTime,omitzero"` UploadTime time.Time `json:"uploadTime,omitzero"` + Atespace string `json:"atespace,omitempty"` + ActorName string `json:"actorName,omitempty"` } // MarshalJSON encodes r as a snapshotRefJSON. @@ -50,7 +59,7 @@ func (r SnapshotRef) MarshalJSON() ([]byte, error) { if err != nil { return nil, err } - return json.Marshal(snapshotRefJSON{Snapshot: snap, CommitTime: r.CommitTime, UploadTime: r.UploadTime}) + return json.Marshal(snapshotRefJSON{Snapshot: snap, CommitTime: r.CommitTime, UploadTime: r.UploadTime, Atespace: r.Actor.Atespace, ActorName: r.Actor.Name}) } // UnmarshalJSON decodes a snapshotRefJSON into r. @@ -63,7 +72,7 @@ func (r *SnapshotRef) UnmarshalJSON(data []byte) error { if err := protojson.Unmarshal(rec.Snapshot, snap); err != nil { return err } - *r = SnapshotRef{Snapshot: snap, CommitTime: rec.CommitTime, UploadTime: rec.UploadTime} + *r = SnapshotRef{Snapshot: snap, CommitTime: rec.CommitTime, UploadTime: rec.UploadTime, Actor: resources.ActorRef{Atespace: rec.Atespace, Name: rec.ActorName}} return nil } @@ -73,6 +82,26 @@ func (r SnapshotRef) clone() *SnapshotRef { return &r } +// uuid returns the snapshot's UUID. +func (r *SnapshotRef) uuid() string { + return r.Snapshot.GetSnapshotUuid() +} + +// sameUUID reports whether a and b are non-nil and have the same UUID. +func sameUUID(a, b *SnapshotRef) bool { + return a != nil && b != nil && a.uuid() == b.uuid() +} + +// uri returns the snapshot's object storage URI. +func (r *SnapshotRef) uri() string { + return r.Snapshot.GetObject().GetSnapshotUri() +} + +// generation returns the snapshot's generation. +func (r *SnapshotRef) generation() int32 { + return r.Snapshot.GetGeneration() +} + // report converts the SnapshotRef for a field of ActorSnapshotState. func (r *SnapshotRef) report() *ateletpb.SnapshotReport { if r == nil { @@ -92,11 +121,63 @@ func (r *SnapshotRef) report() *ateletpb.SnapshotReport { // snapshot state file. type actorSnapshotsRecord struct { // LatestLocal is the newest snapshot currently stored in the actor's - // node-local checkpoint directory. Nil after an UPLOAD_MODE_IMMEDIATE - // checkpoint. + // node-local checkpoint directory. Nil once detached. LatestLocal *SnapshotRef `json:"latestLocal,omitempty"` + // Uploading is the snapshot a background upload has begun making durable. + // It is saved in the state file and stays set across checkpoints, retry + // backoffs, and atelet restarts until the upload succeeds, fails and a + // newer local snapshot replaces it, or a newer snapshot becomes durable, + // unlike runningUpload, which is only set while an upload goroutine is + // running. + Uploading *SnapshotRef `json:"uploading,omitempty"` // LatestDurable is the newest snapshot confirmed in object storage. LatestDurable *SnapshotRef `json:"latestDurable,omitempty"` + + // UndurableSince is what uploadDelay is measured from: the CommitTime of + // the oldest local checkpoint taken since the last durable snapshot, or, + // for one taken while an upload ran, when that upload succeeded. Zero + // once the latest local checkpoint is durable, or an immediate + // checkpoint has superseded it. + // + // Measuring from UndurableSince rather than LatestLocal.CommitTime + // ensures an actor that pauses and resumes repeatedly still uploads + // within uploadDelay instead of postponing durability indefinitely. An + // upload changes UndurableSince only once it succeeds, and a failed or + // interrupted one leaves it as it was. + UndurableSince time.Time `json:"undurableSince,omitzero"` + // DetachAt is when atelet next checks whether to detach the actor. Zero + // means no detach is scheduled (see detachScheduled). + // + // It is set or updated: + // - After committing a local snapshot (CommitLocal). + // - After a failed detach. + // + // It is cleared: + // - When a Run or Restore of the actor begins (markStarted), whether or + // not it succeeds. + // - When the latest local snapshot is detached, or a newer durable + // snapshot supersedes it (clearLatestLocal). + // - When Terminate forgets the actor. + // + // It is not updated, but the actor is queued again, when a worker pulls + // the actor out of the workqueue and: + // - A Run, Restore, Checkpoint, or PromoteSnapshot of the actor is in + // progress (BeginStart, BeginCheckpoint, or BeginPromote). Its end + // queues the actor again. + // - Another operation on the actor, such as a Detach call, is in + // progress. Its end queues the actor again. + // - The detach is not due yet. The worker queues the actor again for + // when it is. + // + // atelet never detaches a snapshot that no detach can make durable (see + // detachTarget). + DetachAt time.Time `json:"detachAt,omitzero"` + // ReportDueSince is when the latest background change atelet owes a + // report of (a new LatestDurable, or local snapshots cleared by a + // detach) finished. Zero once no report is due: the control plane + // acknowledged or refused it, a commit or Detach call returned the + // state, or atelet gave up after reportQueue.giveUpAfter. + ReportDueSince time.Time `json:"reportDueSince,omitzero"` } // actorSnapshots is what atelet tracks of one actor's snapshots on this node. @@ -104,14 +185,345 @@ type actorSnapshotsRecord struct { type actorSnapshots struct { actorUID string - // actorSnapshotsRecord holds what the snapshot state file keeps. + // actorSnapshotsRecord holds what the snapshot state file keeps. The + // fields after it live only in memory. actorSnapshotsRecord - // op is the actor's background operation in progress, or nil. An actor - // has at most one. The entry stays in memory while op is set. + // op is the actor's operation in progress, an upload or a detach, or nil. + // A background worker or an RPC call such as Detach runs it; an actor has + // at most one. The entry stays in memory while op is set. op *actorOp + // holds counts the Runs, Restores, Checkpoints, and PromoteSnapshots of + // the actor in progress on this node. While it is nonzero, atelet does not + // detach the actor or report its state, and the entry stays in memory. + holds int + // inFlight holds the snapshots of Checkpoints and PromoteSnapshots of the + // actor currently in progress on this node. newest includes them, so an + // older generation cannot start while a newer one is in flight. + inFlight []*SnapshotRef + // promotes counts the PromoteSnapshots of the actor in progress on this + // node. While it is nonzero, atelet does not start a background upload of + // the actor. + promotes int + // reportDeferred is set when a report of the actor's state is held back + // while holds is nonzero, so holdLocked queues it once holds reaches zero. + reportDeferred bool + // retryAt holds the actor's next background upload or detach back after + // one failed. The entry stays in memory while retryAt is set. + retryAt time.Time + // uploadingFailed is set after a background upload of Uploading failed and + // Uploading was kept to retry. The entry stays in memory while it is set. + uploadingFailed bool // unsaved is set while the snapshot state file lacks a change to the // entry because writing the file failed. The entry stays in memory while // unsaved is set. unsaved bool } + +// localNeeded reports whether CommitLocal should record snap as LatestLocal: +// LatestLocal does not already hold snap or a newer snapshot, and no other +// committed snapshot has a higher generation or the same generation under +// another UUID. +func (a *actorSnapshots) localNeeded(snap SnapshotRef) bool { + if a.LatestLocal != nil && a.LatestLocal.generation() >= snap.generation() { + return false + } + for _, r := range []*SnapshotRef{a.Uploading, a.LatestDurable} { + if r == nil { + continue + } + if r.generation() > snap.generation() { + return false + } + if r.generation() == snap.generation() && !sameUUID(r, &snap) { + return false + } + } + return true +} + +// reportDue reports whether atelet owes the control plane a report of the +// actor's state. +func (a *actorSnapshots) reportDue() bool { + return !a.ReportDueSince.IsZero() +} + +// latestDurableActor returns the actor LatestDurable belongs to, as its commit +// named it. A report is due only once LatestDurable is set. +func (a *actorSnapshots) latestDurableActor() resources.ActorRef { + if a.LatestDurable == nil { + return resources.ActorRef{} + } + return a.LatestDurable.Actor +} + +// uploadNeeded reports whether ref still needs uploading: neither it nor a +// newer snapshot is durable. +func (a *actorSnapshots) uploadNeeded(ref SnapshotRef) bool { + return a.LatestDurable == nil || a.LatestDurable.generation() < ref.generation() +} + +// canUpload reports whether ref is non-nil, still needs uploading, and names +// an object storage location where atelet can upload it. +func (a *actorSnapshots) canUpload(ref *SnapshotRef) bool { + if ref == nil || !a.uploadNeeded(*ref) || ref.uri() == "" { + return false + } + _, err := a.isLatestDurable(*ref) + return err == nil +} + +// localSuperseded reports whether ref is no longer kept as LatestLocal: +// LatestLocal was cleared or has moved on to a higher generation. +func (a *actorSnapshots) localSuperseded(ref *SnapshotRef) bool { + if sameUUID(a.LatestLocal, ref) { + return false + } + return a.LatestLocal == nil || a.LatestLocal.generation() > ref.generation() +} + +// keepUploading reports whether Uploading is still in use: an upload is +// running it, or it can still be uploaded and has not failed with a newer +// uploadable local snapshot ready to replace it. +func (a *actorSnapshots) keepUploading() bool { + if a.Uploading == nil { + return false + } + if a.runningUpload() != nil { + return true + } + if !a.canUpload(a.Uploading) { + return false + } + if a.uploadingFailed && a.localSuperseded(a.Uploading) && a.canUpload(a.LatestLocal) { + return false + } + return true +} + +// clearSupersededUploading clears Uploading when keepUploading reports it is +// no longer kept. It returns the local copy to delete, or nil, and whether +// Uploading was cleared. +func (a *actorSnapshots) clearSupersededUploading() (removed *SnapshotRef, cleared bool) { + if a.Uploading == nil || a.keepUploading() { + return nil, false + } + old := a.Uploading + a.Uploading = nil + a.uploadingFailed = false + if !a.localSuperseded(old) { + return nil, true + } + return old, true +} + +// nextUploadTarget returns the snapshot the next background upload makes +// durable. An earlier upload recorded in Uploading is retried or resumed when +// no upload goroutine is running it (runningUpload is nil) before moving on to +// LatestLocal, unless that upload failed and LatestLocal can replace it, or a +// newer snapshot is already durable. It returns nil when nothing is waiting to +// upload or that snapshot is already uploading. +func (a *actorSnapshots) nextUploadTarget() *SnapshotRef { + running := a.runningUpload() + if a.keepUploading() && running == nil { + return a.Uploading + } + if !sameUUID(running, a.LatestLocal) && a.canUpload(a.LatestLocal) { + return a.LatestLocal + } + return nil +} + +// isLatestDurable reports whether ref is LatestDurable. It fails with +// FailedPrecondition when LatestDurable has ref's UUID but a different +// generation, fidelity, or URI: the control plane named one snapshot two +// ways, so refuse rather than guess which is right. A later checkpoint clears +// the condition. +func (a *actorSnapshots) isLatestDurable(ref SnapshotRef) (bool, error) { + d := a.LatestDurable + if !sameUUID(d, &ref) { + return false, nil + } + if d.generation() != ref.generation() || d.uri() != ref.uri() { + return false, apierror.FailedPrecondition("snapshot %s is durable as generation %d at %q, but requested as generation %d at %q", + ref.uuid(), d.generation(), d.uri(), ref.generation(), ref.uri()) + } + if d.Snapshot.GetFidelity() != ref.Snapshot.GetFidelity() { + return false, apierror.FailedPrecondition("snapshot %s is durable with fidelity %s, but requested with fidelity %s", + ref.uuid(), d.Snapshot.GetFidelity(), ref.Snapshot.GetFidelity()) + } + return true, nil +} + +// recordDurable records that d is durable, unless that generation or a newer +// snapshot already is, and reports whether it recorded d. +func (a *actorSnapshots) recordDurable(d *SnapshotRef) bool { + if !a.uploadNeeded(*d) { + return false + } + a.LatestDurable = d + return true +} + +// runningUpload returns the snapshot an opUpload goroutine is currently +// uploading, or nil when no upload is running right now (such as while +// Uploading waits in retry backoff or after an atelet restart). +func (a *actorSnapshots) runningUpload() *SnapshotRef { + if a.op == nil || a.op.kind != opUpload { + return nil + } + return &a.op.target +} + +// runningDetach returns the snapshot an opDetach goroutine is currently +// detaching, or nil. +func (a *actorSnapshots) runningDetach() *SnapshotRef { + if a.op == nil || a.op.kind != opDetach { + return nil + } + return &a.op.target +} + +// opTarget returns the snapshot an operation in progress acts on, or nil. +func (a *actorSnapshots) opTarget() *SnapshotRef { + if a.op == nil { + return nil + } + return &a.op.target +} + +// newest returns the actor's snapshot with the highest generation, or nil +// when atelet holds no snapshot for the actor. It includes snapshots whose +// Checkpoint or PromoteSnapshot is in flight. +func (a *actorSnapshots) newest() *SnapshotRef { + var newest *SnapshotRef + consider := func(r *SnapshotRef) { + if r != nil && (newest == nil || r.generation() > newest.generation()) { + newest = r + } + } + consider(a.LatestLocal) + consider(a.Uploading) + consider(a.LatestDurable) + consider(a.opTarget()) + for _, r := range a.inFlight { + consider(r) + } + return newest +} + +// checkGeneration returns a FailedPrecondition error if snap has a stale +// generation for the actor: below the actor's newest snapshot, or equal to +// it under another snapshot_uuid. +func (a *actorSnapshots) checkGeneration(snap *ateletpb.Snapshot) error { + newest := a.newest() + if newest == nil { + return nil + } + switch { + case snap.GetGeneration() < newest.generation(): + return apierror.FailedPrecondition("snapshot %q has generation %d, below the actor's newest snapshot %q at generation %d", snap.GetSnapshotUuid(), snap.GetGeneration(), newest.uuid(), newest.generation()) + case snap.GetGeneration() == newest.generation() && snap.GetSnapshotUuid() != newest.uuid(): + return apierror.FailedPrecondition("snapshot %q has generation %d, which the actor's newest snapshot %q already holds", snap.GetSnapshotUuid(), snap.GetGeneration(), newest.uuid()) + default: + return nil + } +} + +// uploadDue returns when atelet next runs a background upload of the actor: +// uploadDelay after UndurableSince, or at retryAt if that is later. An upload +// or promote in progress can delay it. ok is false when there is nothing +// waiting to upload (see nextUploadTarget). +func (a *actorSnapshots) uploadDue(uploadDelay time.Duration) (due time.Time, ok bool) { + target := a.nextUploadTarget() + if target == nil { + return time.Time{}, false + } + since := a.UndurableSince + if since.IsZero() { + since = target.CommitTime + } + return later(since.Add(uploadDelay), a.retryAt), true +} + +// uploadSucceeded records that an upload succeeded at now. UndurableSince +// becomes zero, or now if a checkpoint committed while the upload ran is not +// durable yet. +func (a *actorSnapshots) uploadSucceeded(now time.Time) { + a.UndurableSince = time.Time{} + if a.LatestLocal != nil && a.uploadNeeded(*a.LatestLocal) { + a.UndurableSince = now + } +} + +// setLatestLocal records snap as the actor's latest local snapshot and +// schedules its detach at detachAt, or none when detachAt is zero. LatestLocal +// keeps snap's Snapshot and CommitTime only: UploadTime is only valid for +// durable snapshots. +func (a *actorSnapshots) setLatestLocal(snap SnapshotRef, detachAt time.Time) { + a.LatestLocal = &SnapshotRef{Snapshot: proto.CloneOf(snap.Snapshot), CommitTime: snap.CommitTime, Actor: snap.Actor} + a.DetachAt = detachAt +} + +// clearLatestLocal records that atelet holds no local snapshot of the actor, +// and zeroes DetachAt to match: with nothing left to detach, no detach is +// scheduled. +func (a *actorSnapshots) clearLatestLocal() { + a.LatestLocal = nil + a.DetachAt = time.Time{} +} + +// markStarted unschedules the actor's detach as a Run or Restore of the actor +// begins on this node: atelet does not detach the actor again until its next +// checkpoint schedules a detach. It reports whether a detach was scheduled. +func (a *actorSnapshots) markStarted() bool { + if a.DetachAt.IsZero() { + return false + } + a.DetachAt = time.Time{} + return true +} + +// detachScheduled reports whether a detach of the actor is scheduled: atelet +// holds a local snapshot of the actor, and no Run or Restore of the actor has +// begun since that snapshot was committed. A scheduled detach runs once it is +// due (see detachDue) and no Run, Restore, Checkpoint, or PromoteSnapshot of +// the actor is in progress. +func (a *actorSnapshots) detachScheduled() bool { + return a.LatestLocal != nil && !a.DetachAt.IsZero() +} + +// detachTarget returns LatestLocal, the snapshot a detach makes durable, and +// whether it is durable already. It fails with FailedPrecondition when no +// detach can make it durable until a commit replaces it: LatestDurable names +// the same snapshot differently, or LatestLocal is not durable and names no +// object storage location to upload it to, as after a pause today. +func (a *actorSnapshots) detachTarget() (target SnapshotRef, alreadyDurable bool, err error) { + target = *a.LatestLocal + alreadyDurable, err = a.isLatestDurable(target) + if err == nil && !alreadyDurable && target.uri() == "" { + err = apierror.FailedPrecondition("local snapshot %s names no object storage location to upload it to", target.uuid()) + } + return target, alreadyDurable, err +} + +// detachDue returns when atelet next checks whether to detach the actor: at +// DetachAt, or at retryAt if that is later. ok is false when there is nothing +// a detach can do: no detach is scheduled (see detachScheduled), or +// detachTarget fails, which only a commit can change. +func (a *actorSnapshots) detachDue() (due time.Time, ok bool) { + if !a.detachScheduled() { + return time.Time{}, false + } + if _, _, err := a.detachTarget(); err != nil { + return time.Time{}, false + } + return later(a.DetachAt, a.retryAt), true +} + +func later(t, u time.Time) time.Time { + if u.After(t) { + return u + } + return t +} diff --git a/cmd/atelet/internal/snapshottracker/background.go b/cmd/atelet/internal/snapshottracker/background.go index 303dfeb5b2..cc8a551428 100644 --- a/cmd/atelet/internal/snapshottracker/background.go +++ b/cmd/atelet/internal/snapshottracker/background.go @@ -18,44 +18,140 @@ import ( "context" "log/slog" "time" + + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/proto/ateletpb" ) -// Fixed until they become flags or per-request settings on UploadConfig. +// Background work settings. All but the Default ones, which atelet's +// --snapshot-upload-delay, --snapshot-detach-delay, --snapshot-upload-workers, +// and --snapshot-detach-workers override, are fixed until they become flags or +// per-request settings on UploadConfig. const ( + // DefaultUploadDelay is how long by default after the oldest + // not-yet-durable snapshot of an actor was taken atelet uploads the + // actor's latest local snapshot, under UPLOAD_MODE_DEFERRED. + DefaultUploadDelay = 5 * time.Minute + // DefaultDetachDelay is how long an actor stays suspended on this node by + // default, its latest snapshot kept locally, before atelet detaches it. + DefaultDetachDelay = 30 * time.Minute // snapshotRetryDelay is how long atelet holds an actor's background work // back after a failure. Each further consecutive failure doubles it, up to // maxSnapshotRetryDelay. snapshotRetryDelay = time.Minute maxSnapshotRetryDelay = 10 * time.Minute - // snapshotWorkers is how many background workers run at once on a node. - snapshotWorkers = 4 + // DefaultUploadWorkers and DefaultDetachWorkers are how many background + // uploads and detaches run at once on a node by default. Each kind has a + // queue and workers of its own, so neither waits behind the other. + DefaultUploadWorkers = 2 + DefaultDetachWorkers = 2 +) + +// opKind is what an operation on an actor's snapshots does. +type opKind int + +const ( + // opUpload makes a snapshot durable: a background upload. + opUpload opKind = iota + // opDetach makes the latest local snapshot durable, then deletes the + // actor's local snapshots: a background detach, or a Detach call. + opDetach + // numOpKinds is how many kinds of operation there are. + numOpKinds ) -// actorOp is an actor's background operation in progress. An actor has at -// most one. The operation holds the actor's lock only while it begins and -// ends, not while it runs, so Checkpoint and Terminate never wait for it. +func (k opKind) String() string { + switch k { + case opUpload: + return "upload" + case opDetach: + return "detach" + default: + return "unknown" + } +} + +// logOp logs how an upload or detach ended. Foreground failures are returned +// to the RPC caller, so only background failures are logged here: as a +// warning, or at info level once ctx has ended, as when Terminate or an atelet +// shutdown cancels the operation. +func logOp(ctx context.Context, actorUID string, kind opKind, target SnapshotRef, uploaded, background bool, elapsed time.Duration, err error) { + attrs := []any{ + slog.String("op", kind.String()), + slog.String("actorUID", actorUID), + slog.String("snapshot", target.uuid()), + slog.Int("generation", int(target.generation())), + slog.Bool("background", background), + slog.Duration("elapsed", elapsed), + } + switch { + case err == nil: + slog.InfoContext(ctx, "Snapshot operation finished", append(attrs, slog.Bool("uploaded", uploaded))...) + case !background: + case ctx.Err() != nil: + slog.InfoContext(ctx, "Snapshot operation canceled", append(attrs, slog.Any("err", err))...) + default: + slog.WarnContext(ctx, "Snapshot operation failed", append(attrs, slog.Any("err", err))...) + } +} + +// actorOp is an actor's operation in progress: an upload or a detach. An +// actor has at most one, of any kind. +// +// A worker runs it in the background once it is due. An atelet RPC such as +// Detach runs it in the RPC's goroutine instead, without going through the +// background queue. +// +// The operation holds the actor's lock only while it begins and ends, not +// while it runs: +// - Terminate never waits for it. +// - Checkpoint never waits for an upload. +// - Run, Restore, and Checkpoint wait for a detach to end, so it never +// deletes the actor's local snapshots underneath them. type actorOp struct { + kind opKind // target is the snapshot the operation acts on. target SnapshotRef // cancel ends the context the operation runs under. cancel context.CancelFunc + // done is closed once the operation has ended, and has finished any change + // it makes to the actor's local snapshots. + done chan struct{} } -// Start queues every actor with a snapshot state file on this node, so its -// background work resumes after an atelet restart, and starts the workers -// that run actors' background work. A worker reads each actor's file and -// queues the actor for when its next background work is due. +// Start reads each actor's snapshot state file on this node, queues the actor +// for when its next background upload or detach is due and for any report due, +// so its background work resumes after an atelet restart, and starts each +// queue's workers and the report workers. With instruments, it also adds each +// actor's undurable local snapshot to their tally before any worker runs. func (t *Tracker) Start() { - actorUIDs, err := savedActorUIDs() - if err != nil { - // Best-effort: each actor's next commit queues it again. - slog.Warn("Failed to find the actors with snapshot state to queue", slog.Any("err", err)) + if t.store != nil || t.reporting.reporter != nil || t.undurableActorsTally != nil { + actorUIDs, err := savedActorUIDs() + if err != nil { + // Best-effort: each actor's next commit queues it again. + slog.Warn("Failed to find the actors with snapshot state to queue", slog.Any("err", err)) + } + for _, actorUID := range actorUIDs { + if !t.locks.Lock(t.ctx, actorUID) { + break + } + if a := t.findLocked(actorUID); a != nil { + t.undurableActorsTally.observe(a) + t.enqueueSavedReportLocked(a) + t.requeueLocked(a) + } + t.unlock(actorUID) + } } - for _, actorUID := range actorUIDs { - t.queue.Add(actorUID) + for kind, n := range t.workers { + for range n { + t.workersDone.Go(func() { t.runWorker(opKind(kind)) }) + } } - for range t.workers { - t.workersDone.Go(t.runWorker) + if t.reporting.reporter != nil { + for range t.reporting.workers { + t.workersDone.Go(t.runReporter) + } } } @@ -75,87 +171,312 @@ func (t *Tracker) Wait(ctx context.Context) error { } } -// runWorker runs the background work of the actors it takes from the queue, -// until the queue shuts down. -func (t *Tracker) runWorker() { +// runWorker runs the background operations of kind of the actors it takes +// from kind's queue, until the queue shuts down. +func (t *Tracker) runWorker(kind opKind) { + q := t.queues[kind] for { - actorUID, shutdown := t.queue.Get() + actorUID, shutdown := q.Get() if shutdown { return } - t.syncActor(actorUID) - t.queue.Done(actorUID) + t.syncActor(actorUID, kind) + q.Done(actorUID) } } -// syncActor runs the actor's due background work, and otherwise queues the -// actor for when its next background work is due. -func (t *Tracker) syncActor(actorUID string) { +// syncActor runs the actor's background operation of kind if beginDueWork +// begins one. +func (t *Tracker) syncActor(actorUID string, kind opKind) { + if run := t.beginDueWork(actorUID, kind); run != nil { + run() + } +} + +// beginDueWork begins the actor's background operation of kind if one is due, +// and returns the func that runs it. Otherwise it returns nil, and queues the +// actor for when its next operations are due, unless an operation, Run, +// Restore, Checkpoint, or PromoteSnapshot in progress on the actor queues it +// again when it ends. +func (t *Tracker) beginDueWork(actorUID string, kind opKind) func() { if !t.locks.Lock(t.ctx, actorUID) { // atelet is shutting down. - return + return nil } defer t.unlock(actorUID) if t.ctx.Err() != nil { // atelet is shutting down. - return + return nil } a := t.findLocked(actorUID) if a == nil || a.op != nil { - // Forgotten, or an operation is in progress, whose caller queues the - // actor again when it ends. - return - } - // TODO: Run the actor's due background work here, such as uploading its - // latest local snapshot. Begin it with beginOpLocked and release the - // actor's lock while it runs, so Checkpoint and Terminate never wait for - // it. Then retake the lock and end it with endOpLocked. + // Forgotten, or an operation is already in progress on the actor: one + // a worker of the other queue began, or one an atelet RPC such as + // Detach runs in the RPC's goroutine. When that operation ends, + // requeueLocked queues the actor again. + return nil + } + now := t.clock.Now() + var cleanup func() + switch kind { + case opDetach: + if due, ok := t.detachDueLocked(a); ok && !due.After(now) { + if a.holds > 0 { + // A Run, Restore, Checkpoint, or PromoteSnapshot of the actor + // is in progress, and queues the actor again when it ends + // (see holdLocked). + return nil + } + if finish, err := t.beginDetachLocked(t.ctx, a, true); err == nil { + t.instruments.recordDetachOverdue(t.ctx, now.Sub(due)) + return func() { _, _ = finish() } + } + } + case opUpload: + if due, ok := t.uploadDueLocked(a); ok && !due.After(now) { + if a.promotes > 0 { + // A PromoteSnapshot of the actor is in progress, and queues + // the actor again when it ends (see holdLocked). + return nil + } + run := t.beginUploadLocked(t.ctx, a) + t.instruments.recordUploadOverdue(t.ctx, now.Sub(due)) + return run + } + // beginUploadLocked clears any superseded Uploading snapshot before + // starting the next upload. When no upload runs (for example after an + // immediate checkpoint or promote made a newer snapshot durable, + // leaving no upload due), clear a superseded Uploading snapshot here. + cleanup = t.cleanupSupersededUploadingLocked(a) + } t.requeueLocked(a) + return cleanup } -// requeueLocked queues the actor for when its next background work is due, -// then drops the actor's entry from memory unless the entry has an operation -// in progress or a change that failed to save. An actor whose directory is -// gone has nothing to save, so its entry is dropped. requeueLocked runs after -// a change to the actor is saved, so it must not change what the snapshot -// state file keeps. No background work is defined, so it queues nothing. +// nextDueLocked returns when the actor's next background detach or upload is +// due, if either is. +func (t *Tracker) nextDueLocked(a *actorSnapshots) (time.Time, bool) { + due, ok := t.detachDueLocked(a) + if upload, uploadOK := t.uploadDueLocked(a); uploadOK && (!ok || upload.Before(due)) { + due, ok = upload, true + } + return due, ok +} + +// requeueLocked queues the actor on each queue for when its next background +// operation of that kind is due, if it has one, then drops the actor's entry +// from memory unless the entry has an operation, or a Run, Restore, +// Checkpoint, or PromoteSnapshot, in progress, a retry hold, a failed upload +// kept to retry, or a change that failed to save. An entry whose Uploading is +// superseded is queued at once on the upload queue and kept in memory until an +// upload worker cleans it up. A save that finds the actor's directory gone +// leaves nothing unsaved. requeueLocked runs after a change to the actor is +// saved, so it must not change what the snapshot state file keeps. func (t *Tracker) requeueLocked(a *actorSnapshots) { - if a.op == nil && !a.unsaved { + now := t.clock.Now() + // CommitDurable and CommitPromoted leave a superseded Uploading snapshot + // for an upload worker to clear from the state file and delete from disk, + // rather than deleting its files on the RPC path. Queue the actor on the + // upload queue now so a worker cleans it up even when no upload is due. + supersededUploading := t.store != nil && a.Uploading != nil && !a.keepUploading() + if supersededUploading { + t.queues[opUpload].Add(a.actorUID) + } else if due, ok := t.uploadDueLocked(a); ok { + t.queues[opUpload].AddAfter(a.actorUID, due.Sub(now)) + } + if due, ok := t.detachDueLocked(a); ok { + t.queues[opDetach].AddAfter(a.actorUID, due.Sub(now)) + } + if !supersededUploading && a.op == nil && a.holds == 0 && a.retryAt.IsZero() && !a.uploadingFailed && !a.unsaved { t.dropLocked(a.actorUID) } } -// beginOpLocked records an operation on target as the actor's background -// operation in progress, which keeps the actor's entry in memory until the -// operation ends, and returns it with the context it runs under. The -// operation and its context are nil when the actor already has an operation -// in progress. The caller releases the actor's lock while the operation runs. -func (t *Tracker) beginOpLocked(a *actorSnapshots, target SnapshotRef) (context.Context, *actorOp) { +// beginOpLocked records an operation of kind on target as the actor's +// operation in progress, and returns it with the context it runs under: one +// derived from ctx, which Forget cancels. +// +// The operation and its context are nil when the actor already has an +// operation in progress, started by a background worker or an atelet RPC. +// +// The operation keeps the actor's entry in memory until it ends. The caller +// releases the actor's lock while the operation runs, and closes the +// operation's done channel once it has ended. +func (t *Tracker) beginOpLocked(ctx context.Context, a *actorSnapshots, kind opKind, target SnapshotRef) (context.Context, *actorOp) { if a.op != nil { return nil, nil } - ctx, cancel := context.WithCancel(t.ctx) - a.op = &actorOp{target: *target.clone(), cancel: cancel} + ctx, cancel := context.WithCancel(ctx) + cloned := target.clone() + if kind == opUpload { + a.Uploading = cloned + a.uploadingFailed = false + } + a.op = &actorOp{kind: kind, target: *cloned, cancel: cancel, done: make(chan struct{})} t.addLocked(a) return ctx, a.op } -// endOpLocked ends op and returns the actor's entry, or nil once Forget has -// abandoned op: the caller must then leave the actor's state and local -// snapshots alone. Otherwise the caller records the operation's outcome on -// the entry, saves the entry if that changed it, and calls requeueLocked. -func (t *Tracker) endOpLocked(actorUID string, op *actorOp) *actorSnapshots { - op.cancel() +// opEntryLocked returns the actor's entry while op is still its operation in +// progress, or nil once Forget has abandoned op. +func (t *Tracker) opEntryLocked(actorUID string, op *actorOp) *actorSnapshots { a := t.findLocked(actorUID) if a == nil || a.op != op { return nil } - a.op = nil return a } -// abandonOpLocked cancels the actor's background operation in progress, if -// any, without waiting for it to end. +// abandonedError returns what an operation on the actor fails with once Forget +// has abandoned it: opErr if the operation failed already, and otherwise +// Aborted. +func abandonedError(actorUID string, opErr error) error { + if opErr != nil { + return opErr + } + return apierror.Aborted("actor %s was forgotten during the operation", actorUID) +} + +// updateOp runs fn under the actor's lock and saves the actor's state while op +// is still the actor's operation in progress. Once Forget has abandoned op, it +// leaves the actor alone and fails with abandonedError. +func (t *Tracker) updateOp(actorUID string, op *actorOp, opErr error, fn func(a *actorSnapshots)) error { + t.lock(actorUID) + defer t.unlock(actorUID) + a := t.opEntryLocked(actorUID, op) + if a == nil { + return abandonedError(actorUID, opErr) + } + fn(a) + t.saveLocked(a) + return nil +} + +// endOpLocked ends op and returns the actor's entry, or nil once Forget has +// abandoned op. +func (t *Tracker) endOpLocked(actorUID string, op *actorOp) *actorSnapshots { + op.cancel() + a := t.opEntryLocked(actorUID, op) + if a != nil { + a.op = nil + } + return a +} + +// endOp ends op, queues the actor for its next background operation and any +// report due, and returns the actor's state. Once Forget has abandoned op, it +// leaves the actor alone and fails with abandonedError; otherwise it fails +// with opErr if non-nil. +func (t *Tracker) endOp(actorUID string, op *actorOp, opErr error) (*ateletpb.ActorSnapshotState, error) { + t.lock(actorUID) + defer t.unlock(actorUID) + a := t.endOpLocked(actorUID, op) + if a == nil { + return nil, abandonedError(actorUID, opErr) + } + t.requeueLocked(a) + t.queueReportLocked(a) + if opErr != nil { + return nil, opErr + } + return t.stateLocked(a), nil +} + +// waitIdleLocked waits, as long as ctx allows, until the actor has no +// operation in progress (whether started by a background worker or an atelet +// RPC), and returns the actor's entry, which may be nil. It releases the +// actor's lock while it waits, so the actor can change meanwhile, and holds +// it again when it returns. check, if not nil, vets the entry on every pass: +// an error from it is returned at once, without waiting further. +func (t *Tracker) waitIdleLocked(ctx context.Context, actorUID string, check func(*actorSnapshots) error) (*actorSnapshots, error) { + for { + a := t.findLocked(actorUID) + if a != nil && check != nil { + if err := check(a); err != nil { + return nil, err + } + } + if a == nil || a.op == nil { + return a, nil + } + if err := t.waitOpLocked(ctx, actorUID, a.op); err != nil { + return nil, err + } + } +} + +// waitForDetachLocked waits, as long as ctx allows, until no detach of the +// actor is in progress, and returns the actor's entry, which may be nil. It +// releases the actor's lock while it waits, and holds it again when it +// returns. check, if not nil, vets the entry on every pass: an error from it +// is returned at once, without waiting further. Checkpoint, Run, and Restore +// call this first so they never race with a detach pruning the actor's local +// directory, without waiting behind an upload. +func (t *Tracker) waitForDetachLocked(ctx context.Context, actorUID string, check func(*actorSnapshots) error) (*actorSnapshots, error) { + for { + a := t.findLocked(actorUID) + if a != nil && check != nil { + if err := check(a); err != nil { + return nil, err + } + } + if a == nil || a.runningDetach() == nil { + return a, nil + } + if err := t.waitOpLocked(ctx, actorUID, a.op); err != nil { + return nil, err + } + } +} + +// waitForDetachOrSnapshotUploadLocked waits, as long as ctx allows, until no +// detach of the actor and no background upload of snap is in progress, and +// returns the actor's entry, which may be nil. It checks snap's generation on +// every pass, so a stale generation fails at once without waiting, and one +// superseded while waiting fails as soon as the wait ends. PromoteSnapshot +// calls this first so it never races with a detach or uploads the same +// snapshot twice. +func (t *Tracker) waitForDetachOrSnapshotUploadLocked(ctx context.Context, actorUID string, snap SnapshotRef) (*actorSnapshots, error) { + for { + a := t.findLocked(actorUID) + if a != nil { + if err := a.checkGeneration(snap.Snapshot); err != nil { + return nil, err + } + } + if a == nil || a.op == nil { + return a, nil + } + switch a.op.kind { + case opDetach: + case opUpload: + if !sameUUID(&a.op.target, &snap) { + return a, nil + } + default: + return a, nil + } + if err := t.waitOpLocked(ctx, actorUID, a.op); err != nil { + return nil, err + } + } +} + +// waitOpLocked waits, releasing the actor's lock meanwhile, until op has +// ended or ctx does. +func (t *Tracker) waitOpLocked(ctx context.Context, actorUID string, op *actorOp) error { + t.unlock(actorUID) + defer t.lock(actorUID) + select { + case <-op.done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// abandonOpLocked cancels the actor's operation in progress, if any, without +// waiting for it to end. func (t *Tracker) abandonOpLocked(actorUID string) { // An entry with an operation in progress is always in memory. t.mu.Lock() @@ -167,14 +488,18 @@ func (t *Tracker) abandonOpLocked(actorUID string) { } } -// backOffLocked records a failure of the actor's background work, and returns -// when the actor may try again: later with each consecutive failure. -func (t *Tracker) backOffLocked(actorUID string) time.Time { - return t.clock.Now().Add(t.retries.When(actorUID)) +// backOffLocked records a failure of the actor's background work, and holds +// that work back until the time it returns: later with each consecutive +// failure. +func (t *Tracker) backOffLocked(a *actorSnapshots) time.Time { + a.retryAt = t.clock.Now().Add(t.retries.When(a.actorUID)) + return a.retryAt } -// resetBackoffLocked forgets the actor's consecutive failures, so its next -// failure holds it back for snapshotRetryDelay again. -func (t *Tracker) resetBackoffLocked(actorUID string) { - t.retries.Forget(actorUID) +// resetBackoffLocked lifts the hold backOffLocked put on the actor's +// background work, and forgets its consecutive failures, so its next failure +// holds it back for snapshotRetryDelay again. +func (t *Tracker) resetBackoffLocked(a *actorSnapshots) { + a.retryAt = time.Time{} + t.retries.Forget(a.actorUID) } diff --git a/cmd/atelet/internal/snapshottracker/background_test.go b/cmd/atelet/internal/snapshottracker/background_test.go index eeb0842794..18facfc447 100644 --- a/cmd/atelet/internal/snapshottracker/background_test.go +++ b/cmd/atelet/internal/snapshottracker/background_test.go @@ -17,9 +17,12 @@ package snapshottracker import ( "context" "errors" + "log/slog" "os" "path/filepath" "slices" + "strings" + "sync" "testing" "time" @@ -61,7 +64,7 @@ func beginOp(t *testing.T, tr *Tracker, target SnapshotRef) (context.Context, *a t.Helper() tr.lock(trackedActorUID) defer tr.unlock(trackedActorUID) - ctx, op := tr.beginOpLocked(tr.findOrAddLocked(trackedActorUID), target) + ctx, op := tr.beginOpLocked(tr.ctx, tr.findOrAddLocked(trackedActorUID), opUpload, target) if op == nil { t.Fatal("beginOpLocked: the actor already has an operation in progress") } @@ -97,11 +100,13 @@ func inMemory(tr *Tracker) bool { func TestWorkersStopWhenContextEnds(t *testing.T) { useTempNodeDirs(t) ctx, cancel := context.WithCancel(t.Context()) - tr := New(ctx, clocktesting.NewFakeClock(time.Unix(1_700_000_000, 0))) + tr := New(ctx, nil, clocktesting.NewFakeClock(time.Unix(1_700_000_000, 0))) tr.Start() tr.lock(trackedActorUID) defer tr.unlock(trackedActorUID) - tr.queue.Add(trackedActorUID) + for _, q := range tr.queues { + q.Add(trackedActorUID) + } cancel() returnsPromptly(t, "Wait", func() { if err := tr.Wait(t.Context()); err != nil { @@ -150,31 +155,36 @@ func TestEntryStaysInMemoryDuringOp(t *testing.T) { } } -// Start queues each actor whose snapshot state file survived an atelet -// restart, and skips actor directories without one. A node with no actor -// directories yet queues nothing. +// Start reads each actor's snapshot state file once, queues the actor only on +// the queues whose work is due, and drops its entry from memory. It skips actor +// directories without a state file, and queues nothing on an empty node. func TestStartQueuesActorsWithSnapshotState(t *testing.T) { useTempNodeDirs(t) // queued starts a tracker, as a restarted atelet does, and returns the - // actors it queued. - queued := func() []string { - tr := newTracker(t) - tr.workers = 0 // Leave the queue for the test to read. + // actors it queued ready to run on each queue. + queued := func() [numOpKinds][]string { + tr := newStoreTracker(t, newFakeSnapshotStore()) + setWorkers(tr, 0, 0) // Leave the queues for the test to read. tr.Start() - var actorUIDs []string - for tr.queue.Len() > 0 { - actorUID, _ := tr.queue.Get() - actorUIDs = append(actorUIDs, actorUID) - tr.queue.Done(actorUID) + if len(tr.actors) != 0 { + t.Errorf("in-memory actors after Start = %d, want 0", len(tr.actors)) + } + var actorUIDs [numOpKinds][]string + for k, q := range tr.queues { + for q.Len() > 0 { + actorUID, _ := q.Get() + actorUIDs[k] = append(actorUIDs[k], actorUID) + q.Done(actorUID) + } + slices.Sort(actorUIDs[k]) } - slices.Sort(actorUIDs) return actorUIDs } - if got := queued(); len(got) != 0 { + if got := queued(); len(got[opUpload]) != 0 || len(got[opDetach]) != 0 { t.Errorf("queued on a node with no actor directories = %v, want none", got) } - for _, actorUID := range []string{"actor-a", "actor-b", "actor-without-state"} { + for _, actorUID := range []string{"actor-both", "actor-upload", "actor-detach", "actor-durable", "actor-without-state"} { if err := os.MkdirAll(ateletpath.ActorPath(actorUID), 0o700); err != nil { t.Fatal(err) } @@ -182,11 +192,28 @@ func TestStartQueuesActorsWithSnapshotState(t *testing.T) { if err := os.WriteFile(filepath.Join(nodepath.ActorsDir, "not-an-actor"), nil, 0o600); err != nil { t.Fatal(err) } - before := newTracker(t) - before.CommitLocal("actor-a", testRef("s1", 1, time.Now())) - before.CommitDurable("actor-b", testRef("s2", 2, time.Now())) - if got, want := queued(), []string{"actor-a", "actor-b"}; !slices.Equal(got, want) { - t.Errorf("queued after a restart = %v, want %v", got, want) + // Commit a detach delay before the restarted tracker's clock, so any + // scheduled upload or detach is due at Start. + t0 := time.Unix(1_700_000_000, 0).Add(-testDetachDelay) + before := New(t.Context(), newFakeSnapshotStore(), clocktesting.NewFakeClock(t0), WithUploadDelay(testUploadDelay), WithDetachDelay(testDetachDelay)) + before.CommitLocal("actor-both", testRef("s1", 1, t0)) + before.CommitLocal("actor-upload", testRef("s2", 1, t0)) + endStart, err := before.BeginStart(t.Context(), "actor-upload") + if err != nil { + t.Fatalf("BeginStart: %v", err) + } + endStart() + s3 := testRef("s3", 1, t0) + before.CommitDurable("actor-detach", s3) + before.CommitLocal("actor-detach", s3) + before.CommitDurable("actor-durable", testRef("s4", 1, t0)) + + got := queued() + if want := []string{"actor-both", "actor-upload"}; !slices.Equal(got[opUpload], want) { + t.Errorf("upload queue after restart = %v, want %v", got[opUpload], want) + } + if want := []string{"actor-both", "actor-detach"}; !slices.Equal(got[opDetach], want) { + t.Errorf("detach queue after restart = %v, want %v", got[opDetach], want) } } @@ -213,16 +240,17 @@ func TestForgetAbandonsInFlightOp(t *testing.T) { if endOp(tr, op) { t.Error("endOpLocked = true after Forget, want the operation abandoned") } - if got := tr.State(trackedActorUID); !proto.Equal(got, &ateletpb.ActorSnapshotState{}) { - t.Errorf("state after forget = %v, want empty", got) + if got, want := tr.State(trackedActorUID), (&ateletpb.ActorSnapshotState{Uploading: s1.report()}); !proto.Equal(got, want) { + t.Errorf("state after forget = %v, want %v: only the operation begun after Forget", got, want) } if !endOp(tr, next) { t.Error("endOpLocked = false for the operation begun after Forget, want the abandoned one's end to leave it alone") } } -// A Checkpoint's commits, PromoteSnapshot's, and State neither wait for the -// actor's background operation in progress nor cancel it. +// A Checkpoint's commits and LocalSnapshotsToKeep, PromoteSnapshot's commit, +// and State neither wait for the actor's background operation in progress nor +// cancel it. func TestCommitsDoNotWaitForInFlightOp(t *testing.T) { useActorDir(t) tr := newTracker(t) @@ -236,7 +264,7 @@ func TestCommitsDoNotWaitForInFlightOp(t *testing.T) { call func() }{ {"CommitLocal", func() { tr.CommitLocal(trackedActorUID, testRef("s2", 2, t0)) }}, - {"ForgetLocal", func() { tr.ForgetLocal(trackedActorUID) }}, + {"LocalSnapshotsToKeep", func() { tr.LocalSnapshotsToKeep(trackedActorUID) }}, {"CommitDurable", func() { tr.CommitDurable(trackedActorUID, testRef("s3", 3, t0)) }}, {"CommitPromoted", func() { tr.CommitPromoted(trackedActorUID, testRef("s4", 4, t0)) }}, {"State", func() { tr.State(trackedActorUID) }}, @@ -260,7 +288,7 @@ func TestBeginOpLockedAllowsOneOpPerActor(t *testing.T) { _, op := beginOp(t, tr, s1) tr.lock(trackedActorUID) - _, second := tr.beginOpLocked(tr.findOrAddLocked(trackedActorUID), s1) + _, second := tr.beginOpLocked(tr.ctx, tr.findOrAddLocked(trackedActorUID), opUpload, s1) tr.unlock(trackedActorUID) if second != nil { t.Error("beginOpLocked began a second operation for the actor, want none") @@ -268,7 +296,7 @@ func TestBeginOpLockedAllowsOneOpPerActor(t *testing.T) { const otherActorUID = "other-actor-uid" tr.lock(otherActorUID) - _, other := tr.beginOpLocked(tr.findOrAddLocked(otherActorUID), s1) + _, other := tr.beginOpLocked(tr.ctx, tr.findOrAddLocked(otherActorUID), opUpload, s1) tr.unlock(otherActorUID) if other == nil { t.Error("beginOpLocked for another actor = nil, want an operation") @@ -285,17 +313,17 @@ func TestBeginOpLockedAllowsOneOpPerActor(t *testing.T) { } // A failure holds the actor's background work back for snapshotRetryDelay, -// doubling with each consecutive failure up to maxSnapshotRetryDelay. Making -// the actor's latest snapshot durable, or forgetting the actor, starts the -// doubling over. +// doubling with each consecutive failure up to maxSnapshotRetryDelay. A +// deferred checkpoint, making the actor's latest snapshot durable, or +// forgetting the actor starts the doubling over. func TestTrackerBackoff(t *testing.T) { useActorDir(t) clk := clocktesting.NewFakeClock(time.Unix(1_700_000_000, 0)) - tr := New(t.Context(), clk) + tr := New(t.Context(), nil, clk) fail := func() time.Duration { tr.lock(trackedActorUID) defer tr.unlock(trackedActorUID) - return tr.backOffLocked(trackedActorUID).Sub(clk.Now()) + return tr.backOffLocked(tr.findOrAddLocked(trackedActorUID)).Sub(clk.Now()) } for i, want := range []time.Duration{time.Minute, 2 * time.Minute, 4 * time.Minute, 8 * time.Minute, 10 * time.Minute, 10 * time.Minute} { if got := fail(); got != want { @@ -307,8 +335,9 @@ func TestTrackerBackoff(t *testing.T) { name string call func() }{ - {"CommitDurable", func() { tr.CommitDurable(trackedActorUID, testRef("s1", 1, clk.Now())) }}, - {"CommitPromoted", func() { tr.CommitPromoted(trackedActorUID, testRef("s2", 2, clk.Now())) }}, + {"CommitLocal", func() { tr.CommitLocal(trackedActorUID, testRef("s1", 1, clk.Now())) }}, + {"CommitDurable", func() { tr.CommitDurable(trackedActorUID, testRef("s2", 2, clk.Now())) }}, + {"CommitPromoted", func() { tr.CommitPromoted(trackedActorUID, testRef("s3", 3, clk.Now())) }}, {"Forget", func() { if err := tr.Forget(trackedActorUID); err != nil { t.Fatalf("Forget: %v", err) @@ -323,3 +352,92 @@ func TestTrackerBackoff(t *testing.T) { } } } + +// recordingHandler is a slog.Handler that keeps the records it handles. +type recordingHandler struct { + mu sync.Mutex + records []slog.Record +} + +func (h *recordingHandler) Enabled(context.Context, slog.Level) bool { return true } + +func (h *recordingHandler) Handle(_ context.Context, r slog.Record) error { + h.mu.Lock() + defer h.mu.Unlock() + h.records = append(h.records, r) + return nil +} + +func (h *recordingHandler) WithAttrs([]slog.Attr) slog.Handler { return h } + +func (h *recordingHandler) WithGroup(string) slog.Handler { return h } + +// logOp logs a finished operation at info level, a failed background one as a +// warning, and a background one canceled by its context at info level. A +// failed foreground operation goes back to its caller unlogged. +func TestLogOp(t *testing.T) { + canceled, cancel := context.WithCancel(context.Background()) + cancel() + for _, tc := range []struct { + name string + ctx context.Context + background bool + err error + wantLevel slog.Level + wantMsg string + }{ + {name: "finished", ctx: context.Background(), background: true, wantLevel: slog.LevelInfo, wantMsg: "Snapshot operation finished"}, + {name: "background failure", ctx: context.Background(), background: true, err: errors.New("boom"), wantLevel: slog.LevelWarn, wantMsg: "Snapshot operation failed"}, + {name: "background canceled", ctx: canceled, background: true, err: context.Canceled, wantLevel: slog.LevelInfo, wantMsg: "Snapshot operation canceled"}, + {name: "foreground failure", ctx: context.Background(), err: errors.New("boom")}, + } { + t.Run(tc.name, func(t *testing.T) { + h := &recordingHandler{} + defaultLogger := slog.Default() + slog.SetDefault(slog.New(h)) + t.Cleanup(func() { slog.SetDefault(defaultLogger) }) + + logOp(tc.ctx, trackedActorUID, opUpload, testRef("s1", 1, time.Time{}), false, tc.background, time.Second, tc.err) + h.mu.Lock() + defer h.mu.Unlock() + if tc.wantMsg == "" { + if len(h.records) != 0 { + t.Errorf("logged %q, want nothing", h.records[0].Message) + } + return + } + if len(h.records) != 1 || h.records[0].Message != tc.wantMsg || h.records[0].Level != tc.wantLevel { + t.Errorf("logged %v, want one %v record %q", h.records, tc.wantLevel, tc.wantMsg) + } + }) + } +} + +// A failed background detach logs a warning rather than a cancellation: +// finishDetach logs before endOp ends the detach's context. +func TestBackgroundDetachFailureLogsWarning(t *testing.T) { + store := newFakeSnapshotStore() + store.setErr(errors.New("boom")) + tr := newIdleTestTracker(t, store) + t0 := tr.clock.Now() + tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) + + h := &recordingHandler{} + defaultLogger := slog.Default() + slog.SetDefault(slog.New(h)) + t.Cleanup(func() { slog.SetDefault(defaultLogger) }) + + stepTo(tr, t0.Add(testDetachDelay)) + processNext(t, tr, opDetach) + h.mu.Lock() + defer h.mu.Unlock() + var got []string + for _, r := range h.records { + if strings.HasPrefix(r.Message, "Snapshot operation") { + got = append(got, r.Level.String()+" "+r.Message) + } + } + if want := []string{"WARN Snapshot operation failed"}; !slices.Equal(got, want) { + t.Errorf("logged %v, want %v", got, want) + } +} diff --git a/cmd/atelet/internal/snapshottracker/detach.go b/cmd/atelet/internal/snapshottracker/detach.go new file mode 100644 index 0000000000..b2f60b78ad --- /dev/null +++ b/cmd/atelet/internal/snapshottracker/detach.go @@ -0,0 +1,165 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package snapshottracker + +import ( + "context" + + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/proto/ateletpb" +) + +// beginDetachLocked begins a detach of the actor's latest local snapshot, +// under a context derived from ctx, and returns the func that finishes it +// once the caller releases the actor's lock. It begins nothing, and fails, +// when no detach can make the snapshot durable (see detachTarget). +func (t *Tracker) beginDetachLocked(ctx context.Context, a *actorSnapshots, background bool) (func() (*ateletpb.ActorSnapshotState, error), error) { + target, alreadyDurable, err := a.detachTarget() + if err != nil { + return nil, err + } + opCtx, op := t.beginOpLocked(ctx, a, opDetach, target) + return func() (*ateletpb.ActorSnapshotState, error) { + return t.finishDetach(opCtx, a.actorUID, op, alreadyDurable, background) + }, nil +} + +// Detach makes the actor's latest local snapshot durable, then deletes the +// actor's local snapshots. It is a no-op when atelet holds no local snapshot +// for the actor. It fails with Aborted while a Run, Restore, Checkpoint, or +// PromoteSnapshot of the actor is in progress on this node, or once a Run or +// Restore of the actor has begun since its latest local snapshot, and with +// FailedPrecondition when no detach can make the snapshot durable. It runs in +// the calling goroutine rather than through the background queue, once any +// operation in progress on the actor has ended. +func (t *Tracker) Detach(ctx context.Context, actorUID string) (*ateletpb.ActorSnapshotState, error) { + if t.store == nil { + return nil, apierror.Unimplemented("atelet has no snapshot store to detach actors with") + } + t.lock(actorUID) + // Wait for any operation in progress (from a background worker or + // another RPC call) to end first: a detach may leave nothing to detach, + // and an upload of the latest local snapshot nothing to upload. + a, err := t.waitIdleLocked(ctx, actorUID, checkNotHeld) + if err != nil { + t.unlock(actorUID) + return nil, err + } + if a == nil || a.LatestLocal == nil { + // No local snapshot is stored on this node, so there is no work to be + // done. + st := &ateletpb.ActorSnapshotState{} + if a != nil { + if a.reportDue() { + t.clearReportDueLocked(a) + } + st = t.stateLocked(a) + } + t.unlock(actorUID) + return st, nil + } + if !a.detachScheduled() { + // A Run or Restore of the actor has begun since its latest local + // snapshot, which a workload restored from it may still use. + t.unlock(actorUID) + return nil, apierror.Aborted("actor %s is in use on this node", actorUID) + } + finish, err := t.beginDetachLocked(ctx, a, false) + t.unlock(actorUID) + if err != nil { + return nil, err + } + return finish() +} + +// checkNotHeld fails with Aborted while a Run, Restore, Checkpoint, or +// PromoteSnapshot of the actor is in progress on this node. +func checkNotHeld(a *actorSnapshots) error { + if a.holds > 0 { + return apierror.Aborted("actor %s is in use on this node", a.actorUID) + } + return nil +} + +// finishDetach runs op, a detach of the actor's latest local snapshot: it +// uploads the snapshot unless already durable, then deletes the actor's local +// snapshots. background reports whether a worker runs it, rather than a Detach +// call; a worker's detach makes a report of what it changed due, while a +// Detach call returns the new state and cancels any pending report. A detach +// Forget abandons leaves the actor alone, and fails. +func (t *Tracker) finishDetach(ctx context.Context, actorUID string, op *actorOp, alreadyDurable, background bool) (*ateletpb.ActorSnapshotState, error) { + defer close(op.done) + uploaded := false + target := op.target + detachStart := t.clock.Now() + var err error + if !alreadyDurable { + err = ctx.Err() + if err == nil { + uploaded, _, err = t.uploadSnapshot(ctx, actorUID, target, background) + } + if err == nil { + err = t.updateOp(actorUID, op, nil, func(a *actorSnapshots) { + if t.recordUploadLocked(a, target, uploaded) && background { + t.markReportDueLocked(a) + } + }) + } + } + + // The upload is recorded before anything is deleted, and LatestLocal is + // cleared only once its files are gone. After a crash or a failed prune in + // between, the detach is retried, finds the snapshot durable, and only + // prunes. Until then LatestLocal can name a half-deleted copy, which is + // safe because the snapshot is durable: Restore reads snapshot.object + // when the local copy is unusable. The prune runs with op in progress + // (before op.done is closed), so Run, Restore, and Checkpoint wait for it. + // TODO: Notify ateapi of the upcoming detach before deleting the actor's + // local snapshots, so it stops sending Restores that rely on them. + recordDetach := err == nil + if recordDetach { + detachStart = t.clock.Now() + // Forget or the caller may have ended ctx as the upload finished. + err = ctx.Err() + } + if err == nil { + err = PruneLocalCheckpoints(ctx, actorUID) + } + if abErr := t.updateOp(actorUID, op, err, func(a *actorSnapshots) { + switch { + case err == nil: + t.clearLatestLocalLocked(a) + if background { + t.markReportDueLocked(a) + } else { + t.cancelReportLocked(a) + } + case ctx.Err() == nil: + // Detach again once the retry delay has passed, and no sooner + // than the detach was due. + a.DetachAt = later(a.DetachAt, t.backOffLocked(a)) + } + }); abErr != nil { + err = abErr + } + + // Log and record before endOp, which ends ctx. + elapsed := t.clock.Since(detachStart) + logOp(ctx, actorUID, op.kind, target, uploaded, background, elapsed, err) + if background && recordDetach { + t.instruments.recordDetach(ctx, target, elapsed, err) + } + return t.endOp(actorUID, op, err) +} diff --git a/cmd/atelet/internal/snapshottracker/detach_test.go b/cmd/atelet/internal/snapshottracker/detach_test.go new file mode 100644 index 0000000000..64a5e69603 --- /dev/null +++ b/cmd/atelet/internal/snapshottracker/detach_test.go @@ -0,0 +1,958 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package snapshottracker + +import ( + "context" + "errors" + "os" + "slices" + "testing" + "time" + + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/proto/ateletpb" + "google.golang.org/grpc/codes" + "google.golang.org/protobuf/proto" +) + +// eventually polls cond until it holds, and fails the test if it does not +// within 10 seconds. +func eventually(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for !cond() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting until %s", what) + } + time.Sleep(time.Millisecond) + } +} + +// detachDueOf returns when atelet detaches the actor, or zero if it does not. +func detachDueOf(tr *Tracker, actorUID string) time.Time { + tr.lock(actorUID) + defer tr.unlock(actorUID) + if a := tr.findLocked(actorUID); a != nil { + due, _ := a.detachDue() + return due + } + return time.Time{} +} + +// detachNow advances the clock to when the tracked actor's detach is due, and +// waits for a worker to detach it. +func detachNow(t *testing.T, tr *Tracker) { + t.Helper() + due := detachDueOf(tr, trackedActorUID) + if due.IsZero() { + t.Fatal("no detach scheduled") + } + stepTo(tr, due) + eventually(t, "the actor is detached", func() bool { + tr.lock(trackedActorUID) + defer tr.unlock(trackedActorUID) + a := tr.findLocked(trackedActorUID) + return a != nil && a.op == nil && a.LatestLocal == nil + }) +} + +// startDetach runs a Detach of the tracked actor in the background, and +// returns a channel that receives its error. +func startDetach(t *testing.T, tr *Tracker) <-chan error { + t.Helper() + detached := make(chan error, 1) + go func() { + _, err := tr.Detach(t.Context(), trackedActorUID) + detached <- err + }() + return detached +} + +// A detach makes the latest local snapshot durable and drops it, once any +// upload in progress has ended, and is a no-op when atelet holds no local +// snapshot for the actor. +func TestTrackerDetach(t *testing.T) { + t.Run("requested", func(t *testing.T) { + store := newFakeSnapshotStore() + tr := newTestTracker(t, store) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + + st, err := tr.Detach(t.Context(), trackedActorUID) + if err != nil { + t.Fatalf("detach: %v", err) + } + if got, wantCalls := store.uploads(), []string{"s1"}; !slices.Equal(got, wantCalls) { + t.Errorf("uploads = %v, want %v", got, wantCalls) + } + if st.GetLatestLocal() != nil { + t.Errorf("state = %v, want no local snapshot", st) + } + durable := st.GetLatestDurable() + if got := durable.GetSnapshot().GetFidelity(); got != ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { + t.Errorf("latest_durable fidelity = %v, want the MEMORY it holds", got) + } + if durable.GetDurableCompleteTime() == nil || durable.GetLocalCommitTime() == nil { + t.Errorf("latest_durable = %v, want its commit and upload times", durable) + } + if !detachDueOf(tr, trackedActorUID).IsZero() { + t.Error("a detach is scheduled after a detach") + } + }) + + t.Run("no-op when no local snapshot", func(t *testing.T) { + store := newFakeSnapshotStore() + tr := newTestTracker(t, store) + st, err := tr.Detach(t.Context(), trackedActorUID) + if err != nil { + t.Fatalf("detach of untracked actor: %v", err) + } + if !proto.Equal(st, &ateletpb.ActorSnapshotState{}) { + t.Errorf("state = %v, want empty", st) + } + if got := store.uploads(); len(got) != 0 { + t.Errorf("uploads = %v, want none", got) + } + }) + + t.Run("by the timer", func(t *testing.T) { + store := newFakeSnapshotStore() + tr := newIdleTestTracker(t, store) + setWorkers(tr, 0, DefaultDetachWorkers) + tr.Start() + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + // With no upload workers, the detach runs and makes s1 durable itself. + detachNow(t, tr) + if got, want := store.uploads(), []string{"s1"}; !slices.Equal(got, want) { + t.Errorf("uploads = %v, want %v", got, want) + } + st := tr.State(trackedActorUID) + if st.GetLatestLocal() != nil || uuidOf(st.GetLatestDurable()) != "s1" { + t.Errorf("state = %v, want s1 durable and nothing local", st) + } + }) + + t.Run("already durable, as after an immediate checkpoint", func(t *testing.T) { + store := newFakeSnapshotStore() + tr := newTestTracker(t, store) + local := testRef("s1", 1, tr.clock.Now()) + durable := local + durable.UploadTime = local.CommitTime.Add(time.Second) + tr.CommitDurable(trackedActorUID, durable) + tr.CommitLocal(trackedActorUID, local) + detachNow(t, tr) + if got := store.uploads(); len(got) != 0 { + t.Errorf("uploads = %v, want none for an already durable snapshot", got) + } + st := tr.State(trackedActorUID) + if st.GetLatestLocal() != nil || uuidOf(st.GetLatestDurable()) != "s1" { + t.Errorf("state = %v, want s1 durable and nothing local", st) + } + if got := timeOf(st.GetLatestDurable().GetDurableCompleteTime()); !got.Equal(durable.UploadTime) { + t.Errorf("durable_complete_time = %v, want %v from the checkpoint", got, durable.UploadTime) + } + }) + + t.Run("already durable, as after a promote", func(t *testing.T) { + store := newFakeSnapshotStore() + tr := newTestTracker(t, store) + local := testRef("s1", 1, tr.clock.Now()) + tr.CommitLocal(trackedActorUID, local) + + promoted := testRef("s1", 1, time.Time{}) + promoted.UploadTime = local.CommitTime.Add(time.Minute) + tr.CommitPromoted(trackedActorUID, promoted) + + detachNow(t, tr) + if got := store.uploads(); len(got) != 0 { + t.Errorf("uploads = %v, want none for a promoted snapshot", got) + } + st := tr.State(trackedActorUID) + if st.GetLatestLocal() != nil || uuidOf(st.GetLatestDurable()) != "s1" { + t.Errorf("state = %v, want s1 durable and nothing local", st) + } + if got := timeOf(st.GetLatestDurable().GetDurableCompleteTime()); !got.Equal(promoted.UploadTime) { + t.Errorf("durable_complete_time = %v, want %v from the promote", got, promoted.UploadTime) + } + }) + + t.Run("by the timer after the upload", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newTestTracker(t, store) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + uploadNow(t, tr, store, nil) + uploadTime := tr.State(trackedActorUID).GetLatestDurable().GetDurableCompleteTime() + + detachNow(t, tr) + if got := len(store.uploads()); got != 1 { + t.Errorf("%d store calls, want only the background upload's", got) + } + st := tr.State(trackedActorUID) + if st.GetLatestLocal() != nil || !proto.Equal(st.GetLatestDurable().GetDurableCompleteTime(), uploadTime) { + t.Errorf("state = %v, want nothing local and s1 durable since %v", st, uploadTime) + } + }) + + t.Run("waits for the upload of its snapshot", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newTestTracker(t, store) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + startUpload(t, tr, store) + detached := startDetach(t, tr) + + store.release <- nil + if err := <-detached; err != nil { + t.Fatalf("detach: %v", err) + } + if got := len(store.uploads()); got != 1 { + t.Errorf("%d store calls, want only the in-progress upload's", got) + } + if got := tr.State(trackedActorUID).GetLatestDurable(); got.GetDurableCompleteTime() == nil { + t.Errorf("latest_durable = %v, want the background upload's time", got) + } + }) + + t.Run("waits for an older snapshot's upload", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newTestTracker(t, store) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + startUpload(t, tr, store) + tr.CommitLocal(trackedActorUID, testRef("s2", 2, tr.clock.Now())) + store.setBlocking(false) + detached := startDetach(t, tr) + select { + case err := <-detached: + t.Fatalf("detach returned (%v) during the upload of s1", err) + case <-time.After(50 * time.Millisecond): + } + + store.release <- nil + if err := <-detached; err != nil { + t.Fatalf("detach: %v", err) + } + if got, want := store.uploads(), []string{"s1", "s2"}; !slices.Equal(got, want) { + t.Errorf("uploads = %v, want %v", got, want) + } + st := tr.State(trackedActorUID) + if st.GetLatestLocal() != nil || uuidOf(st.GetLatestDurable()) != "s2" { + t.Errorf("state = %v, want s2 durable and nothing local", st) + } + }) + + // A refused detach keeps the local snapshot and uploads nothing. + for _, tc := range []struct { + name string + modifyLocal func(*SnapshotRef) + setup func(t *testing.T, tr *Tracker) + wantCode codes.Code + }{ + { + name: "refused after a start", + setup: func(t *testing.T, tr *Tracker) { + end, err := tr.BeginStart(t.Context(), trackedActorUID) + if err != nil { + t.Fatalf("beginStart: %v", err) + } + end() + }, + wantCode: codes.Aborted, + }, + { + name: "refused while a start is in progress", + setup: func(t *testing.T, tr *Tracker) { + end, err := tr.BeginStart(t.Context(), trackedActorUID) + if err != nil { + t.Fatalf("beginStart: %v", err) + } + t.Cleanup(end) + }, + wantCode: codes.Aborted, + }, + { + name: "refused while a checkpoint is in progress", + setup: func(t *testing.T, tr *Tracker) { + end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, testRef("s2", 2, time.Time{}).Snapshot) + if err != nil { + t.Fatalf("beginCheckpoint: %v", err) + } + t.Cleanup(end) + }, + wantCode: codes.Aborted, + }, + { + name: "refused when durable generation mismatches local", + setup: func(t *testing.T, tr *Tracker) { + tr.CommitDurable(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + tr.CommitLocal(trackedActorUID, testRef("s1", 2, tr.clock.Now())) + }, + wantCode: codes.FailedPrecondition, + }, + { + name: "refused when durable URI mismatches local", + modifyLocal: func(local *SnapshotRef) { + local.Snapshot.Object.SnapshotUri += "-other" + }, + setup: func(t *testing.T, tr *Tracker) { + tr.CommitDurable(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + }, + wantCode: codes.FailedPrecondition, + }, + { + name: "refused without an object storage location", + modifyLocal: func(local *SnapshotRef) { + local.Snapshot.Object = nil + }, + wantCode: codes.FailedPrecondition, + }, + } { + t.Run(tc.name, func(t *testing.T) { + store := newFakeSnapshotStore() + tr := newTestTracker(t, store) + local := testRef("s1", 1, tr.clock.Now()) + if tc.modifyLocal != nil { + tc.modifyLocal(&local) + } + tr.CommitLocal(trackedActorUID, local) + if tc.setup != nil { + tc.setup(t, tr) + } + if _, err := tr.Detach(t.Context(), trackedActorUID); apierror.Code(err) != tc.wantCode { + t.Errorf("detach = %v, want %v", err, tc.wantCode) + } + if got := store.uploads(); len(got) != 0 { + t.Errorf("uploads = %v, want none", got) + } + if got := uuidOf(tr.State(trackedActorUID).GetLatestLocal()); got != "s1" { + t.Errorf("latest_local = %q, want s1 kept", got) + } + }) + } +} + +// A local snapshot no detach can make durable, as a pause takes today, never +// comes due for a background detach, rather than failing it again and again. +// A checkpoint that replaces it schedules a detach again. +func TestTrackerSkipsUndetachableSnapshots(t *testing.T) { + for _, tc := range []struct { + name string + commit func(tr *Tracker) + }{ + { + name: "no object storage location", + commit: func(tr *Tracker) { + local := testRef("s1", 1, tr.clock.Now()) + local.Snapshot.Object = nil + tr.CommitLocal(trackedActorUID, local) + }, + }, + { + name: "durable elsewhere", + commit: func(tr *Tracker) { + tr.CommitDurable(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + local := testRef("s1", 1, tr.clock.Now()) + local.Snapshot.Object.SnapshotUri += "-other" + tr.CommitLocal(trackedActorUID, local) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + store := newFakeSnapshotStore() + tr := newTestTracker(t, store) + tc.commit(tr) + if due := detachDueOf(tr, trackedActorUID); !due.IsZero() { + t.Fatalf("detach due at %v, want none", due) + } + + tr.CommitLocal(trackedActorUID, testRef("s2", 2, tr.clock.Now())) + detachNow(t, tr) + if got, want := store.uploads(), []string{"s2"}; !slices.Equal(got, want) { + t.Errorf("uploads = %v, want %v", got, want) + } + }) + } +} + +// BeginStart unschedules the actor's detach, and saves that, before the start +// begins, so neither a restart during the start nor a failed start brings the +// detach back: only the actor's next checkpoint schedules one again. While a +// Checkpoint is in progress, a worker that finds the detach due skips it +// without changing DetachAt or queuing the actor again. The end of the +// checkpoint queues the actor again, and the detach runs then. +func TestTrackerDetachTimer(t *testing.T) { + store := newFakeSnapshotStore() + tr := newIdleTestTracker(t, store) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + + endStart, err := tr.BeginStart(t.Context(), trackedActorUID) + if err != nil { + t.Fatalf("beginStart: %v", err) + } + if due := detachDueOf(newTracker(t), trackedActorUID); !due.IsZero() { + t.Fatalf("detachDue on a fresh tracker during the start = %v, want zero from the state file", due) + } + endStart() + if due := detachDueOf(tr, trackedActorUID); !due.IsZero() { + t.Fatalf("detachDue after the start = %v, want zero whether or not it succeeded", due) + } + + tr.CommitLocal(trackedActorUID, testRef("s2", 2, tr.clock.Now())) + due := detachDueOf(tr, trackedActorUID) + endCheckpoint, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, testRef("s3", 3, time.Time{}).Snapshot) + if err != nil { + t.Fatalf("beginCheckpoint: %v", err) + } + stepTo(tr, due) + processNext(t, tr, opDetach) + if got := detachDueOf(tr, trackedActorUID); !got.Equal(due) { + t.Errorf("detachDue after a worker found the actor held = %v, want %v kept", got, due) + } + if n := tr.queues[opDetach].Len(); n != 0 { + t.Errorf("actors queued for a detach while the actor is held = %d, want none until the hold ends", n) + } + if got := uuidOf(tr.State(trackedActorUID).GetLatestLocal()); got != "s2" { + t.Fatalf("latest_local = %q while the actor is held, want s2 kept", got) + } + + endCheckpoint() + processNext(t, tr, opDetach) + if got := tr.State(trackedActorUID).GetLatestLocal(); got != nil { + t.Errorf("latest_local = %v once the checkpoint ended, want unset", got) + } + if got, want := store.uploads(), []string{"s2"}; !slices.Equal(got, want) { + t.Errorf("uploads = %v, want %v", got, want) + } +} + +// A start that cannot save that it unscheduled the actor's detach fails before +// anything starts, and leaves the detach scheduled. +func TestTrackerStartFailsIfItCannotSave(t *testing.T) { + tr := newIdleTestTracker(t, newFakeSnapshotStore()) + stateFile := ateletpath.ActorSnapshotStateFile(trackedActorUID) + // A directory where the state file goes makes every save fail, so the + // entry stays in memory. + if err := os.MkdirAll(stateFile, 0o700); err != nil { + t.Fatal(err) + } + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + due := detachDueOf(tr, trackedActorUID) + if due.IsZero() { + t.Fatal("no detach scheduled after the checkpoint") + } + + if _, err := tr.BeginStart(t.Context(), trackedActorUID); apierror.Code(err) != codes.Unavailable { + t.Fatalf("beginStart = %v, want Unavailable", err) + } + if got := detachDueOf(tr, trackedActorUID); !got.Equal(due) { + t.Errorf("detachDue after the failed start = %v, want %v kept", got, due) + } + if holds := tr.actors[trackedActorUID].holds; holds != 0 { + t.Errorf("holds after the failed start = %d, want none", holds) + } + + // Once saves work again, the start goes ahead. + if err := os.Remove(stateFile); err != nil { + t.Fatal(err) + } + end, err := tr.BeginStart(t.Context(), trackedActorUID) + if err != nil { + t.Fatalf("beginStart once saves work: %v", err) + } + end() + if due := detachDueOf(newTracker(t), trackedActorUID); !due.IsZero() { + t.Errorf("detachDue from the state file = %v, want zero", due) + } +} + +// A restore or a checkpoint that arrives during a detach waits for it to end. +func TestTrackerWaitsForDetach(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newTestTracker(t, store) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + results := map[string]<-chan error{"detach": startDetach(t, tr)} + waitStarted(t, store) + + for name, begin := range map[string]func(context.Context, string) (func(), error){ + "beginStart": tr.BeginStart, + "beginCheckpoint": func(ctx context.Context, uid string) (func(), error) { + return tr.BeginCheckpoint(ctx, uid, testRef("s2", 2, time.Time{}).Snapshot) + }, + } { + result := make(chan error, 1) + results[name] = result + go func() { + end, err := begin(t.Context(), trackedActorUID) + if err == nil { + end() + } + result <- err + }() + } + for name, result := range results { + select { + case err := <-result: + t.Fatalf("%s returned (%v) during a detach", name, err) + case <-time.After(50 * time.Millisecond): + } + } + + store.release <- nil + for name, result := range results { + select { + case err := <-result: + if err != nil { + t.Errorf("%s: %v", name, err) + } + case <-time.After(10 * time.Second): + t.Fatalf("%s did not return after the detach", name) + } + } + if got := tr.State(trackedActorUID).GetLatestLocal(); got != nil { + t.Errorf("latest_local = %v after the detach, want unset", got) + } +} + +// BeginCheckpoint and BeginPromote fail at once during a detach when the +// requested snapshot is already stale, and fail once the detach ends when a +// newer snapshot was committed while they waited. +func TestTrackerDetachWaitChecksGeneration(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newTestTracker(t, store) + tr.CommitLocal(trackedActorUID, testRef("s1", 2, tr.clock.Now())) + detached := startDetach(t, tr) + waitStarted(t, store) + + stale := testRef("s0", 1, time.Time{}) + returnsPromptly(t, "BeginCheckpoint of stale generation during detach", func() { + if _, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, stale.Snapshot); apierror.Code(err) != codes.FailedPrecondition { + t.Errorf("BeginCheckpoint = %v, want FailedPrecondition", err) + } + }) + returnsPromptly(t, "BeginPromote of stale generation during detach", func() { + if _, _, err := tr.BeginPromote(t.Context(), trackedActorUID, stale); apierror.Code(err) != codes.FailedPrecondition { + t.Errorf("BeginPromote = %v, want FailedPrecondition", err) + } + }) + + superseded := testRef("s2", 3, time.Time{}) + results := make(map[string]chan error) + for name, begin := range map[string]func() (func(), error){ + "beginCheckpoint": func() (func(), error) { + return tr.BeginCheckpoint(t.Context(), trackedActorUID, superseded.Snapshot) + }, + "beginPromote": func() (func(), error) { + _, end, err := tr.BeginPromote(t.Context(), trackedActorUID, superseded) + return end, err + }, + } { + result := make(chan error, 1) + results[name] = result + go func() { + end, err := begin() + if err == nil { + end() + } + result <- err + }() + } + for name, result := range results { + select { + case err := <-result: + t.Fatalf("%s returned (%v) during a detach", name, err) + case <-time.After(50 * time.Millisecond): + } + } + + tr.CommitDurable(trackedActorUID, testRef("s3", 4, tr.clock.Now())) + store.release <- nil + if err := <-detached; err != nil { + t.Fatalf("detach: %v", err) + } + for name, result := range results { + select { + case err := <-result: + if apierror.Code(err) != codes.FailedPrecondition { + t.Errorf("%s = %v, want FailedPrecondition", name, err) + } + case <-time.After(10 * time.Second): + t.Fatalf("%s did not return after the detach", name) + } + } +} + +// A failed detach keeps the local snapshot, and is retried no sooner than the +// retry delay after it, nor than it was due. The upload is held back the retry +// delay too. +func TestTrackerFailedDetach(t *testing.T) { + t.Run("requested", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setErr(errors.New("boom")) + tr := newIdleTestTracker(t, store) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now().Add(-testUploadDelay))) + due := detachDueOf(tr, trackedActorUID) + if _, err := tr.Detach(t.Context(), trackedActorUID); err == nil { + t.Fatal("detach succeeded, want the store's error") + } + retryAt := tr.clock.Now().Add(testRetryDelay) + st := tr.State(trackedActorUID) + if uuidOf(st.GetLatestLocal()) != "s1" || !timeOf(st.GetUploadDueTime()).Equal(retryAt) { + t.Errorf("state = %v, want s1 kept and its upload due at %v", st, retryAt) + } + // testRetryDelay is shorter than testDetachDelay. + if got := detachDueOf(tr, trackedActorUID); !got.Equal(due) { + t.Errorf("detach due at %v after the failed one, want %v as scheduled", got, due) + } + }) + + t.Run("background", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setErr(errors.New("boom")) + tr := newIdleTestTracker(t, store) + setWorkers(tr, 0, DefaultDetachWorkers) + tr.Start() + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + due := detachDueOf(tr, trackedActorUID) + stepTo(tr, due) + retryAt := due.Add(testRetryDelay) + eventually(t, "the failed detach is retried later", func() bool { + return detachDueOf(tr, trackedActorUID).Equal(retryAt) + }) + if got := uuidOf(tr.State(trackedActorUID).GetLatestLocal()); got != "s1" { + t.Errorf("latest_local = %q after the failed detach, want s1 kept", got) + } + + store.setErr(nil) + detachNow(t, tr) + if got := tr.State(trackedActorUID).GetLatestLocal(); got != nil { + t.Errorf("latest_local = %v after the retried detach, want unset", got) + } + }) +} + +// A detach whose prune fails keeps LatestLocal, now durable, and is retried: +// the retry finds the snapshot durable and only prunes. +func TestTrackerDetachRetriesFailedPrune(t *testing.T) { + store := newFakeSnapshotStore() + tr := newTestTracker(t, store) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + // A file where the local checkpoint directory belongs fails the prune. + if err := os.WriteFile(ateletpath.LocalCheckpointsDir(trackedActorUID), nil, 0o600); err != nil { + t.Fatal(err) + } + if _, err := tr.Detach(t.Context(), trackedActorUID); err == nil { + t.Fatal("detach succeeded, want the prune's error") + } + if st := tr.State(trackedActorUID); uuidOf(st.GetLatestLocal()) != "s1" || uuidOf(st.GetLatestDurable()) != "s1" { + t.Errorf("state = %v, want s1 still local and now durable", st) + } + + if err := os.Remove(ateletpath.LocalCheckpointsDir(trackedActorUID)); err != nil { + t.Fatal(err) + } + detachNow(t, tr) + if got, want := store.uploads(), []string{"s1"}; !slices.Equal(got, want) { + t.Errorf("uploads = %v, want %v, as the retry only prunes", got, want) + } + if got := tr.State(trackedActorUID).GetLatestLocal(); got != nil { + t.Errorf("latest_local = %v after the retried detach, want unset", got) + } +} + +// Detach fails on a tracker without a store, rather than uploading through it. +func TestTrackerDetachWithoutStore(t *testing.T) { + useActorDir(t) + tr := newTracker(t) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + if _, err := tr.Detach(t.Context(), trackedActorUID); apierror.Code(err) != codes.Unimplemented { + t.Errorf("detach = %v, want Unimplemented", err) + } + if got := uuidOf(tr.State(trackedActorUID).GetLatestLocal()); got != "s1" { + t.Errorf("latest_local = %q, want s1 kept", got) + } +} + +// A Checkpoint holds off detaches even when atelet tracked nothing of the +// actor as it began, as on the actor's first checkpoint on this node. +func TestTrackerHoldsFirstCheckpoint(t *testing.T) { + store := newFakeSnapshotStore() + tr := newTestTracker(t, store) + s1 := testRef("s1", 1, tr.clock.Now()) + end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, s1.Snapshot) + if err != nil { + t.Fatalf("beginCheckpoint: %v", err) + } + tr.CommitLocal(trackedActorUID, s1) + if _, err := tr.Detach(t.Context(), trackedActorUID); apierror.Code(err) != codes.Aborted { + t.Errorf("detach during the checkpoint = %v, want Aborted", err) + } + end() + if _, err := tr.Detach(t.Context(), trackedActorUID); err != nil { + t.Errorf("detach after the checkpoint: %v", err) + } +} + +// A start when no upload is due, or a checkpoint replacing the local snapshot, +// lifts the backoff of the actor's failed detaches, which have nothing left to +// retry, so the entry leaves memory. +func TestTrackerLiftsBackoffWithDetach(t *testing.T) { + for name, drop := range map[string]func(t *testing.T, tr *Tracker){ + "start": func(t *testing.T, tr *Tracker) { + end, err := tr.BeginStart(t.Context(), trackedActorUID) + if err != nil { + t.Fatalf("beginStart: %v", err) + } + end() + }, + "checkpoint": func(t *testing.T, tr *Tracker) { + tr.CommitLocal(trackedActorUID, testRef("s2", 2, tr.clock.Now())) + }, + } { + t.Run(name, func(t *testing.T) { + store := newFakeSnapshotStore() + tr := newTestTracker(t, store) + s1 := testRef("s1", 1, tr.clock.Now()) + tr.CommitDurable(trackedActorUID, s1) + tr.CommitLocal(trackedActorUID, s1) + // A file where the local checkpoint directory belongs fails the prune. + if err := os.WriteFile(ateletpath.LocalCheckpointsDir(trackedActorUID), nil, 0o600); err != nil { + t.Fatal(err) + } + if _, err := tr.Detach(t.Context(), trackedActorUID); err == nil { + t.Fatal("detach succeeded, want the prune's error") + } + if !inMemory(tr) { + t.Fatal("entry not in memory after a failed detach, want it kept for its backoff") + } + drop(t, tr) + if inMemory(tr) { + t.Error("entry in memory, want it dropped with no backoff left") + } + }) + } +} + +// Forget during a Run, Restore, or Checkpoint drops the actor for good: the +// start's end leaves it forgotten. +func TestTrackerForgetDuringStart(t *testing.T) { + store := newFakeSnapshotStore() + tr := newTestTracker(t, store) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + end, err := tr.BeginStart(t.Context(), trackedActorUID) + if err != nil { + t.Fatalf("beginStart: %v", err) + } + if err := tr.Forget(trackedActorUID); err != nil { + t.Fatalf("Forget: %v", err) + } + end() + if st := tr.State(trackedActorUID); !proto.Equal(st, &ateletpb.ActorSnapshotState{}) { + t.Errorf("state = %v, want empty", st) + } + if _, err := os.Stat(ateletpath.ActorSnapshotStateFile(trackedActorUID)); !os.IsNotExist(err) { + t.Errorf("state file after the start ended: stat = %v, want none", err) + } + if inMemory(tr) { + t.Error("entry in memory after Forget, want none") + } +} + +// Forget does not wait for a detach in progress: it cancels the detach, which +// fails and leaves the forgotten actor's local snapshots for Terminate to +// prune. +func TestTrackerForgetAbandonsDetach(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newTestTracker(t, store) + if err := os.MkdirAll(ateletpath.LocalSnapshotDir(trackedActorUID, "s1"), 0o700); err != nil { + t.Fatal(err) + } + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + detached := startDetach(t, tr) + waitStarted(t, store) + + returnsPromptly(t, "Forget during a detach", func() { + if err := tr.Forget(trackedActorUID); err != nil { + t.Errorf("Forget: %v", err) + } + }) + if err := <-detached; !errors.Is(err, context.Canceled) { + t.Errorf("detach = %v, want %v", err, context.Canceled) + } + if st := tr.State(trackedActorUID); !proto.Equal(st, &ateletpb.ActorSnapshotState{}) { + t.Errorf("state after forget = %v, want empty", st) + } + if _, err := os.Stat(ateletpath.LocalSnapshotDir(trackedActorUID, "s1")); err != nil { + t.Errorf("s1 after the cancelled detach: %v, want it left for Terminate to prune", err) + } +} + +// Detach runs in the calling goroutine, so it does not wait behind other +// actors' background work, and a worker that takes the actor afterwards finds +// nothing left to do. +func TestTrackerForegroundDetachBypassesQueue(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newIdleTestTracker(t, store) + setWorkers(tr, 1, 0) + tr.Start() + for _, id := range []string{"actor-1", "actor-2", "actor-3"} { + if err := os.MkdirAll(ateletpath.ActorPath(id), 0o700); err != nil { + t.Fatal(err) + } + } + + taken := tr.clock.Now().Add(-testUploadDelay) + tr.CommitLocal("actor-1", testRef("a1-s1", 1, taken)) + if got := waitStarted(t, store); got != "a1-s1" { + t.Fatalf("started = %q, want a1-s1", got) + } + // actor-2's upload is due too, but the only worker is busy. + tr.CommitLocal("actor-2", testRef("a2-s1", 1, taken)) + + store.setBlocking(false) + st, err := tr.Detach(t.Context(), "actor-2") + if err != nil { + t.Fatalf("detach of actor-2: %v", err) + } + if got := waitStarted(t, store); got != "a2-s1" { + t.Fatalf("started = %q, want a2-s1", got) + } + if st.GetLatestLocal() != nil || uuidOf(st.GetLatestDurable()) != "a2-s1" { + t.Errorf("actor-2 state = %v, want detached with a2-s1 durable", st) + } + + done1 := opDone(t, tr, "actor-1") + store.release <- nil + waitClosed(t, done1, "actor-1's upload") + // actor-3 is queued behind actor-2, so its upload starts only once the + // worker has taken actor-2. + tr.CommitLocal("actor-3", testRef("a3-s1", 1, taken)) + if got := waitStarted(t, store); got != "a3-s1" { + t.Fatalf("started = %q, want a3-s1", got) + } + wantCalls := []string{"a1-s1", "a2-s1", "a3-s1"} + if got := store.uploads(); !slices.Equal(got, wantCalls) { + t.Errorf("uploads = %v, want %v", got, wantCalls) + } +} + +// The queue holds an actor until the earliest time it was queued for. A worker +// that takes the actor before its detach is due, as after a later checkpoint +// moved the detach out, leaves the actor's local snapshot and queues it again +// for when its detach is due. Both snapshots are durable, so no upload is due +// either. +func TestTrackerRequeuesUntilDetachIsDue(t *testing.T) { + store := newFakeSnapshotStore() + tr := newIdleTestTracker(t, store) + t0 := tr.clock.Now() + commitImmediate := func(ref SnapshotRef) { + tr.CommitDurable(trackedActorUID, ref) + tr.CommitLocal(trackedActorUID, ref) + } + commitImmediate(testRef("s1", 1, t0)) + stepTo(tr, t0.Add(time.Hour)) + commitImmediate(testRef("s2", 2, tr.clock.Now())) + + stepTo(tr, t0.Add(testDetachDelay)) + processNext(t, tr, opDetach) + if got := uuidOf(tr.State(trackedActorUID).GetLatestLocal()); got != "s2" { + t.Errorf("latest_local = %q before the detach is due, want s2", got) + } + if got := store.uploads(); len(got) != 0 { + t.Errorf("uploads = %v, want none: both snapshots are durable", got) + } + + stepTo(tr, t0.Add(time.Hour+testDetachDelay)) + processNext(t, tr, opDetach) + if got := tr.State(trackedActorUID).GetLatestLocal(); got != nil { + t.Errorf("latest_local = %v once the detach was due, want unset", got) + } +} + +// cancelingStore is a Store whose upload succeeds just after cancel ends the +// detach's context, as Forget or the Detach caller can. +type cancelingStore struct{ cancel context.CancelFunc } + +func (s *cancelingStore) UploadLocalSnapshot(context.Context, string, SnapshotRef) (bool, error) { + s.cancel() + return true, nil +} + +// A detach whose context ends as its upload succeeds records the upload, but +// deletes no local snapshot, and fails with the context's error. +func TestTrackerDetachDoesNotPruneOnceCancelled(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + tr := newIdleTestTracker(t, &cancelingStore{cancel: cancel}) + if err := os.MkdirAll(ateletpath.LocalSnapshotDir(trackedActorUID, "s1"), 0o700); err != nil { + t.Fatal(err) + } + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + + if _, err := tr.Detach(ctx, trackedActorUID); !errors.Is(err, context.Canceled) { + t.Fatalf("detach = %v, want %v", err, context.Canceled) + } + if _, err := os.Stat(ateletpath.LocalSnapshotDir(trackedActorUID, "s1")); err != nil { + t.Errorf("s1 after the cancelled detach: %v, want it kept", err) + } + if st := tr.State(trackedActorUID); uuidOf(st.GetLatestLocal()) != "s1" || uuidOf(st.GetLatestDurable()) != "s1" { + t.Errorf("state = %v, want s1 still local, and durable", st) + } +} + +// TestTrackerDetachActsForTheActor checks that a detach's upload names the +// actor the snapshot was committed for, across an atelet restart that reloads +// the snapshot state file, and that a promote keeps it on the local copy. +func TestTrackerDetachActsForTheActor(t *testing.T) { + t.Run("after a restart", func(t *testing.T) { + store := newFakeSnapshotStore() + first := newIdleTestTracker(t, store) + ref := testRef("s1", 1, first.clock.Now()) + ref.Actor = testActor + first.CommitLocal(trackedActorUID, ref) + + // A new tracker over the same node directories reloads the state file. + tr := newStoreTracker(t, store) + tr.Start() + if _, err := tr.Detach(t.Context(), trackedActorUID); err != nil { + t.Fatalf("detach: %v", err) + } + if got := store.actorOf("s1"); got != testActor { + t.Errorf("upload acted for %+v, want %+v", got, testActor) + } + }) + + t.Run("kept by a promote", func(t *testing.T) { + store := newFakeSnapshotStore() + tr := newTestTracker(t, store) + ref := testRef("s1", 1, tr.clock.Now()) + ref.Actor = testActor + tr.CommitLocal(trackedActorUID, ref) + promoted := testRef("s1", 1, time.Time{}) + promoted.Actor = testActor + tr.CommitPromoted(trackedActorUID, promoted) + + tr.lock(trackedActorUID) + got := tr.findLocked(trackedActorUID).LatestLocal.Actor + tr.unlock(trackedActorUID) + if got != testActor { + t.Errorf("local copy's actor after promote = %+v, want %+v", got, testActor) + } + }) +} diff --git a/cmd/atelet/internal/snapshottracker/lifecycle.go b/cmd/atelet/internal/snapshottracker/lifecycle.go index 3a35b3ddd2..0144bcc938 100644 --- a/cmd/atelet/internal/snapshottracker/lifecycle.go +++ b/cmd/atelet/internal/snapshottracker/lifecycle.go @@ -15,94 +15,323 @@ package snapshottracker import ( + "context" + "slices" + "time" + + "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/proto/ateletpb" + "google.golang.org/protobuf/proto" ) -// CommitLocal records snap, just stored locally under UPLOAD_MODE_DEFERRED, as -// the actor's latest local snapshot, and returns the actor's state. +// CommitLocal records snap, just stored in the actor's local checkpoint +// directory by a Checkpoint in either upload mode, as the actor's latest local +// snapshot, schedules the actor's detach and, if snap needs an upload (see +// uploadDue), its upload, and returns the actor's state. A commit of a +// snapshot already recorded or superseded by a newer committed snapshot leaves +// the state as it is. An older snapshot's upload in progress runs on, but an +// Uploading that keepUploading no longer keeps is dropped. The Checkpoint then +// prunes the local snapshots LocalSnapshotsToKeep no longer names. func (t *Tracker) CommitLocal(actorUID string, snap SnapshotRef) *ateletpb.ActorSnapshotState { t.lock(actorUID) defer t.unlock(actorUID) a := t.findOrAddLocked(actorUID) - a.LatestLocal = snap.clone() - // TODO: Use succeededLocked here once UPLOAD_MODE_DEFERRED uploads are - // implemented: a deferred checkpoint is then a successful end state, which - // starts the actor's backoff over. + if !a.localNeeded(snap) { + if a.reportDue() { + t.clearReportDueLocked(a) + } else { + t.requeueLocked(a) + } + return t.stateLocked(a) + } + t.setLatestLocalLocked(a, snap) + if a.UndurableSince.IsZero() && a.uploadNeeded(snap) { + a.UndurableSince = snap.CommitTime + } + // Drop a failed upload that snap replaces in this save rather than leave + // it for a worker: keepUploading tells it apart by uploadingFailed, which + // the state file does not keep, so an atelet restart would resume it from + // a local copy the Checkpoint's prune deletes. + a.clearSupersededUploading() + // Start the backoff over unless Uploading is still kept and snap is not + // the next upload target: Uploading's retry then still waits it out. + if !a.keepUploading() || sameUUID(a.nextUploadTarget(), &snap) { + t.resetBackoffLocked(a) + } + t.cancelReportLocked(a) t.saveLocked(a) t.requeueLocked(a) return t.stateLocked(a) } -// CommitDurable records snap, uploaded during Checkpoint under -// UPLOAD_MODE_IMMEDIATE, as the actor's latest snapshot. The actor keeps no -// local snapshot. It returns the actor's state. +// CommitDurable records snap, just uploaded by a Checkpoint under +// UPLOAD_MODE_IMMEDIATE, as the actor's latest durable snapshot, and returns +// the actor's state. A commit of a snapshot already durable or older than +// LatestDurable leaves the state as it is. It drops an older LatestLocal, +// which snap supersedes, so the Checkpoint's prune deletes it even if no local +// copy of snap is kept. The Checkpoint then records its local copy with +// CommitLocal, if it manages to keep one. An older snapshot's upload in +// progress runs on, and removes that snapshot's local copy when it ends (see +// removeSupersededLocalCopy). func (t *Tracker) CommitDurable(actorUID string, snap SnapshotRef) *ateletpb.ActorSnapshotState { t.lock(actorUID) defer t.unlock(actorUID) a := t.findOrAddLocked(actorUID) + if !a.uploadNeeded(snap) { + if a.reportDue() { + t.clearReportDueLocked(a) + } else { + t.requeueLocked(a) + } + return t.stateLocked(a) + } a.LatestDurable = snap.clone() - // TODO: change this when we add detach. - a.LatestLocal = nil - t.succeededLocked(a) + if a.LatestLocal != nil && a.LatestLocal.generation() < snap.generation() { + a.clearLatestLocal() + } + t.cancelReportLocked(a) + t.durableCommittedLocked(a) return t.stateLocked(a) } -// ForgetLocal drops the actor's latest local snapshot. Call it before deleting -// the actor's local checkpoints, so the state never names a deleted snapshot, -// even if the caller then fails. -func (t *Tracker) ForgetLocal(actorUID string) { +// LocalSnapshotsToKeep returns the actor's local snapshots atelet still needs: +// LatestLocal, and Uploading while a background upload is reading it or will +// retry it. A Checkpoint prunes every other local snapshot once it has +// committed its own. Until the next commit, a background upload only reads one +// of these, so the prune never deletes a snapshot an upload needs. +func (t *Tracker) LocalSnapshotsToKeep(actorUID string) []string { t.lock(actorUID) defer t.unlock(actorUID) a := t.findLocked(actorUID) - if a == nil || a.LatestLocal == nil { - return + if a == nil { + return nil } - a.LatestLocal = nil - t.addLocked(a) - t.saveLocked(a) - t.requeueLocked(a) + var keep []string + if a.LatestLocal != nil { + keep = append(keep, a.LatestLocal.uuid()) + } + if a.keepUploading() && !sameUUID(a.Uploading, a.LatestLocal) { + keep = append(keep, a.Uploading.uuid()) + } + return keep } -// CommitPromoted records snap, the local snapshot PromoteSnapshot just -// uploaded, as the actor's latest snapshot, keeping the time its checkpoint -// committed. PromoteSnapshot then deletes the local copy, so the actor keeps -// no local snapshot. A retried promote of a snapshot already recorded keeps -// what was recorded. It returns the actor's state. +// CommitPromoted records snap, a local snapshot PromoteSnapshot made durable or +// found durable already, as the actor's latest durable snapshot, keeping its +// local copy, the time its checkpoint committed, and any UploadTime on snap. +// It leaves DetachAt as it is. A promote of a snapshot already durable, or +// older than LatestDurable, keeps what was recorded. It returns the actor's +// state. func (t *Tracker) CommitPromoted(actorUID string, snap SnapshotRef) *ateletpb.ActorSnapshotState { t.lock(actorUID) defer t.unlock(actorUID) a := t.findOrAddLocked(actorUID) - uuid := snap.Snapshot.GetSnapshotUuid() - if d := a.LatestDurable; d == nil || d.Snapshot.GetSnapshotUuid() != uuid { - if l := a.LatestLocal; l != nil && l.Snapshot.GetSnapshotUuid() == uuid { - snap.CommitTime = l.CommitTime + // Skip when snap is already LatestDurable (so a retried promote keeps the + // original CommitTime and UploadTime) or older than LatestDurable (a + // newer checkpoint became durable while PromoteSnapshot ran). + if !a.uploadNeeded(snap) { + if a.LatestLocal != nil && a.reportDue() { + t.clearReportDueLocked(a) + } else { + t.requeueLocked(a) } - a.LatestDurable = snap.clone() + return t.stateLocked(a) + } + // Carry over the checkpoint commit time when promoting the tracked + // local snapshot. snap.UploadTime is already set by the caller when + // it uploaded the snapshot. + if sameUUID(a.LatestLocal, &snap) { + snap.CommitTime = a.LatestLocal.CommitTime } - // TODO: change this when PromoteSnapshot keeps the local copy. - a.LatestLocal = nil - t.succeededLocked(a) + a.LatestDurable = snap.clone() + // A promote with a local snapshot still on the node returns the latest + // state to the control plane. Once LatestLocal is nil, a pending detach + // report stays due, since the control plane does not read a promote's + // response as a detach. + if a.LatestLocal != nil { + t.cancelReportLocked(a) + } + t.durableCommittedLocked(a) return t.stateLocked(a) } +// BeginStart records that a Run or Restore is starting the actor on this node, +// once any detach in progress has finished, and returns the func to call when +// the start ends, whether or not it succeeded. It first unschedules the +// actor's detach (see markStarted) and saves that, so atelet does not detach +// the actor while a workload restored from its local snapshot may use it, even +// if atelet restarts or the start fails partway. A failed Run or Restore +// leaves the detach unscheduled, since atelet cannot tell whether the workload +// is running on the worker; the local snapshot stays until the next checkpoint +// or Terminate on this node. When it cannot save that, it fails with +// Unavailable, and the caller must not start the actor. +func (t *Tracker) BeginStart(ctx context.Context, actorUID string) (end func(), err error) { + t.lock(actorUID) + defer t.unlock(actorUID) + a, err := t.waitForDetachLocked(ctx, actorUID, nil) + if err != nil { + return nil, err + } + if a != nil { + // markStarted clears DetachAt, and reports whether it was set: + // only then is there a change to save. If the save fails, put + // DetachAt and unsaved back as they were, matching the state file, + // so the detach stays scheduled. + if detachAt, unsaved := a.DetachAt, a.unsaved; a.markStarted() { + if err := t.writeStateLocked(a); err != nil { + a.DetachAt, a.unsaved = detachAt, unsaved + return nil, apierror.Unavailable("while unscheduling the detach of actor %s before starting it: %v", actorUID, err) + } + } + // No detach is scheduled now, so failed detaches have nothing left + // to retry. Keep the backoff when an upload is still due. + if _, ok := t.uploadDueLocked(a); !ok { + t.resetBackoffLocked(a) + } + } + return t.holdLocked(actorUID, a, nil, false), nil +} + +// BeginCheckpoint records that a Checkpoint of snap is in progress on this +// node, once any detach in progress has finished, and returns the func to call +// when the checkpoint ends, whether or not it succeeded. It fails with +// FailedPrecondition when snap has a stale generation for the actor: below the +// actor's newest snapshot on this node, or equal to it under another +// snapshot_uuid. atelet does not detach the actor in between, so a detach +// cannot delete the local snapshot the checkpoint writes. +func (t *Tracker) BeginCheckpoint(ctx context.Context, actorUID string, snap *ateletpb.Snapshot) (end func(), err error) { + t.lock(actorUID) + defer t.unlock(actorUID) + a, err := t.waitForDetachLocked(ctx, actorUID, func(a *actorSnapshots) error { + return a.checkGeneration(snap) + }) + if err != nil { + return nil, err + } + // Unlike BeginStart, leave DetachAt as it is. Either it is already + // zero, because the actor's Run or Restore on this node unscheduled the + // detach, or this is a retry of a Checkpoint that already committed its + // snapshot, whose detach stays scheduled. + return t.holdLocked(actorUID, a, &SnapshotRef{Snapshot: proto.CloneOf(snap)}, false), nil +} + +// BeginPromote records that a PromoteSnapshot of snap is in progress on this +// node, once any detach in progress and any background upload of snap have +// finished. It fails with FailedPrecondition when snap has a stale generation +// for the actor or conflicts with LatestDurable. It reports whether snap is +// already the actor's latest durable snapshot, and returns the func to call +// when the promote ends. atelet does not start a detach or a background upload +// of the actor in between. +func (t *Tracker) BeginPromote(ctx context.Context, actorUID string, snap SnapshotRef) (alreadyDurable bool, end func(), err error) { + t.lock(actorUID) + defer t.unlock(actorUID) + a, err := t.waitForDetachOrSnapshotUploadLocked(ctx, actorUID, snap) + if err != nil { + return false, nil, err + } + if a != nil { + alreadyDurable, err = a.isLatestDurable(snap) + if err != nil { + return false, nil, err + } + } + return alreadyDurable, t.holdLocked(actorUID, a, snap.clone(), true), nil +} + +// holdLocked counts a Run, Restore, Checkpoint, or PromoteSnapshot of the +// actor in progress on its entry a, which holds detaches and state reports of +// the actor off (and background uploads when promote is true), and returns the +// func that ends it and queues the actor again. When snap is non-nil +// (Checkpoint and PromoteSnapshot), it stays in a.inFlight until end runs, so +// newest includes it. An actor atelet tracks nothing of yet (a nil a) is held +// too, so no detach or report can begin on a snapshot its first Checkpoint +// commits before it ends. +func (t *Tracker) holdLocked(actorUID string, a *actorSnapshots, snap *SnapshotRef, promote bool) (end func()) { + if a == nil { + a = &actorSnapshots{actorUID: actorUID} + } + t.addLocked(a) + a.holds++ + if snap != nil { + a.inFlight = append(a.inFlight, snap) + } + if promote { + a.promotes++ + } + return func() { + t.lock(actorUID) + defer t.unlock(actorUID) + a.holds-- + if snap != nil { + a.inFlight = slices.DeleteFunc(a.inFlight, func(r *SnapshotRef) bool { return r == snap }) + } + if promote { + a.promotes-- + } + if !t.tracksLocked(a) { + // Forget (via Terminate) dropped the actor while the Run, Restore, + // Checkpoint, or PromoteSnapshot was in progress. + return + } + if a.holds == 0 && a.reportDeferred { + t.queueReportLocked(a) + } + t.requeueLocked(a) + } +} + // Forget drops the actor's state, in memory and on disk. It does not wait for -// the actor's background operation in progress: it cancels it, and the -// operation finds itself abandoned when it ends. +// the actor's operation in progress, an upload or a detach: it cancels it, and +// the operation finds itself abandoned when it ends. func (t *Tracker) Forget(actorUID string) error { t.lock(actorUID) defer t.unlock(actorUID) t.abandonOpLocked(actorUID) - t.resetBackoffLocked(actorUID) + // Start the actor's backoff over; its entry, and any hold, goes below. + t.retries.Forget(actorUID) + t.reporting.retries.Forget(actorUID) t.dropLocked(actorUID) + t.undurableActorsTally.forget(actorUID) return removeSnapshotState(actorUID) } +// setLatestLocalLocked records snap as the actor's latest local snapshot and +// schedules the actor's detach a detach delay from now. +func (t *Tracker) setLatestLocalLocked(a *actorSnapshots, snap SnapshotRef) { + a.setLatestLocal(snap, t.clock.Now().Add(t.detachDelay)) +} + +// clearLatestLocalLocked records that atelet holds no local snapshot of the +// actor, as after Detach pruned its local checkpoints, zeroes UndurableSince, +// and lifts the backoff of its failed background work, which has nothing left +// to retry. +func (t *Tracker) clearLatestLocalLocked(a *actorSnapshots) { + a.Uploading = nil + a.uploadingFailed = false + a.UndurableSince = time.Time{} + a.clearLatestLocal() + t.resetBackoffLocked(a) +} + +// durableCommittedLocked finishes CommitDurable or CommitPromoted once +// LatestDurable is updated. When LatestLocal is now durable too (or nil), it +// clears UndurableSince and lifts any retry backoff; when a newer LatestLocal +// still needs uploading, its UndurableSince and backoff stay in place. +func (t *Tracker) durableCommittedLocked(a *actorSnapshots) { + if a.LatestLocal == nil || !a.uploadNeeded(*a.LatestLocal) { + a.UndurableSince = time.Time{} + t.resetBackoffLocked(a) + } + t.saveLocked(a) + t.requeueLocked(a) +} + // succeededLocked finishes a change that leaves the actor in a successful end -// state, such as its latest snapshot being durable: it starts the actor's -// backoff over, saves the actor's state, and queues the actor for its next -// background work. +// state, such as a background upload or detach: it starts the actor's backoff +// over, saves the actor's state, and queues the actor for its next background +// work. func (t *Tracker) succeededLocked(a *actorSnapshots) { - t.resetBackoffLocked(a.actorUID) + t.resetBackoffLocked(a) t.saveLocked(a) t.requeueLocked(a) } diff --git a/cmd/atelet/local_checkpoints.go b/cmd/atelet/internal/snapshottracker/localcheckpoints.go similarity index 57% rename from cmd/atelet/local_checkpoints.go rename to cmd/atelet/internal/snapshottracker/localcheckpoints.go index 642122f700..dd6648d17c 100644 --- a/cmd/atelet/local_checkpoints.go +++ b/cmd/atelet/internal/snapshottracker/localcheckpoints.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package snapshottracker import ( "context" @@ -21,17 +21,18 @@ import ( "log/slog" "os" "path/filepath" + "slices" "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" ) -// pruneLocalCheckpoints removes every local snapshot of the actor. A missing -// directory is not an error, so retries are safe. -func pruneLocalCheckpoints(ctx context.Context, actorUID string) error { - return pruneLocalCheckpointDir(ctx, ateletpath.LocalCheckpointsDir(actorUID)) +// PruneLocalCheckpoints removes every local snapshot of the actor except those +// named in keep. A missing directory is not an error, so retries are safe. +func PruneLocalCheckpoints(ctx context.Context, actorUID string, keep ...string) error { + return pruneLocalCheckpointDir(ctx, ateletpath.LocalCheckpointsDir(actorUID), keep...) } -func pruneLocalCheckpointDir(ctx context.Context, dir string) error { +func pruneLocalCheckpointDir(ctx context.Context, dir string, keep ...string) error { entries, err := os.ReadDir(dir) if err != nil { if os.IsNotExist(err) { @@ -42,7 +43,12 @@ func pruneLocalCheckpointDir(ctx context.Context, dir string) error { // Every entry is attempted: one undeletable snapshot must not strand the // others on disk. var errs []error + kept := false for _, entry := range entries { + if slices.Contains(keep, entry.Name()) { + kept = true + continue + } path := filepath.Join(dir, entry.Name()) if err := os.RemoveAll(path); err != nil { errs = append(errs, fmt.Errorf("while pruning local checkpoint %s: %w", path, err)) @@ -53,8 +59,21 @@ func pruneLocalCheckpointDir(ctx context.Context, dir string) error { if len(errs) > 0 { return errors.Join(errs...) } - if err := os.Remove(dir); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("while removing local checkpoints dir %s: %w", dir, err) + if !kept { + if err := os.Remove(dir); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("while removing local checkpoints dir %s: %w", dir, err) + } + } + return nil +} + +// removeLocalCheckpoint removes one local snapshot of the actor. A missing +// snapshot is not an error. +func removeLocalCheckpoint(ctx context.Context, actorUID, name string) error { + path := ateletpath.LocalSnapshotDir(actorUID, name) + if err := os.RemoveAll(path); err != nil { + return fmt.Errorf("while removing local checkpoint %s: %w", path, err) } + slog.InfoContext(ctx, "removed local checkpoint", slog.String("path", path)) return nil } diff --git a/cmd/atelet/local_checkpoints_test.go b/cmd/atelet/internal/snapshottracker/localcheckpoints_test.go similarity index 72% rename from cmd/atelet/local_checkpoints_test.go rename to cmd/atelet/internal/snapshottracker/localcheckpoints_test.go index 3c53919786..350321c652 100644 --- a/cmd/atelet/local_checkpoints_test.go +++ b/cmd/atelet/internal/snapshottracker/localcheckpoints_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package snapshottracker import ( "context" @@ -48,6 +48,39 @@ func TestPruneRemovesEverySnapshot(t *testing.T) { } } +func TestPruneKeepsSpecifiedSnapshots(t *testing.T) { + for _, tc := range []struct { + name string + keep []string + removed []string + }{ + {name: "one", keep: []string{"pause-2"}, removed: []string{"pause-1", "pause-3"}}, + {name: "the latest and one still uploading", keep: []string{"pause-3", "pause-1"}, removed: []string{"pause-2"}}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + writeSnapshotDir(t, dir, "pause-1") + writeSnapshotDir(t, dir, "pause-2") + writeSnapshotDir(t, dir, "pause-3") + + if err := pruneLocalCheckpointDir(context.Background(), dir, tc.keep...); err != nil { + t.Fatalf("pruneLocalCheckpointDir() = %v, want nil", err) + } + + for _, removed := range tc.removed { + if _, err := os.Stat(filepath.Join(dir, removed)); !os.IsNotExist(err) { + t.Errorf("%s still exists (err=%v), want removed", removed, err) + } + } + for _, kept := range tc.keep { + if _, err := os.Stat(filepath.Join(dir, kept, "memory.img")); err != nil { + t.Errorf("kept snapshot %s missing: %v", kept, err) + } + } + }) + } +} + func TestPruneMissingDirIsNoop(t *testing.T) { if err := pruneLocalCheckpointDir(context.Background(), filepath.Join(t.TempDir(), "absent")); err != nil { t.Fatalf("pruneLocalCheckpointDir() = %v, want nil", err) diff --git a/cmd/atelet/internal/snapshottracker/metrics.go b/cmd/atelet/internal/snapshottracker/metrics.go new file mode 100644 index 0000000000..914f17b041 --- /dev/null +++ b/cmd/atelet/internal/snapshottracker/metrics.go @@ -0,0 +1,363 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package snapshottracker + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "time" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "k8s.io/utils/clock" + + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" +) + +const ( + uploadDurationMetric = "atelet.snapshot.upload.duration" + detachDurationMetric = "atelet.snapshot.detach.duration" + uploadOverdueDurationMetric = "atelet.snapshot.upload.overdue.duration" + detachOverdueDurationMetric = "atelet.snapshot.detach.overdue.duration" + uploadOverdueActorsMetric = "atelet.snapshot.upload.overdue.actors" + undurableAgeMaxMetric = "atelet.snapshot.undurable.age.max" +) + +// overdueBand indexes the ate.snapshot.upload_overdue_band values. +type overdueBand int + +const ( + overdueBandUnder1m overdueBand = iota + overdueBand1mTo5m + overdueBand5mTo15m + overdueBandOver15m + numOverdueBands +) + +var overdueBandValues = [numOverdueBands]string{ + overdueBandUnder1m: ateattr.SnapshotUploadOverdueBandUnder1m, + overdueBand1mTo5m: ateattr.SnapshotUploadOverdueBand1mTo5m, + overdueBand5mTo15m: ateattr.SnapshotUploadOverdueBand5mTo15m, + overdueBandOver15m: ateattr.SnapshotUploadOverdueBandOver15m, +} + +func classifyOverdueBand(overdue time.Duration) overdueBand { + switch { + case overdue <= time.Minute: + return overdueBandUnder1m + case overdue <= 5*time.Minute: + return overdueBand1mTo5m + case overdue <= 15*time.Minute: + return overdueBand5mTo15m + default: + return overdueBandOver15m + } +} + +// opDurationBuckets cover background uploads and detaches: deleting local +// snapshots or uploading a small volumes snapshot takes milliseconds, while +// uploading a multi-GiB memory snapshot runs for tens of seconds. +var opDurationBuckets = []float64{0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30, 60} + +// overdueBuckets cover how late background work starts: within 100ms of its +// due time while a worker is free, and up to an hour late while every worker +// is busy with a backlog of memory snapshot uploads, or for work that came due +// while atelet was down. +var overdueBuckets = []float64{0.1, 1, 5, 10, 30, 60, 120, 300, 600, 1200, 1800, 3600} + +// Instruments holds the snapshot tracker's metric instruments. A nil +// *Instruments is a valid no-op, so call sites need no guard. +type Instruments struct { + uploadDuration metric.Float64Histogram + detachDuration metric.Float64Histogram + uploadOverdueDuration metric.Float64Histogram + detachOverdueDuration metric.Float64Histogram + + // undurableActorsTally is the tally the observable instruments report. + // NewInstruments registers their callback before New creates the tracker + // and attaches its tally, so a metric reader can invoke the callback while + // New stores the pointer. + undurableActorsTally atomic.Pointer[undurableActorsTally] +} + +// NewInstruments creates the snapshot tracker's instruments against meter. +func NewInstruments(meter metric.Meter) (*Instruments, error) { + uploadDuration, err := meter.Float64Histogram( + uploadDurationMetric, + metric.WithUnit("s"), + metric.WithDescription("Duration of one background upload of a local snapshot to object storage on atelet."), + metric.WithExplicitBucketBoundaries(opDurationBuckets...), + ) + if err != nil { + return nil, fmt.Errorf("create %s histogram: %w", uploadDurationMetric, err) + } + detachDuration, err := meter.Float64Histogram( + detachDurationMetric, + metric.WithUnit("s"), + metric.WithDescription("Duration of deleting an actor's local snapshots during a background detach on atelet, excluding any snapshot upload."), + metric.WithExplicitBucketBoundaries(opDurationBuckets...), + ) + if err != nil { + return nil, fmt.Errorf("create %s histogram: %w", detachDurationMetric, err) + } + uploadOverdueDuration, err := meter.Float64Histogram( + uploadOverdueDurationMetric, + metric.WithUnit("s"), + metric.WithDescription("Time from when a background upload of a local snapshot on atelet is due to when a worker starts it."), + metric.WithExplicitBucketBoundaries(overdueBuckets...), + ) + if err != nil { + return nil, fmt.Errorf("create %s histogram: %w", uploadOverdueDurationMetric, err) + } + detachOverdueDuration, err := meter.Float64Histogram( + detachOverdueDurationMetric, + metric.WithUnit("s"), + metric.WithDescription("Time from when a background detach of an actor on atelet is due to when a worker starts it."), + metric.WithExplicitBucketBoundaries(overdueBuckets...), + ) + if err != nil { + return nil, fmt.Errorf("create %s histogram: %w", detachOverdueDurationMetric, err) + } + // The tracker owns the tally of actors with overdue uploads, and the + // overdue bands partition it, so this is an UpDownCounter and not a gauge. + uploadOverdueActors, err := meter.Int64ObservableUpDownCounter( + uploadOverdueActorsMetric, + metric.WithUnit("{actor}"), + metric.WithDescription("Number of actors on atelet's node whose background snapshot upload is overdue, by how long it has been overdue."), + ) + if err != nil { + return nil, fmt.Errorf("create %s updowncounter: %w", uploadOverdueActorsMetric, err) + } + undurableAgeMax, err := meter.Float64ObservableGauge( + undurableAgeMaxMetric, + metric.WithUnit("s"), + metric.WithDescription("Age of the oldest local snapshot on atelet's node that object storage does not hold yet, or zero if there is none."), + ) + if err != nil { + return nil, fmt.Errorf("create %s gauge: %w", undurableAgeMaxMetric, err) + } + + i := &Instruments{ + uploadDuration: uploadDuration, + detachDuration: detachDuration, + uploadOverdueDuration: uploadOverdueDuration, + detachOverdueDuration: detachOverdueDuration, + } + var bandAttrs [numOverdueBands]metric.ObserveOption + for b, v := range overdueBandValues { + bandAttrs[b] = metric.WithAttributes(ateattr.SnapshotUploadOverdueBandKey.String(v)) + } + _, err = meter.RegisterCallback(func(_ context.Context, o metric.Observer) error { + tally := i.undurableActorsTally.Load() + if tally == nil { + return nil + } + c := tally.counts() + // Every overdue band is observed even at zero, so alerts read 0 rather + // than a missing series. + for b, count := range c.overdueBands { + o.ObserveInt64(uploadOverdueActors, count, bandAttrs[b]) + } + o.ObserveFloat64(undurableAgeMax, c.oldestUndurable.Seconds()) + return nil + }, uploadOverdueActors, undurableAgeMax) + if err != nil { + return nil, fmt.Errorf("register undurable actors callback: %w", err) + } + return i, nil +} + +// watch makes tally the one the observable instruments report. +func (i *Instruments) watch(tally *undurableActorsTally) { + if i == nil { + return + } + i.undurableActorsTally.Store(tally) +} + +// recordUpload records one background upload attempt. A probe that found the +// snapshot's manifest committed already without uploading (err == nil && +// !uploaded) is skipped so metadata probes do not skew upload percentiles. +func (i *Instruments) recordUpload(ctx context.Context, target SnapshotRef, elapsed time.Duration, uploaded bool, err error) { + if i == nil || i.uploadDuration == nil { + return + } + if err == nil && !uploaded { + return + } + i.uploadDuration.Record(ctx, elapsed.Seconds(), metric.WithAttributes(outcomeAttrs(ctx, target, err)...)) +} + +// recordDetach records one background detach once target is durable, excluding +// any upload that ran first. +func (i *Instruments) recordDetach(ctx context.Context, target SnapshotRef, elapsed time.Duration, err error) { + if i == nil || i.detachDuration == nil { + return + } + i.detachDuration.Record(ctx, elapsed.Seconds(), metric.WithAttributes(outcomeAttrs(ctx, target, err)...)) +} + +// recordUploadOverdue records how long after a background upload came due a +// worker began it. +func (i *Instruments) recordUploadOverdue(ctx context.Context, overdue time.Duration) { + if i == nil || i.uploadOverdueDuration == nil { + return + } + i.uploadOverdueDuration.Record(ctx, overdue.Seconds()) +} + +// recordDetachOverdue records how long after a background detach came due a +// worker began it. +func (i *Instruments) recordDetachOverdue(ctx context.Context, overdue time.Duration) { + if i == nil || i.detachOverdueDuration == nil { + return + } + i.detachOverdueDuration.Record(ctx, overdue.Seconds()) +} + +// outcomeAttrs returns the labels of a finished upload or detach of target: +// its fidelity, and error.type if it failed. +func outcomeAttrs(ctx context.Context, target SnapshotRef, err error) []attribute.KeyValue { + attrs := make([]attribute.KeyValue, 0, 2) + attrs = append(attrs, ateattr.SnapshotFidelityKey.String(ateattr.SnapshotFidelityValue(target.Snapshot.GetFidelity()))) + if err != nil { + attrs = append(attrs, ateattr.ErrorTypeKey.String(errorType(ctx, err))) + } + return attrs +} + +// errorType returns the error.type of a failed upload or detach: Canceled +// once ctx has ended, as when Terminate or an atelet shutdown cancels the +// operation, and otherwise the code an apierror or a plugin's gRPC status +// carries, or Internal. +func errorType(ctx context.Context, err error) string { + if ctxErr := ctx.Err(); ctxErr != nil { + return status.FromContextError(ctxErr).Code().String() + } + if s, ok := apierror.FromError(err); ok { + return s.Code().String() + } + if s, ok := status.FromError(err); ok { + return s.Code().String() + } + return codes.Internal.String() +} + +// undurableActorsTally tracks, for the observable instruments, the actors that +// hold an undurable local snapshot on this node. It follows what each actor's +// snapshot state file records, so unlike Tracker.actors it covers every actor +// atelet tracks, in memory or not. +type undurableActorsTally struct { + clock clock.PassiveClock + uploadDelay time.Duration + hasStore bool + + mu sync.Mutex + // actors maps the UID of each actor with an undurable local snapshot to + // what the tally keeps of that snapshot. + actors map[string]undurableActorSummary +} + +// undurableActorSummary is what an undurableActorsTally keeps of an actor's +// undurable local snapshot. +type undurableActorSummary struct { + // undurableSince is when the actor's state on this node stopped being + // durable, as uploadDue measures from. Zero if unknown. + undurableSince time.Time + // uploadable is set when atelet has an object storage location to upload + // the actor's undurable snapshot to. + uploadable bool +} + +// undurableActorsCounts is one reading of an undurableActorsTally. +type undurableActorsCounts struct { + // overdueBands counts the actors whose background upload is past + // uploadDelay, by how long past it they are. + overdueBands [numOverdueBands]int64 + // oldestUndurable is the age of the oldest undurable local snapshot: zero + // when there is none. + oldestUndurable time.Duration +} + +func newUndurableActorsTally(clk clock.PassiveClock, uploadDelay time.Duration, hasStore bool) *undurableActorsTally { + return &undurableActorsTally{ + clock: clk, + uploadDelay: uploadDelay, + hasStore: hasStore, + actors: map[string]undurableActorSummary{}, + } +} + +// observe records the actor's undurable local snapshot, or drops the actor +// once its latest local snapshot is durable or detached. The caller holds the +// actor's lock. +func (l *undurableActorsTally) observe(a *actorSnapshots) { + if l == nil { + return + } + if a.LatestLocal == nil || !a.uploadNeeded(*a.LatestLocal) { + l.forget(a.actorUID) + return + } + keepUploading := a.keepUploading() + since := a.UndurableSince + if since.IsZero() && keepUploading { + since = a.Uploading.CommitTime + } + if since.IsZero() { + since = a.LatestLocal.CommitTime + } + s := undurableActorSummary{ + undurableSince: since, + uploadable: l.hasStore && (a.canUpload(a.LatestLocal) || keepUploading), + } + l.mu.Lock() + defer l.mu.Unlock() + l.actors[a.actorUID] = s +} + +// forget drops the actor from the tally. +func (l *undurableActorsTally) forget(actorUID string) { + if l == nil { + return + } + l.mu.Lock() + defer l.mu.Unlock() + delete(l.actors, actorUID) +} + +// counts reads the tally. +func (l *undurableActorsTally) counts() undurableActorsCounts { + now := l.clock.Now() + l.mu.Lock() + defer l.mu.Unlock() + var c undurableActorsCounts + for _, s := range l.actors { + if s.undurableSince.IsZero() { + continue + } + age := now.Sub(s.undurableSince) + c.oldestUndurable = max(c.oldestUndurable, age) + if s.uploadable && age > l.uploadDelay { + c.overdueBands[classifyOverdueBand(age-l.uploadDelay)]++ + } + } + return c +} diff --git a/cmd/atelet/internal/snapshottracker/metrics_test.go b/cmd/atelet/internal/snapshottracker/metrics_test.go new file mode 100644 index 0000000000..d30c9b2f4f --- /dev/null +++ b/cmd/atelet/internal/snapshottracker/metrics_test.go @@ -0,0 +1,789 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package snapshottracker + +import ( + "context" + "errors" + "fmt" + "os" + "slices" + "testing" + "time" + + "go.opentelemetry.io/otel/attribute" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/proto/ateletpb" +) + +func newTestInstruments(t *testing.T) (*Instruments, *sdkmetric.ManualReader) { + t.Helper() + reader := sdkmetric.NewManualReader() + mp := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + inst, err := NewInstruments(mp.Meter("atelet")) + if err != nil { + t.Fatalf("NewInstruments: %v", err) + } + return inst, reader +} + +// collectMetric returns the instrument named name from reader, if collected. +func collectMetric(t *testing.T, reader *sdkmetric.ManualReader, name string) (metricdata.Metrics, bool) { + t.Helper() + var rm metricdata.ResourceMetrics + if err := reader.Collect(context.Background(), &rm); err != nil { + t.Fatalf("collect: %v", err) + } + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name == name { + return m, true + } + } + } + return metricdata.Metrics{}, false +} + +func collectUploadDuration(t *testing.T, reader *sdkmetric.ManualReader) (metricdata.Metrics, bool) { + t.Helper() + return collectMetric(t, reader, uploadDurationMetric) +} + +// recordedErrorTypes returns the error.type of each upload recorded on reader, +// "" for one that succeeded. +func recordedErrorTypes(t *testing.T, reader *sdkmetric.ManualReader) []string { + t.Helper() + return errorTypesOf(t, reader, uploadDurationMetric) +} + +// errorTypesOf returns the error.type of each operation recorded on the +// histogram named name, "" for one that succeeded. +func errorTypesOf(t *testing.T, reader *sdkmetric.ManualReader, name string) []string { + t.Helper() + m, ok := collectMetric(t, reader, name) + if !ok { + return nil + } + var types []string + for _, dp := range m.Data.(metricdata.Histogram[float64]).DataPoints { + v, _ := dp.Attributes.Value(ateattr.ErrorTypeKey) + for range dp.Count { + types = append(types, v.AsString()) + } + } + return types +} + +func TestUploadDurationShape(t *testing.T) { + inst, reader := newTestInstruments(t) + store := newFakeSnapshotStore() + store.setBlocking(true) + useActorDir(t) + tr := newStoreTracker(t, store, WithInstruments(inst)) + tr.Start() + + ref := testRef("s1", 1, tr.clock.Now()) + ref.Snapshot.Fidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES + tr.CommitLocal(trackedActorUID, ref) + done := startUpload(t, tr, store) + stepTo(tr, tr.clock.Now().Add(2*time.Second)) + store.release <- nil + waitClosed(t, done, "the upload") + + m, ok := collectUploadDuration(t, reader) + if !ok { + t.Fatalf("metric %q not collected", uploadDurationMetric) + } + if m.Unit != "s" { + t.Errorf("unit = %q, want %q", m.Unit, "s") + } + if m.Description == "" { + t.Error("description is empty") + } + hist, ok := m.Data.(metricdata.Histogram[float64]) + if !ok { + t.Fatalf("%s is %T, want a float64 histogram", m.Name, m.Data) + } + if len(hist.DataPoints) != 1 { + t.Fatalf("datapoints = %d, want 1", len(hist.DataPoints)) + } + dp := hist.DataPoints[0] + if dp.Count != 1 || dp.Sum != 2 { + t.Errorf("count = %d, sum = %v, want 1 and 2", dp.Count, dp.Sum) + } + if got, _ := dp.Attributes.Value(ateattr.SnapshotFidelityKey); got.AsString() != ateattr.SnapshotFidelityVolumes { + t.Errorf("fidelity = %q, want %q", got.AsString(), ateattr.SnapshotFidelityVolumes) + } + if _, ok := dp.Attributes.Value(ateattr.ErrorTypeKey); ok { + t.Error("error.type present on a successful upload") + } + if _, ok := dp.Attributes.Value(attribute.Key("ate.actor.uid")); ok { + t.Error("actor identity must never reach a metric datapoint") + } +} + +func TestUploadDurationFailureAndUnknownFidelity(t *testing.T) { + inst, reader := newTestInstruments(t) + store := newFakeSnapshotStore() + store.setBlocking(true) + useActorDir(t) + tr := newStoreTracker(t, store, WithInstruments(inst)) + tr.Start() + + ref := testRef("s1", 1, tr.clock.Now()) + ref.Snapshot.Fidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED + tr.CommitLocal(trackedActorUID, ref) + uploadNow(t, tr, store, apierror.Unavailable("object store down")) + + m, ok := collectUploadDuration(t, reader) + if !ok { + t.Fatalf("metric %q not collected", uploadDurationMetric) + } + hist := m.Data.(metricdata.Histogram[float64]) + if len(hist.DataPoints) != 1 { + t.Fatalf("datapoints = %d, want 1", len(hist.DataPoints)) + } + dp := hist.DataPoints[0] + if dp.Count != 1 { + t.Errorf("count = %d, want 1", dp.Count) + } + if got, _ := dp.Attributes.Value(ateattr.SnapshotFidelityKey); got.AsString() != "unknown" { + t.Errorf("fidelity = %q, want unknown", got.AsString()) + } + if got, _ := dp.Attributes.Value(ateattr.ErrorTypeKey); got.AsString() != codes.Unavailable.String() { + t.Errorf("error.type = %q, want %q", got.AsString(), codes.Unavailable.String()) + } +} + +// Background detaches that upload a snapshot record on atelet.snapshot.upload.duration, +// while foreground detaches, already-durable background detaches, and probes +// that find the remote manifest committed already do not. +func TestUploadDurationSkipsNonUploadingPaths(t *testing.T) { + t.Run("background detach that uploads records", func(t *testing.T) { + inst, reader := newTestInstruments(t) + store := newFakeSnapshotStore() + useActorDir(t) + tr := newStoreTracker(t, store, WithInstruments(inst)) + setWorkers(tr, 0, DefaultDetachWorkers) + tr.Start() + + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + detachNow(t, tr) + m, ok := collectUploadDuration(t, reader) + if !ok { + t.Fatalf("metric %q not collected after a background detach that uploaded", uploadDurationMetric) + } + var uploads uint64 + for _, dp := range m.Data.(metricdata.Histogram[float64]).DataPoints { + uploads += dp.Count + } + if uploads != 1 { + t.Errorf("recorded uploads = %d, want 1", uploads) + } + }) + + t.Run("foreground detach does not record", func(t *testing.T) { + inst, reader := newTestInstruments(t) + store := newFakeSnapshotStore() + useActorDir(t) + tr := newStoreTracker(t, store, WithInstruments(inst)) + tr.Start() + + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + if _, err := tr.Detach(t.Context(), trackedActorUID); err != nil { + t.Fatalf("Detach: %v", err) + } + if _, ok := collectUploadDuration(t, reader); ok { + t.Error("foreground Detach recorded on atelet.snapshot.upload.duration") + } + }) + + t.Run("already durable background detach does not record", func(t *testing.T) { + inst, reader := newTestInstruments(t) + store := newFakeSnapshotStore() + useActorDir(t) + tr := newStoreTracker(t, store, WithInstruments(inst)) + tr.Start() + + ref := testRef("s1", 1, tr.clock.Now()) + tr.CommitDurable(trackedActorUID, ref) + tr.CommitLocal(trackedActorUID, ref) + detachNow(t, tr) + if _, ok := collectUploadDuration(t, reader); ok { + t.Error("already-durable background detach recorded on atelet.snapshot.upload.duration") + } + }) + + t.Run("probe finding remote manifest committed does not record", func(t *testing.T) { + inst, reader := newTestInstruments(t) + store := newFakeSnapshotStore() + store.setBlocking(true) + useActorDir(t) + tr := newStoreTracker(t, store, WithInstruments(inst)) + tr.Start() + + ref := testRef("s1", 1, tr.clock.Now()) + store.uploaded = map[string]bool{ref.uri(): true} + tr.CommitLocal(trackedActorUID, ref) + uploadNow(t, tr, store, nil) + if _, ok := collectUploadDuration(t, reader); ok { + t.Error("probe without upload recorded on atelet.snapshot.upload.duration") + } + }) +} + +// A failed upload's error.type is Canceled once its context has ended, as +// when Terminate or shutdown cancels it, even though the plugin reports a +// gRPC status, and otherwise the code an apierror or a wrapped gRPC status +// carries. +func TestErrorType(t *testing.T) { + canceled, cancel := context.WithCancel(context.Background()) + cancel() + for _, tc := range []struct { + name string + ctx context.Context + err error + want codes.Code + }{ + { + name: "canceled", + ctx: canceled, + err: fmt.Errorf("while uploading snapshot files: %w", status.Error(codes.Canceled, "context canceled")), + want: codes.Canceled, + }, + { + name: "apierror", + ctx: context.Background(), + err: apierror.Unavailable("plugin: unreachable"), + want: codes.Unavailable, + }, + { + name: "plugin status", + ctx: context.Background(), + err: fmt.Errorf("while uploading snapshot files: %w", status.Error(codes.PermissionDenied, "denied")), + want: codes.PermissionDenied, + }, + { + name: "plain error", + ctx: context.Background(), + err: errors.New("boom"), + want: codes.Internal, + }, + } { + t.Run(tc.name, func(t *testing.T) { + if got := errorType(tc.ctx, tc.err); got != tc.want.String() { + t.Errorf("errorType = %q, want %q", got, tc.want.String()) + } + }) + } +} + +// histogramOf returns the float64 histogram named name from reader, failing +// the test unless it was collected with unit seconds. +func histogramOf(t *testing.T, reader *sdkmetric.ManualReader, name string) metricdata.Histogram[float64] { + t.Helper() + m, ok := collectMetric(t, reader, name) + if !ok { + t.Fatalf("metric %q not collected", name) + } + if m.Unit != "s" { + t.Errorf("%s unit = %q, want %q", name, m.Unit, "s") + } + if m.Description == "" { + t.Errorf("%s description is empty", name) + } + hist, ok := m.Data.(metricdata.Histogram[float64]) + if !ok { + t.Fatalf("%s is %T, want a float64 histogram", name, m.Data) + } + for _, dp := range hist.DataPoints { + if _, ok := dp.Attributes.Value(attribute.Key("ate.actor.uid")); ok { + t.Errorf("%s carries actor identity, which must never reach a metric datapoint", name) + } + } + return hist +} + +// waitDetachEnded waits until a background detach of the tracked actor has +// ended, and recorded how: its operation is over and it either dropped the +// local snapshot or set a retry. +func waitDetachEnded(t *testing.T, tr *Tracker) { + t.Helper() + eventually(t, "the detach ends", func() bool { + tr.lock(trackedActorUID) + defer tr.unlock(trackedActorUID) + a := tr.findLocked(trackedActorUID) + return a != nil && a.op == nil && (a.LatestLocal == nil || !a.retryAt.IsZero()) + }) +} + +// A background detach records how long its local cleanup took, labeled with +// the snapshot's fidelity, and excludes any time spent uploading the snapshot +// first (which atelet.snapshot.upload.duration records instead). +func TestDetachDurationShape(t *testing.T) { + inst, reader := newTestInstruments(t) + store := newFakeSnapshotStore() + store.setBlocking(true) + useActorDir(t) + tr := newStoreTracker(t, store, WithInstruments(inst)) + setWorkers(tr, 0, DefaultDetachWorkers) + tr.Start() + + ref := testRef("s1", 1, tr.clock.Now()) + ref.Snapshot.Fidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES + tr.CommitLocal(trackedActorUID, ref) + stepTo(tr, detachDueOf(tr, trackedActorUID)) + waitStarted(t, store) + done := opDone(t, tr, trackedActorUID) + stepTo(tr, tr.clock.Now().Add(3*time.Second)) + store.release <- nil + waitClosed(t, done, "the detach") + + uploadHist := histogramOf(t, reader, uploadDurationMetric) + if len(uploadHist.DataPoints) != 1 || uploadHist.DataPoints[0].Count != 1 || uploadHist.DataPoints[0].Sum != 3 { + t.Errorf("upload histogram = %+v, want count=1 sum=3", uploadHist.DataPoints) + } + + hist := histogramOf(t, reader, detachDurationMetric) + if len(hist.DataPoints) != 1 { + t.Fatalf("datapoints = %d, want 1", len(hist.DataPoints)) + } + dp := hist.DataPoints[0] + if dp.Count != 1 || dp.Sum != 0 { + t.Errorf("count = %d, sum = %v, want 1 and 0 (upload time excluded)", dp.Count, dp.Sum) + } + if got, _ := dp.Attributes.Value(ateattr.SnapshotFidelityKey); got.AsString() != ateattr.SnapshotFidelityVolumes { + t.Errorf("fidelity = %q, want %q", got.AsString(), ateattr.SnapshotFidelityVolumes) + } + if _, ok := dp.Attributes.Value(ateattr.ErrorTypeKey); ok { + t.Error("error.type present on a successful detach") + } +} + +// A background detach whose local cleanup fails, or that Terminate cancels, +// records error.type; one whose upload fails stops there and records only on +// atelet.snapshot.upload.duration; a foreground Detach records nothing. +func TestDetachDurationOutcomes(t *testing.T) { + newTracker := func(t *testing.T, store *fakeSnapshotStore) (*Tracker, *sdkmetric.ManualReader) { + t.Helper() + inst, reader := newTestInstruments(t) + useActorDir(t) + tr := newStoreTracker(t, store, WithInstruments(inst)) + setWorkers(tr, 0, DefaultDetachWorkers) + tr.Start() + return tr, reader + } + + t.Run("failed", func(t *testing.T) { + store := newFakeSnapshotStore() + tr, reader := newTracker(t, store) + ref := testRef("s1", 1, tr.clock.Now()) + tr.CommitDurable(trackedActorUID, ref) + tr.CommitLocal(trackedActorUID, ref) + // A file where the local checkpoint directory belongs fails the prune. + if err := os.WriteFile(ateletpath.LocalCheckpointsDir(trackedActorUID), nil, 0o600); err != nil { + t.Fatal(err) + } + stepTo(tr, detachDueOf(tr, trackedActorUID)) + waitDetachEnded(t, tr) + if got, want := errorTypesOf(t, reader, detachDurationMetric), []string{codes.Internal.String()}; !slices.Equal(got, want) { + t.Errorf("error types = %v, want %v", got, want) + } + }) + + t.Run("canceled", func(t *testing.T) { + inst, reader := newTestInstruments(t) + store := newFakeSnapshotStore() + useActorDir(t) + tr := newStoreTracker(t, store, WithInstruments(inst)) + ref := testRef("s1", 1, tr.clock.Now()) + tr.CommitDurable(trackedActorUID, ref) + tr.CommitLocal(trackedActorUID, ref) + stepTo(tr, detachDueOf(tr, trackedActorUID)) + run := tr.beginDueWork(trackedActorUID, opDetach) + if run == nil { + t.Fatal("beginDueWork returned nil, want due detach") + } + if err := tr.Forget(trackedActorUID); err != nil { + t.Fatalf("Forget: %v", err) + } + run() + if got, want := errorTypesOf(t, reader, detachDurationMetric), []string{codes.Canceled.String()}; !slices.Equal(got, want) { + t.Errorf("error types = %v, want %v", got, want) + } + }) + + t.Run("upload failure does not record", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr, reader := newTracker(t, store) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + stepTo(tr, detachDueOf(tr, trackedActorUID)) + waitStarted(t, store) + done := opDone(t, tr, trackedActorUID) + store.release <- apierror.Unavailable("object store down") + waitClosed(t, done, "the detach") + if got, want := errorTypesOf(t, reader, uploadDurationMetric), []string{codes.Unavailable.String()}; !slices.Equal(got, want) { + t.Errorf("upload error types = %v, want %v", got, want) + } + if _, ok := collectMetric(t, reader, detachDurationMetric); ok { + t.Errorf("failed upload during detach recorded on %s", detachDurationMetric) + } + }) + + t.Run("already durable", func(t *testing.T) { + store := newFakeSnapshotStore() + tr, reader := newTracker(t, store) + ref := testRef("s1", 1, tr.clock.Now()) + tr.CommitDurable(trackedActorUID, ref) + tr.CommitLocal(trackedActorUID, ref) + detachNow(t, tr) + if got, want := errorTypesOf(t, reader, detachDurationMetric), []string{""}; !slices.Equal(got, want) { + t.Errorf("error types = %v, want %v", got, want) + } + if _, ok := collectUploadDuration(t, reader); ok { + t.Error("a detach that uploaded nothing recorded an upload") + } + }) + + t.Run("foreground", func(t *testing.T) { + store := newFakeSnapshotStore() + tr, reader := newTracker(t, store) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + if _, err := tr.Detach(t.Context(), trackedActorUID); err != nil { + t.Fatalf("Detach: %v", err) + } + if _, ok := collectMetric(t, reader, detachDurationMetric); ok { + t.Errorf("foreground Detach recorded on %s", detachDurationMetric) + } + }) +} + +// Each background upload and detach records how long after it came due a +// worker started it, on its own histogram. +func TestOverdueDuration(t *testing.T) { + inst, reader := newTestInstruments(t) + store := newFakeSnapshotStore() + store.setBlocking(true) + useActorDir(t) + tr := newStoreTracker(t, store, WithInstruments(inst)) + tr.Start() + + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + uploadDue := timeOf(tr.State(trackedActorUID).GetUploadDueTime()) + stepTo(tr, uploadDue.Add(5*time.Second)) + waitStarted(t, store) + done := opDone(t, tr, trackedActorUID) + store.release <- nil + waitClosed(t, done, "the upload") + + // The snapshot is durable now, so the detach only deletes it. + stepTo(tr, detachDueOf(tr, trackedActorUID).Add(7*time.Second)) + waitDetachEnded(t, tr) + + for _, tc := range []struct { + metric string + want float64 + }{ + {metric: uploadOverdueDurationMetric, want: 5}, + {metric: detachOverdueDurationMetric, want: 7}, + } { + hist := histogramOf(t, reader, tc.metric) + if len(hist.DataPoints) != 1 { + t.Fatalf("%s datapoints = %d, want 1", tc.metric, len(hist.DataPoints)) + } + dp := hist.DataPoints[0] + if dp.Attributes.Len() != 0 { + t.Errorf("%s labels = %v, want none", tc.metric, dp.Attributes.ToSlice()) + } + if dp.Count != 1 || dp.Sum != tc.want { + t.Errorf("%s count = %d, sum = %v, want 1 and %v", tc.metric, dp.Count, dp.Sum, tc.want) + } + } +} + +// undurableReading is what the undurable actors instruments report. +type undurableReading struct { + under1m, m1to5, m5to15, over15 int64 + // age is the age of the oldest undurable local snapshot, in seconds. + age float64 +} + +// readUndurable collects the undurable actors instruments from reader, failing +// the test unless each has the shape the registry gives it. +func readUndurable(t *testing.T, reader *sdkmetric.ManualReader) undurableReading { + t.Helper() + var rm metricdata.ResourceMetrics + if err := reader.Collect(context.Background(), &rm); err != nil { + t.Fatalf("collect: %v", err) + } + var got undurableReading + var overdueSeen, ageSeen bool + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + switch m.Name { + case uploadOverdueActorsMetric: + overdueSeen = true + if m.Unit != "{actor}" { + t.Errorf("%s unit = %q, want {actor}", m.Name, m.Unit) + } + sum, ok := m.Data.(metricdata.Sum[int64]) + if !ok || sum.IsMonotonic { + t.Fatalf("%s is %T, want a non-monotonic int64 sum", m.Name, m.Data) + } + if len(sum.DataPoints) != int(numOverdueBands) { + t.Errorf("%s has %d series, want %d", m.Name, len(sum.DataPoints), numOverdueBands) + } + for _, dp := range sum.DataPoints { + if dp.Attributes.Len() != 1 { + t.Errorf("%s series has labels %v, want only %s", m.Name, dp.Attributes.ToSlice(), ateattr.SnapshotUploadOverdueBandKey) + } + switch v, _ := dp.Attributes.Value(ateattr.SnapshotUploadOverdueBandKey); v.AsString() { + case ateattr.SnapshotUploadOverdueBandUnder1m: + got.under1m = dp.Value + case ateattr.SnapshotUploadOverdueBand1mTo5m: + got.m1to5 = dp.Value + case ateattr.SnapshotUploadOverdueBand5mTo15m: + got.m5to15 = dp.Value + case ateattr.SnapshotUploadOverdueBandOver15m: + got.over15 = dp.Value + default: + t.Errorf("%s band = %q", m.Name, v.AsString()) + } + } + case undurableAgeMaxMetric: + ageSeen = true + if m.Unit != "s" { + t.Errorf("%s unit = %q, want s", m.Name, m.Unit) + } + gauge, ok := m.Data.(metricdata.Gauge[float64]) + if !ok || len(gauge.DataPoints) != 1 || gauge.DataPoints[0].Attributes.Len() != 0 { + t.Fatalf("%s is %+v, want a float64 gauge with one unlabeled series", m.Name, m.Data) + } + got.age = gauge.DataPoints[0].Value + } + } + } + if !overdueSeen || !ageSeen { + t.Fatalf("collected %s: %v, %s: %v; want both", + uploadOverdueActorsMetric, overdueSeen, undurableAgeMaxMetric, ageSeen) + } + return got +} + +// The undurable actors instruments bucket the actors whose upload is past +// uploadDelay by how long it has been overdue, and report how long the oldest +// undurable snapshot has been undurable, as checkpoints and an upload change +// what the node holds. +func TestUndurableActorsInstruments(t *testing.T) { + inst, reader := newTestInstruments(t) + store := newFakeSnapshotStore() + store.setBlocking(true) + useActorDir(t) + tr := newStoreTracker(t, store, WithInstruments(inst)) + tr.Start() + t0 := tr.clock.Now() + + if got, want := readUndurable(t, reader), (undurableReading{}); got != want { + t.Errorf("with nothing held: %+v, want %+v", got, want) + } + + // A newer checkpoint before the upload leaves the age running from the + // first. + tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) + stepTo(tr, t0.Add(10*time.Minute)) + tr.CommitLocal(trackedActorUID, testRef("s2", 2, tr.clock.Now())) + if got, want := readUndurable(t, reader), (undurableReading{age: (10 * time.Minute).Seconds()}); got != want { + t.Errorf("before uploadDelay: %+v, want %+v", got, want) + } + + stepTo(tr, t0.Add(testUploadDelay+30*time.Second)) + waitStarted(t, store) + if got, want := readUndurable(t, reader), (undurableReading{under1m: 1, age: (testUploadDelay + 30*time.Second).Seconds()}); got != want { + t.Errorf("30s past uploadDelay: %+v, want %+v", got, want) + } + + stepTo(tr, t0.Add(testUploadDelay+5*time.Minute)) + if got, want := readUndurable(t, reader), (undurableReading{m1to5: 1, age: (testUploadDelay + 5*time.Minute).Seconds()}); got != want { + t.Errorf("5m past uploadDelay: %+v, want %+v", got, want) + } + + stepTo(tr, t0.Add(testUploadDelay+10*time.Minute)) + if got, want := readUndurable(t, reader), (undurableReading{m5to15: 1, age: (testUploadDelay + 10*time.Minute).Seconds()}); got != want { + t.Errorf("10m past uploadDelay: %+v, want %+v", got, want) + } + + stepTo(tr, t0.Add(testUploadDelay+20*time.Minute)) + if got, want := readUndurable(t, reader), (undurableReading{over15: 1, age: (testUploadDelay + 20*time.Minute).Seconds()}); got != want { + t.Errorf("20m past uploadDelay: %+v, want %+v", got, want) + } + + done := opDone(t, tr, trackedActorUID) + store.release <- nil + waitClosed(t, done, "the upload") + if got, want := readUndurable(t, reader), (undurableReading{}); got != want { + t.Errorf("after the upload: %+v, want %+v", got, want) + } +} + +// An undurable local snapshot with no object storage URI still advances +// undurable.age.max, while upload.overdue.actors does not count it because no +// background upload can run for it. +func TestUndurableActorsInstrumentsUnuploadable(t *testing.T) { + inst, reader := newTestInstruments(t) + useActorDir(t) + tr := newStoreTracker(t, newFakeSnapshotStore(), WithInstruments(inst)) + tr.Start() + t0 := tr.clock.Now() + + ref := testRef("s1", 1, t0) + ref.Snapshot.Object = nil + tr.CommitLocal(trackedActorUID, ref) + stepTo(tr, t0.Add(testUploadDelay+10*time.Minute)) + if got, want := readUndurable(t, reader), (undurableReading{age: (testUploadDelay + 10*time.Minute).Seconds()}); got != want { + t.Errorf("unuploadable snapshot 10m past uploadDelay: %+v, want %+v", got, want) + } +} + +// Forget, as Terminate calls it, drops the actor from the undurable actors +// instruments, as does a save that finds the actor's directory gone. +func TestUndurableActorsInstrumentsDropActor(t *testing.T) { + t.Run("forget", func(t *testing.T) { + inst, reader := newTestInstruments(t) + useActorDir(t) + tr := newStoreTracker(t, newFakeSnapshotStore(), WithInstruments(inst)) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + stepTo(tr, tr.clock.Now().Add(time.Minute)) + if got, want := readUndurable(t, reader), (undurableReading{age: 60}); got != want { + t.Errorf("before Forget: %+v, want %+v", got, want) + } + if err := tr.Forget(trackedActorUID); err != nil { + t.Fatalf("Forget: %v", err) + } + if got, want := readUndurable(t, reader), (undurableReading{}); got != want { + t.Errorf("after Forget: %+v, want %+v", got, want) + } + }) + + t.Run("directory gone", func(t *testing.T) { + inst, reader := newTestInstruments(t) + useActorDir(t) + tr := newStoreTracker(t, newFakeSnapshotStore(), WithInstruments(inst)) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + stepTo(tr, tr.clock.Now().Add(time.Minute)) + if got, want := readUndurable(t, reader), (undurableReading{age: 60}); got != want { + t.Errorf("before directory removal: %+v, want %+v", got, want) + } + if err := os.RemoveAll(ateletpath.ActorPath(trackedActorUID)); err != nil { + t.Fatal(err) + } + tr.CommitLocal(trackedActorUID, testRef("s2", 2, tr.clock.Now())) + if got, want := readUndurable(t, reader), (undurableReading{}); got != want { + t.Errorf("after a save without the actor's directory: %+v, want %+v", got, want) + } + }) +} + +// After an atelet restart, Start counts the undurable local snapshots that the +// actors' snapshot state files record, before any worker reads them. +func TestUndurableActorsInstrumentsAfterRestart(t *testing.T) { + useActorDir(t) + store := newFakeSnapshotStore() + before := newStoreTracker(t, store) + before.CommitLocal(trackedActorUID, testRef("s1", 1, before.clock.Now())) + + inst, reader := newTestInstruments(t) + after := newStoreTracker(t, store, WithInstruments(inst)) + stepTo(after, after.clock.Now().Add(time.Minute)) + if got, want := readUndurable(t, reader), (undurableReading{}); got != want { + t.Errorf("before Start: %+v, want %+v", got, want) + } + after.Start() + if got, want := readUndurable(t, reader), (undurableReading{age: 60}); got != want { + t.Errorf("after Start: %+v, want %+v", got, want) + } +} + +// When UndurableSince is zero, observe falls back to Uploading.CommitTime +// before LatestLocal.CommitTime, matching uploadDue. +func TestUndurableActorsInstrumentsZeroUndurableSince(t *testing.T) { + inst, reader := newTestInstruments(t) + useActorDir(t) + tr := newStoreTracker(t, newFakeSnapshotStore(), WithInstruments(inst)) + t0 := tr.clock.Now() + stepTo(tr, t0.Add(2*time.Minute)) + tr.undurableActorsTally.observe(&actorSnapshots{ + actorUID: trackedActorUID, + actorSnapshotsRecord: actorSnapshotsRecord{ + Uploading: testRef("s1", 1, t0).clone(), + LatestLocal: testRef("s2", 2, t0.Add(30*time.Second)).clone(), + }, + }) + if got, want := readUndurable(t, reader), (undurableReading{age: 120}); got != want { + t.Errorf("zero UndurableSince with Uploading: %+v, want %+v", got, want) + } +} + +// With more than one undurable actor on the node, upload.overdue.actors sums +// them by band and undurable.age.max reports the oldest one, falling back to +// the next oldest once the oldest leaves. +func TestUndurableActorsInstrumentsMultipleActors(t *testing.T) { + const secondActorUID = "actor-uid-2" + inst, reader := newTestInstruments(t) + useActorDir(t) + if err := os.MkdirAll(ateletpath.ActorPath(secondActorUID), 0o700); err != nil { + t.Fatal(err) + } + tr := newStoreTracker(t, newFakeSnapshotStore(), WithInstruments(inst)) + t0 := tr.clock.Now() + + tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) + stepTo(tr, t0.Add(30*time.Second)) + tr.CommitLocal(secondActorUID, testRef("s2", 1, tr.clock.Now())) + + stepTo(tr, t0.Add(testUploadDelay+45*time.Second)) + if got, want := readUndurable(t, reader), (undurableReading{ + under1m: 2, + age: (testUploadDelay + 45*time.Second).Seconds(), + }); got != want { + t.Errorf("both in under_1m: %+v, want %+v", got, want) + } + + stepTo(tr, t0.Add(testUploadDelay+time.Minute+15*time.Second)) + if got, want := readUndurable(t, reader), (undurableReading{ + under1m: 1, + m1to5: 1, + age: (testUploadDelay + time.Minute + 15*time.Second).Seconds(), + }); got != want { + t.Errorf("split across bands: %+v, want %+v", got, want) + } + + if err := tr.Forget(trackedActorUID); err != nil { + t.Fatalf("Forget: %v", err) + } + if got, want := readUndurable(t, reader), (undurableReading{ + under1m: 1, + age: (testUploadDelay + 45*time.Second).Seconds(), + }); got != want { + t.Errorf("after dropping the oldest actor: %+v, want %+v", got, want) + } +} diff --git a/cmd/atelet/internal/snapshottracker/queues_test.go b/cmd/atelet/internal/snapshottracker/queues_test.go new file mode 100644 index 0000000000..9c626d62ee --- /dev/null +++ b/cmd/atelet/internal/snapshottracker/queues_test.go @@ -0,0 +1,227 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package snapshottracker + +import ( + "os" + "slices" + "testing" + "time" + + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" +) + +// Uploads and detaches have queues and workers of their own, so a due detach +// runs while a slow upload holds the only upload worker, and an upload that +// comes due meanwhile waits for that worker. +func TestTrackerDetachDoesNotWaitBehindUploads(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newIdleTestTracker(t, store) + setWorkers(tr, 1, 1) + tr.Start() + for _, id := range []string{"actor-1", "actor-2", "actor-3"} { + if err := os.MkdirAll(ateletpath.ActorPath(id), 0o700); err != nil { + t.Fatal(err) + } + } + + // actor-2's snapshot is durable already, as after an immediate + // checkpoint, so its detach only prunes. + t0 := tr.clock.Now() + a2 := testRef("a2-s1", 1, t0) + tr.CommitDurable("actor-2", a2) + tr.CommitLocal("actor-2", a2) + stepTo(tr, t0.Add(testDetachDelay-time.Minute)) + + taken := tr.clock.Now().Add(-testUploadDelay) + tr.CommitLocal("actor-1", testRef("a1-s1", 1, taken)) + if got := waitStarted(t, store); got != "a1-s1" { + t.Fatalf("started = %q, want a1-s1", got) + } + tr.CommitLocal("actor-3", testRef("a3-s1", 1, taken)) + + stepTo(tr, t0.Add(testDetachDelay)) + eventually(t, "actor-2 is detached while actor-1 uploads", func() bool { + return tr.State("actor-2").GetLatestLocal() == nil + }) + select { + case got := <-store.started: + t.Fatalf("upload of %s started while actor-1's holds the only upload worker", got) + default: + } + + store.release <- nil + if got := waitStarted(t, store); got != "a3-s1" { + t.Fatalf("started = %q, want a3-s1 once actor-1's upload ended", got) + } + store.release <- nil +} + +// When both an upload and a detach are due for the same actor, whichever +// starts first holds a.op and causes the other queue's worker to skip the +// actor. When the upload runs first, endOp requeues the detach, which then +// prunes the now-durable snapshot without uploading again. +func TestTrackerSameActorUploadAndDetachDue(t *testing.T) { + t.Run("upload first", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newIdleTestTracker(t, store) + t0 := tr.clock.Now() + tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) + stepTo(tr, t0.Add(testDetachDelay)) + + uploadDone := make(chan struct{}) + go func() { + defer close(uploadDone) + processNext(t, tr, opUpload) + }() + if got := waitStarted(t, store); got != "s1" { + t.Fatalf("started = %q, want s1", got) + } + + // The detach worker finds a.op held by the upload and skips the actor. + processNext(t, tr, opDetach) + select { + case got := <-store.started: + t.Fatalf("store called for %q while upload holds a.op", got) + default: + } + if got := tr.queues[opDetach].Len(); got != 0 { + t.Errorf("detach queue len while upload runs = %d, want 0", got) + } + if got := uuidOf(tr.State(trackedActorUID).GetLatestLocal()); got != "s1" { + t.Fatalf("latest_local while upload runs = %q, want s1", got) + } + + store.release <- nil + waitClosed(t, uploadDone, "upload worker") + st := tr.State(trackedActorUID) + if uuidOf(st.GetLatestLocal()) != "s1" || uuidOf(st.GetLatestDurable()) != "s1" { + t.Fatalf("state after upload = %v, want s1 both local and durable", st) + } + if got := tr.queues[opDetach].Len(); got != 1 { + t.Fatalf("detach queue len after upload endOp = %d, want 1", got) + } + + // The requeued detach sees s1 durable and prunes without uploading again. + processNext(t, tr, opDetach) + if got, want := store.uploads(), []string{"s1"}; !slices.Equal(got, want) { + t.Errorf("uploads = %v, want %v", got, want) + } + st = tr.State(trackedActorUID) + if st.GetLatestLocal() != nil || uuidOf(st.GetLatestDurable()) != "s1" { + t.Errorf("state after detach = %v, want nothing local and s1 durable", st) + } + if got := queuedNow(tr); got != 0 { + t.Errorf("queued after detach = %d, want 0", got) + } + }) + + t.Run("detach first", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newIdleTestTracker(t, store) + t0 := tr.clock.Now() + tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) + stepTo(tr, t0.Add(testDetachDelay)) + + detachDone := make(chan struct{}) + go func() { + defer close(detachDone) + processNext(t, tr, opDetach) + }() + if got := waitStarted(t, store); got != "s1" { + t.Fatalf("started = %q, want s1", got) + } + + // The upload worker finds a.op held by the detach and skips the actor. + processNext(t, tr, opUpload) + select { + case got := <-store.started: + t.Fatalf("store called for %q while detach holds a.op", got) + default: + } + if got := tr.queues[opUpload].Len(); got != 0 { + t.Errorf("upload queue len while detach runs = %d, want 0", got) + } + + store.release <- nil + waitClosed(t, detachDone, "detach worker") + if got, want := store.uploads(), []string{"s1"}; !slices.Equal(got, want) { + t.Errorf("uploads = %v, want %v", got, want) + } + st := tr.State(trackedActorUID) + if st.GetLatestLocal() != nil || uuidOf(st.GetLatestDurable()) != "s1" { + t.Errorf("state after detach = %v, want nothing local and s1 durable", st) + } + if got := queuedNow(tr); got != 0 { + t.Errorf("queued after detach = %d, want 0", got) + } + }) +} + +// A Checkpoint hold (a.holds > 0) keeps opDetach from pruning local snapshots +// underneath the checkpoint, while opUpload still runs in the background. When +// the hold ends, opDetach prunes the now-durable snapshot without uploading +// again. +func TestTrackerUploadRunsDuringCheckpointHoldWhileDetachWaits(t *testing.T) { + store := newFakeSnapshotStore() + tr := newIdleTestTracker(t, store) + t0 := tr.clock.Now() + tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) + + endCheckpoint, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, testRef("s2", 2, time.Time{}).Snapshot) + if err != nil { + t.Fatalf("BeginCheckpoint: %v", err) + } + stepTo(tr, t0.Add(testDetachDelay)) + + processNext(t, tr, opDetach) + if got := store.uploads(); len(got) != 0 { + t.Fatalf("uploads while held = %v, want none from opDetach", got) + } + if got := uuidOf(tr.State(trackedActorUID).GetLatestLocal()); got != "s1" { + t.Fatalf("latest_local while held = %q, want s1", got) + } + if got := tr.queues[opDetach].Len(); got != 0 { + t.Errorf("detach queue len while held = %d, want 0 until hold or op ends", got) + } + + processNext(t, tr, opUpload) + if got, want := store.uploads(), []string{"s1"}; !slices.Equal(got, want) { + t.Fatalf("uploads after opUpload = %v, want %v", got, want) + } + st := tr.State(trackedActorUID) + if uuidOf(st.GetLatestLocal()) != "s1" || uuidOf(st.GetLatestDurable()) != "s1" { + t.Fatalf("state after opUpload = %v, want s1 both local and durable", st) + } + + // endOp from opUpload requeues opDetach, which still waits while held. + processNext(t, tr, opDetach) + if got := uuidOf(tr.State(trackedActorUID).GetLatestLocal()); got != "s1" { + t.Fatalf("latest_local while still held = %q, want s1", got) + } + + endCheckpoint() + processNext(t, tr, opDetach) + if got, want := store.uploads(), []string{"s1"}; !slices.Equal(got, want) { + t.Errorf("uploads after hold ended = %v, want %v without a second upload", got, want) + } + st = tr.State(trackedActorUID) + if st.GetLatestLocal() != nil || uuidOf(st.GetLatestDurable()) != "s1" { + t.Errorf("state after detach = %v, want nothing local and s1 durable", st) + } +} diff --git a/cmd/atelet/internal/snapshottracker/report.go b/cmd/atelet/internal/snapshottracker/report.go new file mode 100644 index 0000000000..ba7bdc8d5e --- /dev/null +++ b/cmd/atelet/internal/snapshottracker/report.go @@ -0,0 +1,352 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package snapshottracker + +import ( + "context" + "log/slog" + "math/rand/v2" + "time" + + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "k8s.io/client-go/util/workqueue" + "k8s.io/utils/clock" +) + +// Report settings. DefaultReportWorkers and DefaultReportGiveUpAfter are +// overridden by atelet's --snapshot-report-workers and +// --snapshot-report-give-up-after. +const ( + // DefaultReportWorkers is how many reports run at once on a node by + // default. + DefaultReportWorkers = 1 + // reportTimeout bounds one attempt to report an actor's state. + reportTimeout = 10 * time.Second + // reportRetryDelay is how long atelet holds an actor's report back after + // an attempt failed, and the base delay for the report queue after a + // control-plane failure. Each further consecutive failure doubles it, up + // to maxReportRetryDelay, until DefaultReportGiveUpAfter (or + // WithReportGiveUpAfter) has passed since the report became due. + reportRetryDelay = 5 * time.Second + maxReportRetryDelay = 5 * time.Minute + // maxReportQueueRetryDelay caps how long the report queue pauses after + // consecutive control-plane failures, so reports resume soon after + // ate-api-server recovers. + maxReportQueueRetryDelay = 30 * time.Second + DefaultReportGiveUpAfter = 10 * time.Minute + // reportStartupJitter spreads the reports atelet recovers from disk at + // startup, so a fleet-wide restart does not hit the control plane at + // once. + reportStartupJitter = 30 * time.Second +) + +// Reporter tells the control plane what atelet holds of an actor's snapshots. +type Reporter interface { + // ReportActorSnapshotState reports state, the actor's snapshot state on + // this node. Each report carries the whole state, so repeating one is + // safe. + ReportActorSnapshotState(ctx context.Context, actorUID string, actor resources.ActorRef, state *ateletpb.ActorSnapshotState) error +} + +// reportQueue holds the queue and settings for reporting snapshot state +// changes to the control plane. +type reportQueue struct { + // reporter tells the control plane what atelet holds of an actor's + // snapshots. Nil without WithReporter, or when workers is 0. + reporter Reporter + // workers is how many report workers drain queue: DefaultReportWorkers + // unless WithReportWorkers sets it. + workers int + // giveUpAfter is how long atelet keeps retrying a failed report since it + // became due before giving up on it: DefaultReportGiveUpAfter unless + // WithReportGiveUpAfter sets it. + giveUpAfter time.Duration + // queue holds the UIDs of actors whose snapshot state has a report due. + queue workqueue.TypedDelayingInterface[string] + // retries says how long an actor's report is held back after a failure: + // longer with each consecutive failure for the actor. + retries workqueue.TypedRateLimiter[string] + // queueRetries says how long the whole report queue pauses after a + // control-plane failure, so actors do not all hit an overloaded ateapi at + // once. + queueRetries workqueue.TypedRateLimiter[string] + // startupJitter is the window over which saved reports are spread at + // startup: reportStartupJitter unless tests override it. + startupJitter time.Duration +} + +func newReportQueue(clk clock.WithTicker) reportQueue { + return reportQueue{ + workers: DefaultReportWorkers, + giveUpAfter: DefaultReportGiveUpAfter, + queue: workqueue.NewTypedDelayingQueueWithConfig(workqueue.TypedDelayingQueueConfig[string]{Clock: clk}), + retries: workqueue.NewTypedItemExponentialFailureRateLimiter[string](reportRetryDelay, maxReportRetryDelay), + queueRetries: workqueue.NewTypedItemExponentialFailureRateLimiter[string](reportRetryDelay, maxReportQueueRetryDelay), + startupJitter: reportStartupJitter, + } +} + +// markReportDueLocked records that a background upload or detach just made a +// change to report, starting a fresh giveUpAfter window and retry backoff. +func (t *Tracker) markReportDueLocked(a *actorSnapshots) { + a.ReportDueSince = t.clock.Now() + t.reporting.retries.Forget(a.actorUID) +} + +// cancelReportLocked clears any pending report of the actor's state and its +// retry backoff. The caller saves the actor's state. +func (t *Tracker) cancelReportLocked(a *actorSnapshots) { + a.ReportDueSince = time.Time{} + a.reportDeferred = false + t.reporting.retries.Forget(a.actorUID) +} + +// queueReportLocked queues the actor's report, if one is due. While a Run, +// Restore, Checkpoint, or PromoteSnapshot holds the actor, it defers the +// report until the hold ends. +func (t *Tracker) queueReportLocked(a *actorSnapshots) { + if t.reporting.reporter == nil || !a.reportDue() { + return + } + if a.holds > 0 { + a.reportDeferred = true + return + } + a.reportDeferred = false + t.reporting.queue.Add(a.actorUID) +} + +// enqueueSavedReportLocked queues a saved actor's report at startup after a +// random delay up to startupJitter, so a fleet-wide atelet restart does not +// hit the control plane at once. +func (t *Tracker) enqueueSavedReportLocked(a *actorSnapshots) { + if t.reporting.reporter == nil || !a.reportDue() { + return + } + var delay time.Duration + if t.reporting.startupJitter > 0 { + delay = time.Nanosecond + rand.N(t.reporting.startupJitter) + } + t.reporting.queue.AddAfter(a.actorUID, delay) +} + +// runReporter sends the reports of the actors it takes from the report queue, +// until the queue shuts down. +func (t *Tracker) runReporter() { + for { + actorUID, shutdown := t.reporting.queue.Get() + if shutdown { + return + } + delay := t.report(actorUID) + t.reporting.queue.Done(actorUID) + if !t.waitReportQueue(delay) { + return + } + } +} + +// waitReportQueue pauses the report worker for delay, or returns false if the +// tracker shuts down first. +func (t *Tracker) waitReportQueue(delay time.Duration) bool { + if delay <= 0 { + return true + } + timer := t.clock.NewTimer(delay) + defer timer.Stop() + select { + case <-timer.C(): + return true + case <-t.ctx.Done(): + return false + } +} + +// report sends the actor's state to the control plane if a report is due, and +// returns how long the report queue should pause before the next report. A +// report the control plane acknowledges or refuses is no longer due, unless a +// background change was made while it ran. Any other failure is retried after +// a backoff, until a commit or detach cancels it or giveUpAfter has passed +// since the report became due; a failure other than Aborted also backs off +// the whole report queue. +func (t *Tracker) report(actorUID string) time.Duration { + actor, sent, dueSince, ok := t.beginReport(actorUID) + if !ok { + return 0 + } + ctx, cancel := context.WithTimeout(t.ctx, reportTimeout) + err := t.reporting.reporter.ReportActorSnapshotState(ctx, actorUID, actor, sent) + cancel() + attrs := []any{ + slog.String("actorUID", actorUID), + slog.Any("actor", actor), + slog.String("latestLocal", sent.GetLatestLocal().GetSnapshot().GetSnapshotUuid()), + slog.String("latestDurable", sent.GetLatestDurable().GetSnapshot().GetSnapshotUuid()), + } + switch { + case err == nil: + slog.Info("Reported the actor's snapshot state", attrs...) + case reportRetryable(err): + if t.ctx.Err() != nil { + // atelet is shutting down: leave the report due for the next + // atelet to send. + slog.Info("Failed to report the actor's snapshot state", append(attrs, slog.Any("err", err))...) + return 0 + } + var queueDelay time.Duration + if status.Code(err) == codes.Aborted { + // The control plane answered; only this actor conflicted. + t.reporting.queueRetries.Forget("") + } else { + queueDelay = jitteredReportDelay(t.reporting.queueRetries.When("")) + } + t.retryReport(actorUID, dueSince, attrs, err) + return queueDelay + case status.Code(err) == codes.NotFound || status.Code(err) == codes.Unimplemented: + // NotFound is expected once the actor is deleted or a suspend has + // released it from this node; Unimplemented is expected during a + // rolling upgrade while ate-api-server is older than atelet. + slog.Info("The control plane refused the actor's snapshot state", append(attrs, slog.Any("err", err))...) + default: + slog.Warn("The control plane refused the actor's snapshot state", append(attrs, slog.Any("err", err))...) + } + t.reporting.queueRetries.Forget("") + t.reporting.retries.Forget(actorUID) + t.endReport(actorUID, sent) + return 0 +} + +// jitteredReportDelay returns a random delay in [d/2, d), so nodes that hit +// the same control-plane failure do not retry in lockstep. +func jitteredReportDelay(d time.Duration) time.Duration { + if half := d / 2; half > 0 { + return half + rand.N(d-half) + } + return d +} + +// beginReport returns the actor, the state to report for it, and when the +// report became due, or false when no report is due or a Run, Restore, +// Checkpoint, or PromoteSnapshot of the actor is in progress. +func (t *Tracker) beginReport(actorUID string) (resources.ActorRef, *ateletpb.ActorSnapshotState, time.Time, bool) { + if !t.locks.Lock(t.ctx, actorUID) { + // atelet is shutting down. + return resources.ActorRef{}, nil, time.Time{}, false + } + defer t.unlock(actorUID) + a := t.findLocked(actorUID) + if a == nil || !a.reportDue() { + t.reporting.retries.Forget(actorUID) + return resources.ActorRef{}, nil, time.Time{}, false + } + if a.holds > 0 { + // A Run, Restore, Checkpoint, or PromoteSnapshot of the actor is in + // progress, and queues the report when it ends (see holdLocked). + a.reportDeferred = true + return resources.ActorRef{}, nil, time.Time{}, false + } + a.reportDeferred = false + return a.latestDurableActor(), t.stateLocked(a), a.ReportDueSince, true +} + +// endReport records that the control plane acknowledged or refused a report +// of sent, so the report is no longer due. If a background change modified +// the actor's state while sent was in flight, it queues the actor to report +// again instead. +func (t *Tracker) endReport(actorUID string, sent *ateletpb.ActorSnapshotState) { + t.lock(actorUID) + defer t.unlock(actorUID) + a := t.findLocked(actorUID) + if a == nil || !a.reportDue() { + return + } + if !sameReport(sent, t.stateLocked(a)) { + t.queueReportLocked(a) + return + } + t.clearReportDueLocked(a) +} + +// retryReport schedules another attempt to send the report that became due at +// dueSince after err, unless a commit or detach canceled it, a newer +// background change replaced it, or giveUpAfter has passed. +func (t *Tracker) retryReport(actorUID string, dueSince time.Time, attrs []any, err error) { + t.lock(actorUID) + a := t.findLocked(actorUID) + if a == nil || !a.reportDue() { + t.reporting.retries.Forget(actorUID) + t.unlock(actorUID) + return + } + if !a.ReportDueSince.Equal(dueSince) { + t.queueReportLocked(a) + t.unlock(actorUID) + return + } + now := t.clock.Now() + if !now.Before(dueSince.Add(t.reporting.giveUpAfter)) { + t.clearReportDueLocked(a) + t.unlock(actorUID) + slog.Warn("Giving up on reporting the actor's snapshot state", append(attrs, slog.Duration("after", now.Sub(dueSince)), slog.Any("err", err))...) + return + } + delay := t.reporting.retries.When(actorUID) + t.unlock(actorUID) + + level := slog.LevelWarn + if status.Code(err) == codes.Aborted { + // Another control-plane operation is changing the actor. + level = slog.LevelInfo + } + slog.Log(t.ctx, level, "Failed to report the actor's snapshot state", append(attrs, slog.Duration("retryAfter", delay), slog.Any("err", err))...) + t.reporting.queue.AddAfter(actorUID, delay) +} + +// clearReportDueLocked clears ReportDueSince, resets the report backoff, and +// saves the actor's state. +func (t *Tracker) clearReportDueLocked(a *actorSnapshots) { + t.addLocked(a) + t.cancelReportLocked(a) + t.saveLocked(a) + t.requeueLocked(a) +} + +// sameReport reports whether a and b name the same LatestLocal and +// LatestDurable. It ignores uploading and upload_due_time, whose changes do +// not make a report due. +func sameReport(a, b *ateletpb.ActorSnapshotState) bool { + return proto.Equal(a.GetLatestLocal(), b.GetLatestLocal()) && + proto.Equal(a.GetLatestDurable(), b.GetLatestDurable()) +} + +// reportRetryable reports whether a report that failed with err is worth +// sending again. The control plane refuses a report with NotFound once the +// actor is gone or no longer on this node, with FailedPrecondition or +// InvalidArgument when it conflicts with the actor's record, and with +// Unimplemented when it does not serve reports: sending it again only repeats +// the refusal. Anything else, such as Aborted while another operation changes +// the actor, or Unavailable, can pass on a later attempt. +func reportRetryable(err error) bool { + switch status.Code(err) { + case codes.NotFound, codes.FailedPrecondition, codes.InvalidArgument, codes.Unimplemented: + return false + default: + return true + } +} diff --git a/cmd/atelet/internal/snapshottracker/report_test.go b/cmd/atelet/internal/snapshottracker/report_test.go new file mode 100644 index 0000000000..f82b8c3f8a --- /dev/null +++ b/cmd/atelet/internal/snapshottracker/report_test.go @@ -0,0 +1,1271 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package snapshottracker + +import ( + "context" + "errors" + "log/slog" + "os" + "slices" + "sync" + "testing" + "time" + + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "k8s.io/client-go/util/workqueue" + clocktesting "k8s.io/utils/clock/testing" +) + +// The report retry delays of the trackers startReporting sets up. A minute is +// well short of the test upload and detach delays, so stepping the clock past +// a retry starts no other background work. +const ( + testReportRetryDelay = time.Minute + testMaxReportRetryDelay = 10 * time.Minute +) + +// sentReport is one report a fakeReporter received. +type sentReport struct { + actorUID string + actor resources.ActorRef + state *ateletpb.ActorSnapshotState +} + +// fakeReporter is a Reporter that records its reports. +type fakeReporter struct { + // reported receives each report as it starts. + reported chan sentReport + // release ends a blocked report with the error it receives. + release chan error + + mu sync.Mutex + sent []sentReport + errs []error + blocking bool +} + +func newFakeReporter() *fakeReporter { + return &fakeReporter{reported: make(chan sentReport, 100), release: make(chan error)} +} + +// failNext makes the next reports fail with errs, in order. +func (f *fakeReporter) failNext(errs ...error) { + f.mu.Lock() + defer f.mu.Unlock() + f.errs = append(f.errs, errs...) +} + +// setBlocking makes later reports wait for release, or their context. +func (f *fakeReporter) setBlocking(blocking bool) { + f.mu.Lock() + defer f.mu.Unlock() + f.blocking = blocking +} + +// reports returns the reports received so far. +func (f *fakeReporter) reports() []sentReport { + f.mu.Lock() + defer f.mu.Unlock() + return slices.Clone(f.sent) +} + +func (f *fakeReporter) ReportActorSnapshotState(ctx context.Context, actorUID string, actor resources.ActorRef, state *ateletpb.ActorSnapshotState) error { + r := sentReport{actorUID: actorUID, actor: actor, state: state} + f.mu.Lock() + f.sent = append(f.sent, r) + var err error + if len(f.errs) > 0 { + err, f.errs = f.errs[0], f.errs[1:] + } + blocking := f.blocking + f.mu.Unlock() + f.reported <- r + if blocking { + select { + case err = <-f.release: + case <-ctx.Done(): + if err == nil { + // As a gRPC client fails a call whose context ends. + err = status.FromContextError(ctx.Err()).Err() + } + } + } + return err +} + +// committedRef is testRef's snapshot as a commit records it, naming testActor. +func committedRef(uuid string, generation int32, commitTime time.Time) SnapshotRef { + ref := testRef(uuid, generation, commitTime) + ref.Actor = testActor + return ref +} + +// startReporting makes tr report through reporter, retrying a failed report +// testReportRetryDelay later, and starts it without report workers: a test +// sends each report with processNextReport. +func startReporting(tr *Tracker, reporter Reporter) { + tr.reporting.reporter = reporter + tr.reporting.workers = 0 + tr.reporting.retries = workqueue.NewTypedItemExponentialFailureRateLimiter[string](testReportRetryDelay, testMaxReportRetryDelay) + tr.reporting.queueRetries = workqueue.NewTypedItemExponentialFailureRateLimiter[string](testReportRetryDelay, testMaxReportRetryDelay) + tr.reporting.startupJitter = 0 + tr.Start() +} + +// newReportingTracker returns a tracker like newTestTracker's that reports +// through reporter, as startReporting sets up. +func newReportingTracker(t *testing.T, store Store, reporter Reporter) *Tracker { + t.Helper() + tr := newIdleTestTracker(t, store) + startReporting(tr, reporter) + return tr +} + +// processNextReport takes the next actor from the reports queue of a tracker +// without report workers, and reports it as a report worker would. +func processNextReport(t *testing.T, tr *Tracker) { + t.Helper() + next := make(chan string, 1) + go func() { + if actorUID, shutdown := tr.reporting.queue.Get(); !shutdown { + next <- actorUID + } + }() + select { + case actorUID := <-next: + tr.report(actorUID) + tr.reporting.queue.Done(actorUID) + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for a report to be queued") + } +} + +// waitReported waits for reporter's next report to start, and returns it. +func waitReported(t *testing.T, reporter *fakeReporter) sentReport { + t.Helper() + select { + case r := <-reporter.reported: + return r + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for a report") + return sentReport{} + } +} + +// reportDue reports whether a report of the tracked actor is due. +func reportDue(tr *Tracker) bool { + tr.lock(trackedActorUID) + defer tr.unlock(trackedActorUID) + a := tr.findLocked(trackedActorUID) + return a != nil && a.reportDue() +} + +// reportDueOnDisk reports whether the tracked actor's snapshot state file +// records a report due. +func reportDueOnDisk() bool { + a := readSnapshotState(trackedActorUID) + return a != nil && a.reportDue() +} + +// A background upload that makes a snapshot durable reports the actor's whole +// state, for the actor the snapshot's commit named. Once the control plane +// acknowledges the report, it is no longer due, in memory or on disk. +func TestTrackerReportsBackgroundUpload(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + reporter := newFakeReporter() + tr := newReportingTracker(t, store, reporter) + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + uploadNow(t, tr, store, nil) + if !reportDueOnDisk() { + t.Fatal("state file after the background upload records no report due") + } + + processNextReport(t, tr) + sent := reporter.reports() + if len(sent) != 1 { + t.Fatalf("reports = %d, want 1", len(sent)) + } + if got := sent[0]; got.actorUID != trackedActorUID || got.actor != testActor { + t.Errorf("report for actor %q %+v, want %q %+v", got.actorUID, got.actor, trackedActorUID, testActor) + } + st := sent[0].state + if uuidOf(st.GetLatestDurable()) != "s1" || uuidOf(st.GetLatestLocal()) != "s1" { + t.Errorf("reported state = %v, want s1 durable and still local", st) + } + if want := tr.State(trackedActorUID); !proto.Equal(st, want) { + t.Errorf("reported state = %v, want the actor's state %v", st, want) + } + if reportDue(tr) || reportDueOnDisk() { + t.Error("report still due once the control plane acknowledged it") + } + if n := tr.reporting.queue.Len(); n != 0 { + t.Errorf("reports queued = %d once acknowledged, want none", n) + } +} + +// A background detach reports the actor's state once its local snapshots are +// gone: nothing local or uploading, which the control plane reads as a +// detach, and the detached snapshot durable. +func TestTrackerReportsBackgroundDetach(t *testing.T) { + wantDetached := func(t *testing.T, got sentReport) { + t.Helper() + st := got.state + if st.GetLatestLocal() != nil || st.GetUploading() != nil || uuidOf(st.GetLatestDurable()) != "s1" { + t.Errorf("reported state = %v, want nothing local or uploading, and s1 durable", st) + } + if got.actor != testActor { + t.Errorf("report for actor %+v, want %+v", got.actor, testActor) + } + } + + t.Run("that uploads the snapshot", func(t *testing.T) { + store := newFakeSnapshotStore() + reporter := newFakeReporter() + tr := newReportingTracker(t, store, reporter) + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + // With no upload workers, the detach runs and makes s1 durable itself, + // and reports once, at its end. + detachNow(t, tr) + processNextReport(t, tr) + sent := reporter.reports() + if len(sent) != 1 { + t.Fatalf("reports = %d, want 1", len(sent)) + } + wantDetached(t, sent[0]) + if reportDue(tr) || tr.reporting.queue.Len() != 0 { + t.Errorf("report due = %v, queued = %d once acknowledged, want neither", reportDue(tr), tr.reporting.queue.Len()) + } + }) + + t.Run("after a background upload", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + reporter := newFakeReporter() + tr := newReportingTracker(t, store, reporter) + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + uploadNow(t, tr, store, nil) + processNextReport(t, tr) + + detachNow(t, tr) + processNextReport(t, tr) + sent := reporter.reports() + if len(sent) != 2 { + t.Fatalf("reports = %d, want one for the upload and one for the detach", len(sent)) + } + wantDetached(t, sent[1]) + }) +} + +// A Detach call makes no report due: its caller learns the actor's state from +// the response. +func TestTrackerDoesNotReportDetachCall(t *testing.T) { + store := newFakeSnapshotStore() + tr := newReportingTracker(t, store, newFakeReporter()) + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + if _, err := tr.Detach(t.Context(), trackedActorUID); err != nil { + t.Fatalf("detach: %v", err) + } + if reportDue(tr) || tr.reporting.queue.Len() != 0 { + t.Errorf("report due = %v, queued = %d after a Detach call, want neither", reportDue(tr), tr.reporting.queue.Len()) + } +} + +// A background upload or detach makes no report due when it changes neither +// LatestDurable nor LatestLocal. +func TestTrackerDoesNotReportWhenNothingChanged(t *testing.T) { + for _, tc := range []struct { + name string + snap SnapshotRef + }{ + {"newer durable snapshot", committedRef("s2", 2, time.Time{})}, + {"same generation already durable", committedRef("s1", 1, time.Time{})}, + } { + t.Run("upload overtaken by "+tc.name, func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newReportingTracker(t, store, newFakeReporter()) + t0 := tr.clock.Now() + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, t0)) + done := startUpload(t, tr, store) + durable := tc.snap + durable.CommitTime = t0.Add(time.Minute) + durable.UploadTime = durable.CommitTime.Add(time.Second) + tr.CommitDurable(trackedActorUID, durable) + + store.release <- nil + waitClosed(t, done, "the upload of s1") + if reportDue(tr) || tr.reporting.queue.Len() != 0 { + t.Errorf("report due = %v, queued = %d after an overtaken upload, want neither", reportDue(tr), tr.reporting.queue.Len()) + } + if got := timeOf(tr.State(trackedActorUID).GetLatestDurable().GetDurableCompleteTime()); !got.Equal(durable.UploadTime) { + t.Errorf("durable_complete_time = %v, want %v from CommitDurable", got, durable.UploadTime) + } + }) + } + + t.Run("failed detach of already durable snapshot", func(t *testing.T) { + store := newFakeSnapshotStore() + tr := newIdleTestTracker(t, store) + tr.reporting.reporter = newFakeReporter() + t0 := tr.clock.Now() + local := committedRef("s1", 1, t0) + durable := local + durable.UploadTime = t0.Add(time.Second) + tr.CommitDurable(trackedActorUID, durable) + tr.CommitLocal(trackedActorUID, local) + // A file where the local checkpoint directory belongs fails the prune. + if err := os.WriteFile(ateletpath.LocalCheckpointsDir(trackedActorUID), nil, 0o600); err != nil { + t.Fatal(err) + } + + stepTo(tr, t0.Add(testDetachDelay)) + processNext(t, tr, opDetach) + if reportDue(tr) || tr.reporting.queue.Len() != 0 { + t.Errorf("report due = %v, queued = %d after a detach that changed nothing, want neither", reportDue(tr), tr.reporting.queue.Len()) + } + }) +} + +// A report that fails for a reason a later attempt can pass is retried once +// the retry delay has passed, not before, and stays due until the control +// plane acknowledges it. Aborted is expected while another operation changes +// the actor, so it is logged at Info; other retryable errors warn. +func TestTrackerRetriesFailedReport(t *testing.T) { + for _, tc := range []struct { + err error + level slog.Level + }{ + {status.Error(codes.Aborted, "another operation is changing the actor"), slog.LevelInfo}, + {status.Error(codes.Unavailable, "ateapi is restarting"), slog.LevelWarn}, + {status.Error(codes.DeadlineExceeded, "ateapi is slow"), slog.LevelWarn}, + {status.Error(codes.Canceled, "call canceled while atelet is still running"), slog.LevelWarn}, + {errors.New("not a status"), slog.LevelWarn}, + } { + t.Run(status.Code(tc.err).String(), func(t *testing.T) { + levels := recordLogLevels(t, "Failed to report the actor's snapshot state") + store := newFakeSnapshotStore() + store.setBlocking(true) + reporter := newFakeReporter() + reporter.failNext(tc.err) + tr := newReportingTracker(t, store, reporter) + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + uploadNow(t, tr, store, nil) + + failedAt := tr.clock.Now() + processNextReport(t, tr) + if !reportDue(tr) || !reportDueOnDisk() { + t.Fatal("report no longer due after a failed attempt") + } + if got := levels(); !slices.Equal(got, []slog.Level{tc.level}) { + t.Errorf("failed report logged at %v, want once at %v", got, tc.level) + } + stepTo(tr, failedAt.Add(testReportRetryDelay-time.Second)) + time.Sleep(50 * time.Millisecond) + if n := tr.reporting.queue.Len(); n != 0 { + t.Fatalf("reports queued = %d before the retry delay passed, want none", n) + } + + stepTo(tr, failedAt.Add(testReportRetryDelay)) + processNextReport(t, tr) + if got := len(reporter.reports()); got != 2 { + t.Errorf("reports = %d, want the failed one and its retry", got) + } + if reportDue(tr) || reportDueOnDisk() { + t.Error("report still due once the retry was acknowledged") + } + }) + } +} + +// A report that is still failing once giveUpAfter has passed since it +// became due is dropped, so atelet stops retrying it. A later background +// change starts a fresh window. +func TestTrackerGivesUpOnReportAfterTimeout(t *testing.T) { + t.Run("drops after 10 minutes", func(t *testing.T) { + gaveUp := recordLogLevels(t, "Giving up on reporting the actor's snapshot state") + store := newFakeSnapshotStore() + store.setBlocking(true) + reporter := newFakeReporter() + unavailable := status.Error(codes.Unavailable, "ateapi is down") + reporter.failNext(unavailable, unavailable, unavailable) + tr := newReportingTracker(t, store, reporter) + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + uploadNow(t, tr, store, nil) + dueAt := tr.clock.Now() + + // Attempt 1 fails at dueAt; retry is queued for +1m. + processNextReport(t, tr) + // Step just short of the 10-minute window (+9m59s): attempt 2 fails + // and queues another retry (+2m, which lands past +10m). + stepTo(tr, dueAt.Add(DefaultReportGiveUpAfter-time.Second)) + processNextReport(t, tr) + if !reportDue(tr) || !reportDueOnDisk() { + t.Fatal("report dropped before DefaultReportGiveUpAfter elapsed") + } + if got := gaveUp(); len(got) != 0 { + t.Fatalf("gave up logged early at %v, want none yet", got) + } + + // Step past the retry delay, now beyond the 10-minute window. + stepTo(tr, tr.clock.Now().Add(2*testReportRetryDelay)) + processNextReport(t, tr) + if reportDue(tr) || reportDueOnDisk() { + t.Error("report still due after DefaultReportGiveUpAfter elapsed") + } + if got := gaveUp(); !slices.Equal(got, []slog.Level{slog.LevelWarn}) { + t.Errorf("give-up logged at %v, want once at Warn", got) + } + stepTo(tr, tr.clock.Now().Add(testMaxReportRetryDelay)) + time.Sleep(50 * time.Millisecond) + if n := tr.reporting.queue.Len(); n != 0 { + t.Errorf("reports queued = %d after giving up, want none", n) + } + + // A later background detach starts a fresh 10-minute window. + detachNow(t, tr) + processNextReport(t, tr) + sent := reporter.reports() + if len(sent) != 4 || sent[3].state.GetLatestLocal() != nil { + t.Errorf("reports after a later detach = %d (last = %v), want 4 with the detach reported", len(sent), sent[len(sent)-1].state) + } + }) + + t.Run("honors WithReportGiveUpAfter", func(t *testing.T) { + const giveUp = 2 * time.Minute + store := newFakeSnapshotStore() + store.setBlocking(true) + reporter := newFakeReporter() + unavailable := status.Error(codes.Unavailable, "ateapi is down") + reporter.failNext(unavailable, unavailable) + useActorDir(t) + tr := New(t.Context(), store, clocktesting.NewFakeClock(time.Unix(1_700_000_000, 0)), + WithUploadDelay(testUploadDelay), WithDetachDelay(testDetachDelay), WithReportGiveUpAfter(giveUp)) + startReporting(tr, reporter) + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + uploadNow(t, tr, store, nil) + dueAt := tr.clock.Now() + + processNextReport(t, tr) + stepTo(tr, dueAt.Add(giveUp)) + processNextReport(t, tr) + if reportDue(tr) || reportDueOnDisk() { + t.Error("report still due after configured give-up duration elapsed") + } + }) + + t.Run("keeps a newer report made due during the last attempt", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + reporter := newFakeReporter() + reporter.setBlocking(true) + tr := newIdleTestTracker(t, store) + tr.reporting.reporter = reporter + tr.Start() + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + uploadNow(t, tr, store, nil) + first := waitReported(t, reporter) + + // Advance past the 10-minute window of the upload report, then run a + // background detach while the upload report is still in flight. + stepTo(tr, tr.clock.Now().Add(DefaultReportGiveUpAfter)) + detachNow(t, tr) + + clk := tr.clock.(*clocktesting.FakeClock) + waitersBefore := clk.Waiters() + reporter.setBlocking(false) + reporter.release <- status.Error(codes.Unavailable, "ateapi was down") + eventually(t, "the report queue backs off", func() bool { return clk.Waiters() > waitersBefore }) + stepTo(tr, tr.clock.Now().Add(reportRetryDelay)) + second := waitReported(t, reporter) + if first.state.GetLatestLocal() == nil || second.state.GetLatestLocal() != nil { + t.Errorf("reports = (%v, %v), want the upload report then the detach report", first.state, second.state) + } + eventually(t, "the detach report is acknowledged", func() bool { return !reportDue(tr) }) + }) +} + +// WithReportWorkers(0) disables background reporting, even when WithReporter +// is set: neither saved reports at startup nor fresh background changes are +// queued or sent. +func TestTrackerWithReportWorkersZeroDisablesReporting(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + before := newTestTracker(t, store) + before.CommitLocal(trackedActorUID, committedRef("s1", 1, before.clock.Now())) + uploadNow(t, before, store, nil) + + reporter := newFakeReporter() + tr := New(t.Context(), store, clocktesting.NewFakeClock(time.Unix(1_700_000_000, 0)), + WithUploadDelay(testUploadDelay), WithDetachDelay(testDetachDelay), + WithReporter(reporter), WithReportWorkers(0)) + tr.Start() + t.Cleanup(func() { + if err := tr.Wait(context.Background()); err != nil { + t.Errorf("Wait: %v", err) + } + }) + + stepTo(tr, tr.clock.Now().Add(reportStartupJitter)) + tr.CommitLocal(trackedActorUID, committedRef("s2", 2, tr.clock.Now())) + uploadNow(t, tr, store, nil) + if n := tr.reporting.queue.Len(); n != 0 { + t.Errorf("reports queued = %d with 0 report workers, want none", n) + } + if got := len(reporter.reports()); got != 0 { + t.Errorf("reports sent = %d with 0 report workers, want none", got) + } +} + +// A control-plane failure backs off the whole report queue, so other actors +// waiting to report do not all hit an overloaded ateapi at once. Once an +// attempt gets a response, the queue backoff resets; Aborted, which means +// only that single actor conflicted, does not hold back other actors. +func TestTrackerBacksOffReportQueueOnServerFailure(t *testing.T) { + makeActorReportDue := func(t *testing.T, tr *Tracker, store *fakeSnapshotStore, actorUID, uuid string) { + t.Helper() + if err := os.MkdirAll(ateletpath.ActorPath(actorUID), 0o700); err != nil { + t.Fatal(err) + } + tr.CommitLocal(actorUID, committedRef(uuid, 1, tr.clock.Now())) + stepTo(tr, timeOf(tr.State(actorUID).GetUploadDueTime())) + waitStarted(t, store) + done := opDone(t, tr, actorUID) + store.release <- nil + waitClosed(t, done, "the upload") + } + + newQueueBackoffTracker := func(t *testing.T, store *fakeSnapshotStore, reporter *fakeReporter) *Tracker { + t.Helper() + tr := newIdleTestTracker(t, store) + tr.reporting.reporter = reporter + // Short of giveUpAfter so the second actor's upload step does not + // expire the first actor's report window. + tr.uploadDelay = 2 * time.Minute + tr.reporting.startupJitter = 0 + tr.reporting.retries = workqueue.NewTypedItemExponentialFailureRateLimiter[string](testReportRetryDelay, testMaxReportRetryDelay) + tr.reporting.queueRetries = workqueue.NewTypedItemExponentialFailureRateLimiter[string](testReportRetryDelay, testMaxReportRetryDelay) + tr.Start() + return tr + } + + t.Run("Unavailable holds back the next actor", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + reporter := newFakeReporter() + reporter.setBlocking(true) + tr := newQueueBackoffTracker(t, store, reporter) + + // Put actor-uid-1 in flight in reporter, and queue actor-uid-2 and + // actor-uid-3 behind it. + makeActorReportDue(t, tr, store, "actor-uid-1", "s1") + if got := waitReported(t, reporter); got.actorUID != "actor-uid-1" { + t.Fatalf("first report for %q, want actor-uid-1", got.actorUID) + } + makeActorReportDue(t, tr, store, "actor-uid-2", "s2") + makeActorReportDue(t, tr, store, "actor-uid-3", "s3") + + clk := tr.clock.(*clocktesting.FakeClock) + waitersBefore := clk.Waiters() + failedAt := clk.Now() + reporter.setBlocking(false) + reporter.release <- status.Error(codes.Unavailable, "ateapi overloaded") + // Wait until both actor-uid-1's retry and the queue backoff timer are waiting. + eventually(t, "the report queue backs off", func() bool { return clk.Waiters() >= waitersBefore+2 }) + + // The jittered queue delay is in [testReportRetryDelay/2, testReportRetryDelay). + stepTo(tr, failedAt.Add(testReportRetryDelay/2-time.Second)) + time.Sleep(50 * time.Millisecond) + if got := len(reporter.reports()); got != 1 { + t.Fatalf("reports before the queue backoff passed = %d, want only the 1 failed attempt", got) + } + + // Step past the jittered queue delay, just short of actor-uid-1's own retry. + stepTo(tr, failedAt.Add(testReportRetryDelay-time.Nanosecond)) + if got := waitReported(t, reporter); got.actorUID != "actor-uid-2" { + t.Errorf("second report for %q, want actor-uid-2", got.actorUID) + } + // actor-uid-2 succeeded, resetting the queue backoff: actor-uid-3 reports right away. + if got := waitReported(t, reporter); got.actorUID != "actor-uid-3" { + t.Errorf("third report for %q, want actor-uid-3 without stepping the clock", got.actorUID) + } + }) + + t.Run("Aborted does not hold back other actors", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + reporter := newFakeReporter() + reporter.setBlocking(true) + tr := newQueueBackoffTracker(t, store, reporter) + + makeActorReportDue(t, tr, store, "actor-uid-1", "s1") + if got := waitReported(t, reporter); got.actorUID != "actor-uid-1" { + t.Fatalf("first report for %q, want actor-uid-1", got.actorUID) + } + makeActorReportDue(t, tr, store, "actor-uid-2", "s2") + + reporter.setBlocking(false) + reporter.release <- status.Error(codes.Aborted, "another operation is changing the actor") + if got := waitReported(t, reporter); got.actorUID != "actor-uid-2" { + t.Errorf("second report for %q, want actor-uid-2 without stepping the clock", got.actorUID) + } + }) + + t.Run("queue backoff caps at maxReportQueueRetryDelay", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + reporter := newFakeReporter() + unavailable := status.Error(codes.Unavailable, "ateapi down") + reporter.failNext(unavailable, unavailable, unavailable, unavailable, unavailable) + tr := newIdleTestTracker(t, store) + tr.reporting.reporter = reporter + tr.reporting.workers = 0 + tr.reporting.startupJitter = 0 + tr.Start() + + var delay time.Duration + for i := range 5 { + actorUID := "actor-uid-cap-" + string(rune('1'+i)) + makeActorReportDue(t, tr, store, actorUID, "s1") + delay = tr.report(actorUID) + } + if delay < maxReportQueueRetryDelay/2 || delay >= maxReportQueueRetryDelay { + t.Errorf("5th queue backoff delay = %v, want in [%v, %v)", delay, maxReportQueueRetryDelay/2, maxReportQueueRetryDelay) + } + }) +} + +// A report the control plane refuses, about an actor gone from this node or +// one that conflicts with its record, or because it does not serve reports, +// is not retried and no longer due. NotFound and Unimplemented are expected, +// so they are logged at Info rather than Warn. +func TestTrackerDropsRefusedReport(t *testing.T) { + for _, tc := range []struct { + code codes.Code + level slog.Level + }{ + {codes.NotFound, slog.LevelInfo}, + {codes.FailedPrecondition, slog.LevelWarn}, + {codes.InvalidArgument, slog.LevelWarn}, + {codes.Unimplemented, slog.LevelInfo}, + } { + t.Run(tc.code.String(), func(t *testing.T) { + levels := recordLogLevels(t, "The control plane refused the actor's snapshot state") + store := newFakeSnapshotStore() + store.setBlocking(true) + reporter := newFakeReporter() + reporter.failNext(status.Error(tc.code, "refused")) + tr := newReportingTracker(t, store, reporter) + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + uploadNow(t, tr, store, nil) + + processNextReport(t, tr) + if reportDue(tr) || reportDueOnDisk() { + t.Error("report still due after the control plane refused it") + } + if got := levels(); !slices.Equal(got, []slog.Level{tc.level}) { + t.Errorf("refusal logged at %v, want once at %v", got, tc.level) + } + stepTo(tr, tr.clock.Now().Add(testMaxReportRetryDelay)) + time.Sleep(50 * time.Millisecond) + if n := tr.reporting.queue.Len(); n != 0 { + t.Errorf("reports queued = %d after a refusal, want none", n) + } + }) + } +} + +// recordLogLevels sends the default logger's records to a handler that +// records the level of each one logged with msg, until the test ends, and +// returns a function that lists those levels. +func recordLogLevels(t *testing.T, msg string) func() []slog.Level { + t.Helper() + h := &levelRecorder{msg: msg} + prev := slog.Default() + slog.SetDefault(slog.New(h)) + t.Cleanup(func() { slog.SetDefault(prev) }) + return func() []slog.Level { + h.mu.Lock() + defer h.mu.Unlock() + return slices.Clone(h.levels) + } +} + +// levelRecorder is a slog.Handler that records the level of each record +// logged with msg. +type levelRecorder struct { + msg string + mu sync.Mutex + levels []slog.Level +} + +func (h *levelRecorder) Enabled(context.Context, slog.Level) bool { return true } + +func (h *levelRecorder) Handle(_ context.Context, r slog.Record) error { + if r.Message == h.msg { + h.mu.Lock() + h.levels = append(h.levels, r.Level) + h.mu.Unlock() + } + return nil +} + +func (h *levelRecorder) WithAttrs([]slog.Attr) slog.Handler { return h } + +func (h *levelRecorder) WithGroup(string) slog.Handler { return h } + +// A report due survives an atelet restart: a tracker started over the same +// node directories sends it. Once it is acknowledged, a further restart sends +// nothing. +func TestTrackerReportsAfterRestart(t *testing.T) { + t.Run("sends saved report and clears it", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + // The first tracker has no reporter: it saves the report due, as an + // atelet that restarts before sending it does, but queues nothing. + before := newTestTracker(t, store) + before.CommitLocal(trackedActorUID, committedRef("s1", 1, before.clock.Now())) + uploadNow(t, before, store, nil) + if n := before.reporting.queue.Len(); n != 0 { + t.Errorf("reports queued = %d without a reporter, want none", n) + } + + reporter := newFakeReporter() + after := newStoreTracker(t, store) + startReporting(after, reporter) + processNextReport(t, after) + sent := reporter.reports() + if len(sent) != 1 || uuidOf(sent[0].state.GetLatestDurable()) != "s1" || sent[0].actor != testActor { + t.Fatalf("reports after the restart = %+v, want one of s1 durable, for %+v", sent, testActor) + } + + again := newStoreTracker(t, store) + startReporting(again, reporter) + if n := again.reporting.queue.Len(); n != 0 { + t.Errorf("reports queued after a second restart = %d, want 0", n) + } + if got := len(reporter.reports()); got != 1 { + t.Errorf("reports after a second restart = %d, want still 1", got) + } + }) + + t.Run("jitters saved reports at startup", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + before := newTestTracker(t, store) + before.CommitLocal(trackedActorUID, committedRef("s1", 1, before.clock.Now())) + uploadNow(t, before, store, nil) + + reporter := newFakeReporter() + after := newStoreTracker(t, store) + after.reporting.reporter = reporter + after.reporting.workers = 0 + after.Start() + if n := after.reporting.queue.Len(); n != 0 { + t.Fatalf("reports queued right at startup = %d, want delayed by startup jitter", n) + } + + stepTo(after, after.clock.Now().Add(reportStartupJitter)) + processNextReport(t, after) + if got := len(reporter.reports()); got != 1 { + t.Errorf("reports after startup jitter elapsed = %d, want 1", got) + } + }) + + t.Run("fresh background change does not wait for startup jitter", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + before := newTestTracker(t, store) + before.CommitLocal(trackedActorUID, committedRef("s1", 1, before.clock.Now())) + uploadNow(t, before, store, nil) + + reporter := newFakeReporter() + after := newStoreTracker(t, store) + after.reporting.reporter = reporter + after.reporting.workers = 0 + after.Start() + + detachNow(t, after) + processNextReport(t, after) + sent := reporter.reports() + if len(sent) != 1 || sent[0].state.GetLatestLocal() != nil { + t.Errorf("reports after fresh detach = %+v, want 1 detached report without waiting for startup jitter", sent) + } + }) +} + +// A report whose actor's LatestLocal or LatestDurable changed in the +// background while it ran is sent again with the new state, and stays due +// until a report of the current state is acknowledged. A change to Uploading +// alone does not queue another report. +func TestTrackerReportsStateChangedDuringReport(t *testing.T) { + t.Run("reports again when LatestLocal changes", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + reporter := newFakeReporter() + reporter.setBlocking(true) + tr := newIdleTestTracker(t, store) + tr.reporting.reporter = reporter + tr.Start() + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + uploadNow(t, tr, store, nil) + first := waitReported(t, reporter) + + detachNow(t, tr) + reporter.setBlocking(false) + reporter.release <- nil + second := waitReported(t, reporter) + if got, want := []string{uuidOf(first.state.GetLatestLocal()), uuidOf(second.state.GetLatestLocal())}, []string{"s1", ""}; !slices.Equal(got, want) { + t.Errorf("reported latest_local = %v, want %v", got, want) + } + eventually(t, "the report is no longer due", func() bool { return !reportDue(tr) }) + }) + + t.Run("ignores Uploading starting during report", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + reporter := newFakeReporter() + reporter.setBlocking(true) + tr := newIdleTestTracker(t, store) + tr.reporting.reporter = reporter + tr.Start() + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + s1Done := startUpload(t, tr, store) + tr.CommitLocal(trackedActorUID, committedRef("s2", 2, tr.clock.Now())) + store.release <- nil + waitClosed(t, s1Done, "s1 upload") + first := waitReported(t, reporter) + if first.state.GetUploading() != nil { + t.Fatalf("first report uploading = %v, want nil", first.state.GetUploading()) + } + + // Start s2's background upload while s1's report is still in flight. + s2Done := startUpload(t, tr, store) + reporter.setBlocking(false) + reporter.release <- nil + eventually(t, "s1 report finishes", func() bool { return !reportDue(tr) }) + if got := len(reporter.reports()); got != 1 { + t.Errorf("reports = %d after Uploading started during s1 report, want only 1", got) + } + + // Releasing s2's upload then reports s2 durable as usual. + store.release <- nil + waitClosed(t, s2Done, "s2 upload") + second := waitReported(t, reporter) + if got := uuidOf(second.state.GetLatestDurable()); got != "s2" { + t.Errorf("second report latest_durable = %q, want s2", got) + } + }) +} + +// A Checkpoint, a PromoteSnapshot while a local snapshot is kept, or a Detach +// call returns the actor's latest state to the control plane, so it cancels +// any pending background report, whether queued in retry backoff or already in +// flight. +func TestTrackerCancelsReportOnForegroundCall(t *testing.T) { + for _, tc := range []struct { + name string + alreadyDetached bool + call func(t *testing.T, tr *Tracker) + }{ + { + name: "CommitLocal", + call: func(_ *testing.T, tr *Tracker) { + tr.CommitLocal(trackedActorUID, committedRef("s2", 2, tr.clock.Now())) + }, + }, + { + name: "CommitDurable", + call: func(_ *testing.T, tr *Tracker) { + tr.CommitDurable(trackedActorUID, committedRef("s2", 2, tr.clock.Now())) + }, + }, + { + name: "CommitPromoted", + call: func(_ *testing.T, tr *Tracker) { + tr.CommitPromoted(trackedActorUID, committedRef("s2", 2, tr.clock.Now())) + }, + }, + { + name: "CommitPromoted of already durable snapshot", + call: func(_ *testing.T, tr *Tracker) { + tr.CommitPromoted(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + }, + }, + { + name: "Detach with local snapshot", + call: func(t *testing.T, tr *Tracker) { + if _, err := tr.Detach(t.Context(), trackedActorUID); err != nil { + t.Fatalf("Detach: %v", err) + } + }, + }, + { + name: "Detach when already detached", + alreadyDetached: true, + call: func(t *testing.T, tr *Tracker) { + if _, err := tr.Detach(t.Context(), trackedActorUID); err != nil { + t.Fatalf("Detach: %v", err) + } + }, + }, + } { + makeReportDue := func(t *testing.T, tr *Tracker, store *fakeSnapshotStore) { + t.Helper() + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + if tc.alreadyDetached { + store.setBlocking(false) + detachNow(t, tr) + } else { + store.setBlocking(true) + uploadNow(t, tr, store, nil) + store.setBlocking(false) + } + } + + t.Run(tc.name+"/queued in retry backoff", func(t *testing.T) { + store := newFakeSnapshotStore() + reporter := newFakeReporter() + reporter.failNext(status.Error(codes.Unavailable, "ateapi is restarting")) + tr := newReportingTracker(t, store, reporter) + makeReportDue(t, tr, store) + processNextReport(t, tr) + if !reportDue(tr) || !reportDueOnDisk() { + t.Fatal("report not due before the foreground call") + } + + tc.call(t, tr) + if reportDue(tr) || reportDueOnDisk() { + t.Error("report still due after the foreground call") + } + stepTo(tr, tr.clock.Now().Add(testMaxReportRetryDelay)) + processNextReport(t, tr) + if got := len(reporter.reports()); got != 1 { + t.Errorf("reports = %d, want only the 1 attempt before the foreground call", got) + } + }) + + for _, inFlight := range []struct { + name string + err error + }{ + {"in flight succeeds", nil}, + {"in flight fails with Aborted", status.Error(codes.Aborted, "actor is SUSPENDING")}, + } { + t.Run(tc.name+"/"+inFlight.name, func(t *testing.T) { + failedLogs := recordLogLevels(t, "Failed to report the actor's snapshot state") + store := newFakeSnapshotStore() + reporter := newFakeReporter() + reporter.setBlocking(true) + tr := newReportingTracker(t, store, reporter) + makeReportDue(t, tr, store) + done := make(chan struct{}) + go func() { + defer close(done) + processNextReport(t, tr) + }() + waitReported(t, reporter) + + tc.call(t, tr) + if reportDue(tr) || reportDueOnDisk() { + t.Fatal("report still due right after the foreground call") + } + reporter.setBlocking(false) + reporter.release <- inFlight.err + waitClosed(t, done, "the in-flight report") + + stepTo(tr, tr.clock.Now().Add(testMaxReportRetryDelay)) + time.Sleep(50 * time.Millisecond) + if reportDue(tr) || reportDueOnDisk() { + t.Error("report became due again after the canceled in-flight report ended") + } + if n := tr.reporting.queue.Len(); n != 0 { + t.Errorf("reports queued = %d after the canceled in-flight report ended, want none", n) + } + if got := failedLogs(); len(got) != 0 { + t.Errorf("canceled in-flight report logged failure at %v, want none", got) + } + }) + } + } +} + +// CommitPromoted after a background detach cleared LatestLocal leaves the +// pending detach report due, since the control plane does not read a promote's +// response as a detach. +func TestTrackerKeepsDetachReportOnCommitPromotedWhenDetached(t *testing.T) { + store := newFakeSnapshotStore() + reporter := newFakeReporter() + tr := newReportingTracker(t, store, reporter) + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + detachNow(t, tr) + if !reportDue(tr) || !reportDueOnDisk() { + t.Fatal("detach report not due before CommitPromoted") + } + + tr.CommitPromoted(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + if !reportDue(tr) || !reportDueOnDisk() { + t.Fatal("detach report canceled by CommitPromoted when LatestLocal was nil") + } + processNextReport(t, tr) + sent := reporter.reports() + if len(sent) != 1 || sent[0].state.GetLatestLocal() != nil || uuidOf(sent[0].state.GetLatestDurable()) != "s1" { + t.Errorf("reports = %+v, want 1 detach report with s1 durable", sent) + } + if reportDue(tr) || reportDueOnDisk() { + t.Error("detach report still due once acknowledged") + } +} + +// Forget does not wait for a report in progress, which records nothing once +// it ends. +func TestTrackerForgetDuringReport(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + reporter := newFakeReporter() + reporter.setBlocking(true) + tr := newIdleTestTracker(t, store) + tr.reporting.reporter = reporter + tr.Start() + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + uploadNow(t, tr, store, nil) + waitReported(t, reporter) + + returnsPromptly(t, "Forget during a report", func() { + if err := tr.Forget(trackedActorUID); err != nil { + t.Errorf("Forget: %v", err) + } + }) + reporter.release <- nil + select { + case r := <-reporter.reported: + t.Errorf("reported %v after Forget, want nothing", r.state) + case <-time.After(50 * time.Millisecond): + } + if _, err := os.Stat(ateletpath.ActorSnapshotStateFile(trackedActorUID)); !os.IsNotExist(err) { + t.Errorf("state file once the report ended: stat = %v, want none", err) + } +} + +// A report in progress as atelet shuts down stays due, for the next atelet to +// send, and is logged at Info rather than Warn, whether it fails with Canceled +// or another retryable error. +func TestTrackerReportStaysDueAtShutdown(t *testing.T) { + for _, tc := range []struct { + name string + err error + }{ + {"canceled by shutdown", nil}, + {"Unavailable during shutdown", status.Error(codes.Unavailable, "connection closing")}, + } { + t.Run(tc.name, func(t *testing.T) { + useActorDir(t) + levels := recordLogLevels(t, "Failed to report the actor's snapshot state") + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + store := newFakeSnapshotStore() + store.setBlocking(true) + reporter := newFakeReporter() + reporter.setBlocking(true) + if tc.err != nil { + reporter.failNext(tc.err) + } + tr := New(ctx, store, clocktesting.NewFakeClock(time.Unix(1_700_000_000, 0)), + WithUploadDelay(testUploadDelay), WithDetachDelay(testDetachDelay), WithReporter(reporter)) + tr.Start() + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + uploadNow(t, tr, store, nil) + waitReported(t, reporter) + + // Step past DefaultReportGiveUpAfter so retryReport would drop the + // report if shutdown fell through to it. + stepTo(tr, tr.clock.Now().Add(DefaultReportGiveUpAfter)) + cancel() + if err := tr.Wait(t.Context()); err != nil { + t.Fatalf("Wait: %v", err) + } + if !reportDueOnDisk() { + t.Error("state file after shutdown records no report due, want the interrupted report still due") + } + if got := levels(); !slices.Equal(got, []slog.Level{slog.LevelInfo}) { + t.Errorf("shutdown-interrupted report logged at %v, want once at Info", got) + } + }) + } +} + +// A report waits while a Run, Restore, Checkpoint, or PromoteSnapshot holds +// the actor, since the control plane holds the actor's lease throughout that +// call and would refuse the report with Aborted. Once the hold ends without a +// commit canceling the report, the report is queued. +func TestTrackerDefersReportDuringHold(t *testing.T) { + holds := []struct { + name string + begin func(t *testing.T, tr *Tracker) func() + }{ + { + name: "BeginStart", + begin: func(t *testing.T, tr *Tracker) func() { + end, err := tr.BeginStart(t.Context(), trackedActorUID) + if err != nil { + t.Fatalf("BeginStart: %v", err) + } + return end + }, + }, + { + name: "BeginCheckpoint", + begin: func(t *testing.T, tr *Tracker) func() { + end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, committedRef("s2", 2, tr.clock.Now()).Snapshot) + if err != nil { + t.Fatalf("BeginCheckpoint: %v", err) + } + return end + }, + }, + { + name: "BeginPromote", + begin: func(t *testing.T, tr *Tracker) func() { + // Promote a snapshot other than s1 so BeginPromote does not + // wait for the background upload of s1. + _, end, err := tr.BeginPromote(t.Context(), trackedActorUID, committedRef("s2", 2, tr.clock.Now())) + if err != nil { + t.Fatalf("BeginPromote: %v", err) + } + return end + }, + }, + } + + for _, h := range holds { + t.Run(h.name+"/upload finishes during hold", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + reporter := newFakeReporter() + tr := newReportingTracker(t, store, reporter) + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + done := startUpload(t, tr, store) + + end := h.begin(t, tr) + store.release <- nil + waitClosed(t, done, "the upload") + if n := tr.reporting.queue.Len(); n != 0 { + t.Fatalf("reports queued during hold = %d, want 0", n) + } + if !reportDue(tr) || !reportDueOnDisk() { + t.Fatal("report not marked due after the upload finished") + } + + end() + processNextReport(t, tr) + if got := len(reporter.reports()); got != 1 { + t.Errorf("reports after hold ended = %d, want 1", got) + } + if reportDue(tr) { + t.Error("report still due once acknowledged") + } + }) + + t.Run(h.name+"/queued report popped during hold", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + reporter := newFakeReporter() + tr := newReportingTracker(t, store, reporter) + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + uploadNow(t, tr, store, nil) + + end := h.begin(t, tr) + processNextReport(t, tr) + if got := len(reporter.reports()); got != 0 { + t.Fatalf("reports sent during hold = %d, want 0", got) + } + if n := tr.reporting.queue.Len(); n != 0 { + t.Fatalf("reports queued while hold still open = %d, want 0", n) + } + if !reportDue(tr) { + t.Fatal("report no longer due while deferred by hold") + } + + end() + processNextReport(t, tr) + if got := len(reporter.reports()); got != 1 { + t.Errorf("reports after hold ended = %d, want 1", got) + } + if reportDue(tr) { + t.Error("report still due once acknowledged") + } + }) + } + + t.Run("commit during hold cancels deferred report", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + reporter := newFakeReporter() + tr := newReportingTracker(t, store, reporter) + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + uploadNow(t, tr, store, nil) + + s2 := committedRef("s2", 2, tr.clock.Now()) + end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, s2.Snapshot) + if err != nil { + t.Fatalf("BeginCheckpoint: %v", err) + } + processNextReport(t, tr) + tr.CommitLocal(trackedActorUID, s2) + end() + + if n := tr.reporting.queue.Len(); n != 0 { + t.Errorf("reports queued after commit during hold = %d, want 0", n) + } + if reportDue(tr) { + t.Error("report still due after commit during hold") + } + }) + + t.Run("hold during retry backoff keeps retry delay", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + reporter := newFakeReporter() + reporter.failNext(status.Error(codes.Unavailable, "ateapi down")) + tr := newReportingTracker(t, store, reporter) + tr.CommitLocal(trackedActorUID, committedRef("s1", 1, tr.clock.Now())) + uploadNow(t, tr, store, nil) + failedAt := tr.clock.Now() + processNextReport(t, tr) + + end, err := tr.BeginStart(t.Context(), trackedActorUID) + if err != nil { + t.Fatalf("BeginStart: %v", err) + } + end() + + stepTo(tr, failedAt.Add(testReportRetryDelay-time.Second)) + time.Sleep(50 * time.Millisecond) + if n := tr.reporting.queue.Len(); n != 0 { + t.Fatalf("reports queued before retry delay passed = %d, want 0", n) + } + stepTo(tr, failedAt.Add(testReportRetryDelay)) + processNextReport(t, tr) + if got := len(reporter.reports()); got != 2 { + t.Errorf("reports after retry delay = %d, want 2", got) + } + }) +} diff --git a/cmd/atelet/internal/snapshottracker/state.go b/cmd/atelet/internal/snapshottracker/state.go index 1184f6ec3b..95fa5d4e96 100644 --- a/cmd/atelet/internal/snapshottracker/state.go +++ b/cmd/atelet/internal/snapshottracker/state.go @@ -32,19 +32,29 @@ import ( // written. Best-effort: if writing fails, the entry stays in memory and atelet // carries on from it, and a restart loses what changed. func (t *Tracker) saveLocked(a *actorSnapshots) { + if err := t.writeStateLocked(a); err != nil { + slog.Warn("Failed to save the actor's snapshot state", slog.String("actorUID", a.actorUID), slog.Any("err", err)) + } +} + +// writeStateLocked writes the actor's snapshot state file as saveLocked does, +// and returns the error if writing fails. +func (t *Tracker) writeStateLocked(a *actorSnapshots) error { if _, err := os.Stat(ateletpath.ActorPath(a.actorUID)); errors.Is(err, os.ErrNotExist) { slog.Warn("Dropping the snapshot state of an actor whose directory is gone", slog.String("actorUID", a.actorUID)) a.unsaved = false - return + t.undurableActorsTally.forget(a.actorUID) + return nil } + // The tally follows the entry, which atelet carries on from even if the + // write fails. + t.undurableActorsTally.observe(a) data, err := json.Marshal(a.actorSnapshotsRecord) if err == nil { err = atomicfile.Write(ateletpath.ActorSnapshotStateFile(a.actorUID), data, 0o600) } a.unsaved = err != nil - if err != nil { - slog.Warn("Failed to save the actor's snapshot state", slog.String("actorUID", a.actorUID), slog.Any("err", err)) - } + return err } // readSnapshotState reads what atelet tracked of the actor from its snapshot diff --git a/cmd/atelet/internal/snapshottracker/tracker.go b/cmd/atelet/internal/snapshottracker/tracker.go index a973f22eb0..61650c3dbd 100644 --- a/cmd/atelet/internal/snapshottracker/tracker.go +++ b/cmd/atelet/internal/snapshottracker/tracker.go @@ -12,43 +12,134 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package snapshottracker tracks each actor's snapshots on an atelet node, and -// runs background work on them. +// Package snapshottracker tracks each actor's snapshots on an atelet node, +// uploads and detaches them in the background, and reports what those change +// to the control plane. package snapshottracker import ( "context" "sync" + "time" "go.ate.dev/substrate/internal/actorlock" "go.ate.dev/substrate/internal/proto/ateletpb" + "google.golang.org/protobuf/types/known/timestamppb" "k8s.io/client-go/util/workqueue" "k8s.io/utils/clock" ) +// Store moves an actor's snapshots between this node and object storage. +type Store interface { + // UploadLocalSnapshot makes ref durable at its snapshot's URI from its + // local copy, acting for ref.Actor. With the local copy gone, it succeeds + // without uploading if a manifest is already committed there. It reports + // whether it uploaded. + UploadLocalSnapshot(ctx context.Context, actorUID string, ref SnapshotRef) (bool, error) +} + // Tracker tracks each actor's snapshots on this node: the latest one stored -// locally, and the latest one confirmed in object storage. State reports them. +// locally, and the latest one confirmed in object storage. State reports them, +// BeginCheckpoint and BeginPromote reject a stale generation before Checkpoint +// or PromoteSnapshot does any work, and commits never replace a snapshot with +// an older generation. It uploads and detaches them in the background. // -// Background work runs off a delaying workqueue of actor UIDs drained by a -// fixed pool of workers. The queue hands an actor to one worker at a time, and -// the number of workers bounds how much background work runs at once. +// Background work runs off two delaying workqueues of actor UIDs, one for +// uploads and one for detaches, each drained by a fixed pool of workers of its +// own, so slow uploads cannot hold up a due detach, nor detaches an upload. +// Each change to an actor's state queues the actor on each queue for when its +// next upload or detach is due: +// 1. A checkpoint records the new snapshot as LatestLocal with CommitLocal, +// and schedules the actor's detach detachDelay after the checkpoint. An +// immediate checkpoint records its upload with CommitDurable first. A +// promote of LatestLocal (CommitPromoted) leaves the schedule as it is. +// If LatestLocal needs an upload (see uploadDue), its upload is due +// uploadDelay after UndurableSince. +// 2. A worker that takes the actor from its queue runs the actor's upload, +// or detach, if it is due then. When it is not due yet, the worker queues +// the actor again for when it is: a queue holds an actor until the +// earliest time it was queued for, so a due time that moves later only +// costs a worker that check. When both are due, whichever worker begins +// first runs; an upload leaves the detach only its prune. +// 3. The worker uploads the LatestLocal of when it runs, so snapshots +// superseded while the actor waits for its upload are never uploaded. +// An upload that has begun (Uploading) goes on across newer checkpoints +// and atelet restarts, though, until it succeeds, a newer snapshot is +// durable, or it fails while a newer local snapshot can replace it. +// 4. Each queue hands an actor to one of its workers at a time, and its +// number of workers bounds how many background uploads, or detaches, run +// at once. An actor can be on both queues at once, but has at most one +// operation in progress (see actorOp): a worker that finds one leaves the +// actor to that operation's end, which queues it again. +// 5. An upload leaves LatestLocal on disk so a same-node Restore still +// reads it. A detach, in the background or via the Detach RPC, makes +// LatestLocal durable and then deletes all the actor's local snapshots. +// 6. A Run or Restore unschedules the actor's detach as it begins +// (BeginStart), and saves that before the workload can start, so atelet +// does not detach an actor whose start has begun, even after a restart or +// a failed start; only the actor's next checkpoint schedules a detach +// again. A background detach that comes due while a Run, Restore, +// Checkpoint, or PromoteSnapshot is in progress (BeginStart, +// BeginCheckpoint, or BeginPromote) waits for it to end, which queues the +// actor again. +// 7. A background upload that records a new LatestDurable, or a background +// detach that records one or clears LatestLocal, makes a report of the +// actor's state due (ReportDueSince), saved with the change so that it +// survives a restart. A Checkpoint, a PromoteSnapshot while a local +// snapshot is kept, or a Detach call returns the latest state to the +// control plane, so it cancels any pending report. Report workers, apart +// from the upload and detach workers, spread saved reports across a +// startup window, send the actor's whole state through the Reporter, and +// retry with per-actor and +// queue-wide backoff until the control plane acknowledges or refuses it, +// a call cancels it, or reporting.giveUpAfter passes. A report waits while +// a Run, Restore, Checkpoint, or PromoteSnapshot is in progress, and a +// background change made while a report ran is reported again. // -// TODO: Commits record the snapshot they are given without comparing -// Snapshot.generation. Checkpoint and PromoteSnapshot should reject a stale -// generation before doing any work, as Snapshot.generation describes, so that -// no commit replaces a snapshot with an older one. +// The Detach RPC runs in the calling goroutine rather than on a worker, so an +// explicit control-plane call does not wait behind other actors' background +// work. It does wait for the actor's own operation in progress, even an +// upload of an older snapshot; Run, Restore, and Checkpoint wait for a detach +// in progress; and PromoteSnapshot waits for a detach or a background upload +// of the same snapshot. A worker leaves an actor with an operation in +// progress to that operation's end, which queues the actor again. type Tracker struct { - // ctx bounds background work, and shuts the queue down when it ends. + // ctx bounds background work, and shuts the queues down when it ends. ctx context.Context - // clock is what due times are measured on, and the queue waits on. + // store uploads actors' snapshots. A tracker without one schedules no + // background work. + store Store + // clock is what due times are measured on, and the queues wait on. clock clock.WithTicker + // uploadDelay is how long after UndurableSince the actor's latest local + // snapshot becomes due for background upload: DefaultUploadDelay unless + // WithUploadDelay sets it. + uploadDelay time.Duration + // detachDelay is how long a suspended actor keeps its local snapshot on + // this node before atelet detaches it: DefaultDetachDelay unless + // WithDetachDelay sets it. + detachDelay time.Duration + // instruments records the background uploads and detaches (how late they + // start, how long they take, and how they end) and reports + // undurableActorsTally. Nil without WithInstruments. + instruments *Instruments + // undurableActorsTally tallies the actors that hold an undurable local + // snapshot on this node, for instruments to report. Nil without + // instruments. + undurableActorsTally *undurableActorsTally // retries says how long an actor's background work is held back after a // failure: longer with each consecutive failure for the actor. retries workqueue.TypedRateLimiter[string] - // workers is how many background workers run at once. - workers int - // queue holds the UIDs of actors with background work scheduled. - queue workqueue.TypedDelayingInterface[string] + // workers is how many workers drain each kind's queue: DefaultUploadWorkers + // and DefaultDetachWorkers unless WithUploadWorkers and WithDetachWorkers + // set them. + workers [numOpKinds]int + // queues hold the UIDs of actors with a background operation of each kind + // scheduled. + queues [numOpKinds]workqueue.TypedDelayingInterface[string] + // reporting holds the queue and settings for reporting snapshot state + // changes to the control plane. + reporting reportQueue // workersDone counts the workers still running. workersDone sync.WaitGroup @@ -59,27 +150,115 @@ type Tracker struct { // mu guards the actors map, and is never held during file I/O. mu sync.Mutex - // actors maps actorUID to the entry of an actor with a background - // operation in progress, or whose state its snapshot state file lacks - // because the save failed. requeueLocked drops any other entry, and - // findLocked then reads the state from the file. Read and change it - // through findLocked, findOrAddLocked, addLocked, and dropLocked. + // actors maps actorUID to the entries requeueLocked keeps in memory (see + // requeueLocked). requeueLocked drops any other entry, and findLocked then + // reads the state from the file. Read and change it through findLocked, + // findOrAddLocked, addLocked, dropLocked, tracksLocked, and heldLocked. actors map[string]*actorSnapshots } -// New returns a tracker that tracks no actor yet and measures time on clk. -// Its workers run once Start is called, until ctx ends. -func New(ctx context.Context, clk clock.WithTicker) *Tracker { +// Option configures a Tracker. +type Option func(*Tracker) + +// WithUploadDelay sets how long after the oldest snapshot of an actor not yet +// durable was taken atelet uploads the actor's latest local snapshot. +func WithUploadDelay(d time.Duration) Option { + return func(t *Tracker) { + t.uploadDelay = d + } +} + +// WithDetachDelay sets how long an actor stays suspended on this node, its +// latest snapshot kept locally, before atelet detaches it. +func WithDetachDelay(d time.Duration) Option { + return func(t *Tracker) { + t.detachDelay = d + } +} + +// WithUploadWorkers sets how many background uploads run at once on this node. +func WithUploadWorkers(n int) Option { + return func(t *Tracker) { + t.workers[opUpload] = n + } +} + +// WithDetachWorkers sets how many background detaches run at once on this +// node. +func WithDetachWorkers(n int) Option { + return func(t *Tracker) { + t.workers[opDetach] = n + } +} + +// WithInstruments attaches the instruments that record the tracker's +// background uploads and detaches, and report the undurable local snapshots on +// this node. A nil *Instruments is a no-op. +func WithInstruments(inst *Instruments) Option { + return func(t *Tracker) { + t.instruments = inst + } +} + +// WithReporter sets what reports an actor's snapshot state to the control +// plane after a background upload or detach. +func WithReporter(r Reporter) Option { + return func(t *Tracker) { + t.reporting.reporter = r + } +} + +// WithReportWorkers sets how many background reports run at once on this node. +// Zero disables background reporting. +func WithReportWorkers(n int) Option { + return func(t *Tracker) { + t.reporting.workers = n + } +} + +// WithReportGiveUpAfter sets how long atelet keeps retrying a failed report +// since it became due before giving up on it. +func WithReportGiveUpAfter(d time.Duration) Option { + return func(t *Tracker) { + t.reporting.giveUpAfter = d + } +} + +// New returns a tracker over store that tracks no actor yet and measures time +// on clk. Its workers run once Start is called, until ctx ends. A nil store +// schedules no uploads or detaches. +func New(ctx context.Context, store Store, clk clock.WithTicker, opts ...Option) *Tracker { t := &Tracker{ - ctx: ctx, - clock: clk, - retries: workqueue.NewTypedItemExponentialFailureRateLimiter[string](snapshotRetryDelay, maxSnapshotRetryDelay), - workers: snapshotWorkers, - queue: workqueue.NewTypedDelayingQueueWithConfig(workqueue.TypedDelayingQueueConfig[string]{Clock: clk}), - locks: actorlock.New(), - actors: map[string]*actorSnapshots{}, - } - context.AfterFunc(ctx, t.queue.ShutDown) + ctx: ctx, + store: store, + clock: clk, + uploadDelay: DefaultUploadDelay, + detachDelay: DefaultDetachDelay, + retries: workqueue.NewTypedItemExponentialFailureRateLimiter[string](snapshotRetryDelay, maxSnapshotRetryDelay), + workers: [numOpKinds]int{opUpload: DefaultUploadWorkers, opDetach: DefaultDetachWorkers}, + reporting: newReportQueue(clk), + locks: actorlock.New(), + actors: map[string]*actorSnapshots{}, + } + for k := range t.queues { + t.queues[k] = workqueue.NewTypedDelayingQueueWithConfig(workqueue.TypedDelayingQueueConfig[string]{Clock: clk}) + } + for _, opt := range opts { + opt(t) + } + if t.reporting.workers <= 0 { + t.reporting.reporter = nil + } + if t.instruments != nil { + t.undurableActorsTally = newUndurableActorsTally(clk, t.uploadDelay, t.store != nil) + t.instruments.watch(t.undurableActorsTally) + } + context.AfterFunc(ctx, func() { + for _, q := range t.queues { + q.ShutDown() + } + t.reporting.queue.ShutDown() + }) return t } @@ -131,12 +310,58 @@ func (t *Tracker) dropLocked(actorUID string) { delete(t.actors, actorUID) } +// tracksLocked reports whether a is still its actor's entry in memory: false +// once Forget has dropped the actor, even if a new entry has been added since. +func (t *Tracker) tracksLocked(a *actorSnapshots) bool { + t.mu.Lock() + defer t.mu.Unlock() + return t.actors[a.actorUID] == a +} + +// heldLocked reports whether a Run, Restore, Checkpoint, or PromoteSnapshot of +// the actor is in progress. An entry with one stays in memory, so this never +// reads the snapshot state file. +func (t *Tracker) heldLocked(actorUID string) bool { + t.mu.Lock() + defer t.mu.Unlock() + a := t.actors[actorUID] + return a != nil && a.holds > 0 +} + +// uploadDueLocked returns when the actor's background upload is due, as +// actorSnapshots.uploadDue does. A tracker without a store uploads nothing. +func (t *Tracker) uploadDueLocked(a *actorSnapshots) (time.Time, bool) { + if t.store == nil { + return time.Time{}, false + } + return a.uploadDue(t.uploadDelay) +} + +// detachDueLocked returns when atelet next checks whether to detach the actor, +// as actorSnapshots.detachDue does. A tracker without a store detaches +// nothing. +func (t *Tracker) detachDueLocked(a *actorSnapshots) (time.Time, bool) { + if t.store == nil { + return time.Time{}, false + } + return a.detachDue() +} + // stateLocked returns what atelet holds for the actor. func (t *Tracker) stateLocked(a *actorSnapshots) *ateletpb.ActorSnapshotState { - return &ateletpb.ActorSnapshotState{ + var uploading *ateletpb.SnapshotReport + if a.keepUploading() { + uploading = a.Uploading.report() + } + st := &ateletpb.ActorSnapshotState{ LatestLocal: a.LatestLocal.report(), + Uploading: uploading, LatestDurable: a.LatestDurable.report(), } + if due, ok := t.uploadDueLocked(a); ok { + st.UploadDueTime = timestamppb.New(due) + } + return st } // State returns what atelet holds for the actor. diff --git a/cmd/atelet/internal/snapshottracker/tracker_test.go b/cmd/atelet/internal/snapshottracker/tracker_test.go index 637617b5ec..9443dcbaa0 100644 --- a/cmd/atelet/internal/snapshottracker/tracker_test.go +++ b/cmd/atelet/internal/snapshottracker/tracker_test.go @@ -15,18 +15,26 @@ package snapshottracker import ( + "context" + "encoding/json" + "errors" "fmt" "os" "path/filepath" + "slices" "sync" "testing" "time" "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/nodepath" "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" + "google.golang.org/grpc/codes" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" + "k8s.io/client-go/util/workqueue" clocktesting "k8s.io/utils/clock/testing" ) @@ -48,7 +56,7 @@ func useTempNodeDirs(t *testing.T) { // newTracker returns a tracker on a fake clock whose context ends with the // test. Its workers are not started. func newTracker(t *testing.T) *Tracker { - return New(t.Context(), clocktesting.NewFakeClock(time.Unix(1_700_000_000, 0))) + return New(t.Context(), nil, clocktesting.NewFakeClock(time.Unix(1_700_000_000, 0))) } // testRef is a MEMORY snapshot of the tracked actor. @@ -64,6 +72,25 @@ func testRef(uuid string, generation int32, commitTime time.Time) SnapshotRef { } } +// testActor is the actor testRef's snapshots belong to. +var testActor = resources.ActorRef{Atespace: "ate-demo", Name: "counter"} + +func TestSnapshotRefJSONKeepsTheActor(t *testing.T) { + ref := testRef("s1", 1, time.Unix(1_700_000_000, 0).UTC()) + ref.Actor = testActor + data, err := json.Marshal(ref) + if err != nil { + t.Fatal(err) + } + var got SnapshotRef + if err := json.Unmarshal(data, &got); err != nil { + t.Fatal(err) + } + if got.Actor != testActor || !proto.Equal(got.Snapshot, ref.Snapshot) || !got.CommitTime.Equal(ref.CommitTime) { + t.Errorf("round trip = %+v, want %+v", got, ref) + } +} + func timeOf(ts *timestamppb.Timestamp) time.Time { if ts == nil { return time.Time{} @@ -75,10 +102,215 @@ func uuidOf(r *ateletpb.SnapshotReport) string { return r.GetSnapshot().GetSnapshotUuid() } -// State reports the latest local snapshot a deferred checkpoint leaves, and -// the latest durable one an immediate checkpoint leaves without a local copy. -// Each update is written to the actor's snapshot state file and evicted from -// memory; Forget deletes the file and drops any in-memory state. +// The delays of the trackers newStoreTracker builds. Nothing is due until a +// test advances the tracker's clock. +const ( + testUploadDelay = time.Hour + testDetachDelay = 4 * time.Hour + testRetryDelay = 30 * time.Minute + testMaxRetryDelay = 2 * time.Hour +) + +// fakeSnapshotStore is a Store that records its calls. A call for a URI it +// already holds uploads nothing. +type fakeSnapshotStore struct { + // started receives each call's snapshot uuid as the call starts. + started chan string + // release ends a blocked call with the error it receives. + release chan error + + mu sync.Mutex + calls []string + actors map[string]resources.ActorRef + uploaded map[string]bool + blocking bool + err error +} + +func newFakeSnapshotStore() *fakeSnapshotStore { + return &fakeSnapshotStore{started: make(chan string, 100), release: make(chan error), actors: map[string]resources.ActorRef{}} +} + +// setBlocking makes later calls wait for release, or their context. +func (f *fakeSnapshotStore) setBlocking(blocking bool) { + f.mu.Lock() + defer f.mu.Unlock() + f.blocking = blocking +} + +// setErr makes later calls fail with err, or succeed when it is nil. +func (f *fakeSnapshotStore) setErr(err error) { + f.mu.Lock() + defer f.mu.Unlock() + f.err = err +} + +func (f *fakeSnapshotStore) uploads() []string { + f.mu.Lock() + defer f.mu.Unlock() + return slices.Clone(f.calls) +} + +// actorOf returns the actor the upload of the snapshot acted for. +func (f *fakeSnapshotStore) actorOf(uuid string) resources.ActorRef { + f.mu.Lock() + defer f.mu.Unlock() + return f.actors[uuid] +} + +func (f *fakeSnapshotStore) UploadLocalSnapshot(ctx context.Context, _ string, ref SnapshotRef) (bool, error) { + f.mu.Lock() + f.calls = append(f.calls, ref.uuid()) + f.actors[ref.uuid()] = ref.Actor + blocking, err := f.blocking, f.err + f.mu.Unlock() + f.started <- ref.uuid() + if blocking { + select { + case err = <-f.release: + case <-ctx.Done(): + return false, ctx.Err() + } + } + if err != nil { + return false, err + } + f.mu.Lock() + defer f.mu.Unlock() + if f.uploaded[ref.uri()] { + return false, nil + } + if f.uploaded == nil { + f.uploaded = map[string]bool{} + } + f.uploaded[ref.uri()] = true + return true, nil +} + +// newStoreTracker returns a tracker over store on a fake clock, with the test +// delays and opts, whose workers have not started: background work starts only +// once a test advances the clock to when it is due (see stepTo). Its workers +// stop once the test ends, before the node directories are restored. +func newStoreTracker(t *testing.T, store Store, opts ...Option) *Tracker { + t.Helper() + opts = append([]Option{WithUploadDelay(testUploadDelay), WithDetachDelay(testDetachDelay)}, opts...) + tr := New(t.Context(), store, clocktesting.NewFakeClock(time.Unix(1_700_000_000, 0)), opts...) + tr.retries = workqueue.NewTypedItemExponentialFailureRateLimiter[string](testRetryDelay, testMaxRetryDelay) + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if err := tr.Wait(ctx); err != nil { + t.Errorf("workers still running a minute after the test ended: %v", err) + } + }) + return tr +} + +// newIdleTestTracker returns a tracker like newStoreTracker's, on fresh node +// directories that hold the tracked actor's directory. +func newIdleTestTracker(t *testing.T, store Store) *Tracker { + t.Helper() + useActorDir(t) + return newStoreTracker(t, store) +} + +// newTestTracker returns a tracker like newIdleTestTracker's, with its workers +// running. +func newTestTracker(t *testing.T, store Store) *Tracker { + t.Helper() + tr := newIdleTestTracker(t, store) + tr.Start() + return tr +} + +// stepTo advances the tracker's fake clock to at, unless it is there already, +// so the background work due by then starts. +func stepTo(tr *Tracker, at time.Time) { + clk := tr.clock.(*clocktesting.FakeClock) + if d := at.Sub(clk.Now()); d > 0 { + clk.Step(d) + } +} + +// waitStarted waits for the next store call to start, and returns the uuid of +// the snapshot it uploads. +func waitStarted(t *testing.T, store *fakeSnapshotStore) string { + t.Helper() + select { + case uuid := <-store.started: + return uuid + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for an upload to start") + return "" + } +} + +// opDone returns a channel closed once the actor's operation in progress has +// ended, deleting the local snapshots it no longer needs included. +func opDone(t *testing.T, tr *Tracker, actorUID string) <-chan struct{} { + t.Helper() + tr.lock(actorUID) + defer tr.unlock(actorUID) + a := tr.findLocked(actorUID) + if a == nil || a.op == nil { + t.Fatal("no operation in progress") + } + return a.op.done +} + +func waitClosed(t *testing.T, done <-chan struct{}, what string) { + t.Helper() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatalf("timed out waiting for %s", what) + } +} + +// processNext takes the next actor from kind's queue of a tracker whose +// workers have not started, and runs its background work as a worker of that +// queue would. +func processNext(t *testing.T, tr *Tracker, kind opKind) { + t.Helper() + q := tr.queues[kind] + next := make(chan string, 1) + go func() { + if actorUID, shutdown := q.Get(); !shutdown { + next <- actorUID + } + }() + select { + case actorUID := <-next: + tr.syncActor(actorUID, kind) + q.Done(actorUID) + case <-time.After(10 * time.Second): + t.Fatalf("timed out waiting for an actor to be queued for %s", kind) + } +} + +// queuedNow returns how many actors tr's queues hold ready for a worker, +// leaving out those queued for later. +func queuedNow(tr *Tracker) int { + n := 0 + for _, q := range tr.queues { + n += q.Len() + } + return n +} + +// setWorkers gives tr uploads upload workers and detaches detach workers in +// place of the defaults, as WithUploadWorkers and WithDetachWorkers do. Call +// it before Start. +func setWorkers(tr *Tracker, uploads, detaches int) { + WithUploadWorkers(uploads)(tr) + WithDetachWorkers(detaches)(tr) +} + +// State reports the latest local snapshot CommitLocal records and the latest +// durable one CommitDurable records, as an immediate checkpoint records its +// upload before its local copy. Each update is written to the actor's snapshot +// state file and evicted from memory; Forget deletes the file and drops any +// in-memory state. func TestTrackerState(t *testing.T) { useTempNodeDirs(t) if err := os.MkdirAll(ateletpath.ActorPath(trackedActorUID), 0o700); err != nil { @@ -98,17 +330,22 @@ func TestTrackerState(t *testing.T) { } s2 := testRef("s2", 2, t0.Add(time.Minute)) - s2.UploadTime = s2.CommitTime.Add(time.Second) - st = tr.CommitDurable(trackedActorUID, s2) + durable := s2 + durable.UploadTime = s2.CommitTime.Add(time.Second) + st = tr.CommitDurable(trackedActorUID, durable) if st.GetLatestLocal() != nil { - t.Errorf("latest_local = %v after an immediate checkpoint, want unset", st.GetLatestLocal()) + t.Errorf("latest_local = %v after CommitDurable, want unset: s2 supersedes s1", st.GetLatestLocal()) } - if got := st.GetLatestDurable(); uuidOf(got) != "s2" || !timeOf(got.GetDurableCompleteTime()).Equal(s2.UploadTime) { - t.Errorf("latest_durable = %v, want s2 uploaded at %v", got, s2.UploadTime) + if got := st.GetLatestDurable(); uuidOf(got) != "s2" || !timeOf(got.GetDurableCompleteTime()).Equal(durable.UploadTime) { + t.Errorf("latest_durable = %v, want s2 uploaded at %v", got, durable.UploadTime) } if len(tr.actors) != 0 { t.Errorf("in-memory actors after CommitDurable = %d, want 0 after writing to file", len(tr.actors)) } + st = tr.CommitLocal(trackedActorUID, s2) + if got := st.GetLatestLocal(); uuidOf(got) != "s2" || !timeOf(got.GetLocalCommitTime()).Equal(s2.CommitTime) || got.GetDurableCompleteTime() != nil { + t.Errorf("latest_local = %v, want s2 committed at %v, with no upload time", got, s2.CommitTime) + } st = tr.CommitLocal(trackedActorUID, testRef("s3", 3, t0.Add(2*time.Minute))) if uuidOf(st.GetLatestLocal()) != "s3" || uuidOf(st.GetLatestDurable()) != "s2" { @@ -132,34 +369,39 @@ func TestTrackerState(t *testing.T) { } } -// ForgetLocal drops the local snapshot and keeps the durable one. With no -// local snapshot to drop, it writes nothing. -func TestTrackerForgetLocal(t *testing.T) { - useTempNodeDirs(t) - if err := os.MkdirAll(ateletpath.ActorPath(trackedActorUID), 0o700); err != nil { - t.Fatal(err) - } +// LocalSnapshotsToKeep names the latest local snapshot. CommitDurable drops an +// older local snapshot, which the durable one supersedes, but keeps a local +// copy of the durable snapshot itself. +func TestTrackerCommitDurableDropsOlderLocal(t *testing.T) { + useActorDir(t) tr := newTracker(t) - tr.ForgetLocal(trackedActorUID) - if _, err := os.Stat(ateletpath.ActorSnapshotStateFile(trackedActorUID)); !os.IsNotExist(err) { - t.Errorf("state file after forgetting nothing: stat = %v, want none written", err) + if keep := tr.LocalSnapshotsToKeep(trackedActorUID); keep != nil { + t.Errorf("LocalSnapshotsToKeep for an untracked actor = %v, want nil", keep) } t0 := time.Now() - tr.CommitDurable(trackedActorUID, testRef("s1", 1, t0)) - tr.CommitLocal(trackedActorUID, testRef("s2", 2, t0.Add(time.Minute))) - tr.ForgetLocal(trackedActorUID) - if st := newTracker(t).State(trackedActorUID); st.GetLatestLocal() != nil || uuidOf(st.GetLatestDurable()) != "s1" { - t.Errorf("state from the file = %v, want no local snapshot and s1 still durable", st) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) + if keep := tr.LocalSnapshotsToKeep(trackedActorUID); !slices.Equal(keep, []string{"s1"}) { + t.Errorf("LocalSnapshotsToKeep = %v, want [s1]", keep) } - if len(tr.actors) != 0 { - t.Errorf("in-memory actors = %d, want 0 after writing to file", len(tr.actors)) + s2 := testRef("s2", 2, t0.Add(time.Minute)) + tr.CommitDurable(trackedActorUID, s2) + if st := newTracker(t).State(trackedActorUID); st.GetLatestLocal() != nil || uuidOf(st.GetLatestDurable()) != "s2" { + t.Errorf("state from the file = %v, want no local snapshot and s2 durable", st) + } + if keep := tr.LocalSnapshotsToKeep(trackedActorUID); len(keep) != 0 { + t.Errorf("LocalSnapshotsToKeep after CommitDurable = %v, want none", keep) + } + + tr.CommitLocal(trackedActorUID, s2) + if st := tr.CommitDurable(trackedActorUID, s2); uuidOf(st.GetLatestLocal()) != "s2" { + t.Errorf("latest_local after CommitDurable of the same snapshot = %v, want s2 kept", st.GetLatestLocal()) } } // CommitPromoted makes the promoted local snapshot the latest durable one, -// keeping the commit time recorded for its local copy, and drops the local -// one. A retried promote of the same snapshot keeps what was recorded. +// keeping the local copy and the commit time recorded for it. A retried +// promote of the same snapshot keeps what was recorded. func TestTrackerCommitPromoted(t *testing.T) { useTempNodeDirs(t) if err := os.MkdirAll(ateletpath.ActorPath(trackedActorUID), 0o700); err != nil { @@ -172,8 +414,8 @@ func TestTrackerCommitPromoted(t *testing.T) { promoted := testRef("s1", 1, time.Time{}) promoted.UploadTime = t0.Add(time.Minute) st := tr.CommitPromoted(trackedActorUID, promoted) - if st.GetLatestLocal() != nil { - t.Errorf("latest_local = %v after the promote, want unset", st.GetLatestLocal()) + if got := st.GetLatestLocal(); uuidOf(got) != "s1" || !timeOf(got.GetLocalCommitTime()).Equal(t0) || got.GetDurableCompleteTime() != nil { + t.Errorf("latest_local = %v, want s1 committed at %v, with no upload time", got, t0) } if got := st.GetLatestDurable(); uuidOf(got) != "s1" || !timeOf(got.GetLocalCommitTime()).Equal(t0) || !timeOf(got.GetDurableCompleteTime()).Equal(promoted.UploadTime) { t.Errorf("latest_durable = %v, want s1 committed at %v and uploaded at %v", got, t0, promoted.UploadTime) @@ -188,13 +430,116 @@ func TestTrackerCommitPromoted(t *testing.T) { t.Errorf("state after a retried promote = %v, want %v", got, st) } - // Promoting a snapshot other than the local one carries no commit time. + // Promoting a snapshot other than the local one carries no commit time and + // leaves the local snapshot untouched. tr.CommitLocal(trackedActorUID, testRef("s2", 2, t0.Add(2*time.Hour))) other := testRef("s3", 3, time.Time{}) other.UploadTime = t0.Add(3 * time.Hour) st = tr.CommitPromoted(trackedActorUID, other) - if got := st.GetLatestDurable(); uuidOf(got) != "s3" || got.GetLocalCommitTime() != nil || st.GetLatestLocal() != nil { - t.Errorf("state = %v, want s3 durable without a commit time, and no local snapshot", st) + if got := st.GetLatestDurable(); uuidOf(got) != "s3" || got.GetLocalCommitTime() != nil || uuidOf(st.GetLatestLocal()) != "s2" { + t.Errorf("state = %v, want s3 durable without a commit time, and s2 still local", st) + } + + // An older promote finishing after a newer checkpoint keeps the newer + // durable and local snapshots. + tr.CommitLocal(trackedActorUID, testRef("s4", 4, t0.Add(4*time.Hour))) + older := testRef("s2", 2, time.Time{}) + older.UploadTime = t0.Add(5 * time.Hour) + st = tr.CommitPromoted(trackedActorUID, older) + if uuidOf(st.GetLatestDurable()) != "s3" || uuidOf(st.GetLatestLocal()) != "s4" { + t.Errorf("state after older promote = %v, want s3 durable and s4 local", st) + } +} + +// A promote of a paused actor's local snapshot leaves its detach schedule as +// it is: the detach its checkpoint scheduled, or none for an actor whose Run +// or Restore has begun since the snapshot was taken. +func TestTrackerCommitPromotedKeepsDetachSchedule(t *testing.T) { + for _, started := range []bool{false, true} { + t.Run(fmt.Sprintf("started=%v", started), func(t *testing.T) { + tr := newIdleTestTracker(t, newFakeSnapshotStore()) + paused := testRef("s1", 1, tr.clock.Now()) + tr.CommitLocal(trackedActorUID, paused) + want := tr.clock.Now().Add(testDetachDelay) + if started { + end, err := tr.BeginStart(t.Context(), trackedActorUID) + if err != nil { + t.Fatalf("beginStart: %v", err) + } + end() + want = time.Time{} + } + stepTo(tr, tr.clock.Now().Add(time.Hour)) + tr.CommitPromoted(trackedActorUID, testRef("s1", 1, time.Time{})) + if got := detachDueOf(tr, trackedActorUID); !got.Equal(want) { + t.Errorf("detachDue after the promote = %v, want %v", got, want) + } + }) + } +} + +// Promoting the latest local snapshot makes it durable and clears its upload +// due time, whereas promoting an older snapshot while newer local snapshots +// still need an upload keeps the upload due when the oldest of them was, +// including any retry backoff. +func TestTrackerCommitPromotedClearsUploadDue(t *testing.T) { + store := newFakeSnapshotStore() + tr := newIdleTestTracker(t, store) + t0 := tr.clock.Now() + st := tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) + if st.GetUploadDueTime() == nil { + t.Fatal("upload_due_time unset before promote, want set") + } + + promoted := testRef("s1", 1, time.Time{}) + promoted.UploadTime = t0.Add(time.Minute) + if st = tr.CommitPromoted(trackedActorUID, promoted); st.GetUploadDueTime() != nil { + t.Errorf("upload_due_time after promoting latest local = %v, want unset", st.GetUploadDueTime()) + } + + t2 := t0.Add(2 * time.Minute) + tr.CommitLocal(trackedActorUID, testRef("s2", 2, t2)) + tr.CommitLocal(trackedActorUID, testRef("s3", 3, t2.Add(time.Minute))) + tr.CommitLocal(trackedActorUID, testRef("s4", 4, t2.Add(2*time.Minute))) + + older := testRef("s0", 0, time.Time{}) + older.UploadTime = t2.Add(3 * time.Minute) + st = tr.CommitPromoted(trackedActorUID, older) + if got := uuidOf(st.GetLatestDurable()); got != "s1" { + t.Errorf("latest_durable after promoting below durable = %q, want s1", got) + } + if got, want := timeOf(st.GetUploadDueTime()), t2.Add(testUploadDelay); !got.Equal(want) { + t.Errorf("upload_due_time after promoting below durable = %v, want %v: an upload delay after s2", got, want) + } + + between := testRef("s2", 2, time.Time{}) + between.UploadTime = t2.Add(3 * time.Minute) + st = tr.CommitPromoted(trackedActorUID, between) + if got := uuidOf(st.GetLatestDurable()); got != "s2" { + t.Errorf("latest_durable after promoting s2 = %q, want s2", got) + } + if got, want := timeOf(st.GetUploadDueTime()), t2.Add(testUploadDelay); !got.Equal(want) { + t.Errorf("upload_due_time after promoting between durable and local = %v, want %v: an upload delay after s2", got, want) + } + + stepTo(tr, t2.Add(testUploadDelay)) + store.setErr(errors.New("boom")) + if _, err := tr.Detach(t.Context(), trackedActorUID); err == nil { + t.Fatal("detach succeeded, want the store's error") + } + wantRetry := tr.clock.Now().Add(testRetryDelay) + st = tr.CommitPromoted(trackedActorUID, older) + if got := timeOf(st.GetUploadDueTime()); !got.Equal(wantRetry) { + t.Errorf("upload_due_time after promoting below durable during backoff = %v, want %v", got, wantRetry) + } + betweenBackoff := testRef("s3", 3, time.Time{}) + betweenBackoff.UploadTime = tr.clock.Now() + st = tr.CommitPromoted(trackedActorUID, betweenBackoff) + if got := uuidOf(st.GetLatestDurable()); got != "s3" { + t.Errorf("latest_durable after promoting s3 during backoff = %q, want s3", got) + } + if got := timeOf(st.GetUploadDueTime()); !got.Equal(wantRetry) { + t.Errorf("upload_due_time after promoting between durable and local during backoff = %v, want %v", got, wantRetry) } } @@ -246,11 +591,8 @@ func TestTrackerDropsActorWithoutDirectory(t *testing.T) { } } -// Commits for one actor take turns, so a racing deferred checkpoint never -// loses an immediate checkpoint's durable snapshot. The control plane never -// sends one actor's checkpoints concurrently: the race only exercises the -// lock, so either commit may land last, even leaving a local snapshot older -// than the durable one. +// Commits for one actor take turns, and an older local commit never replaces +// a newer durable snapshot. func TestTrackerSerializesEachActor(t *testing.T) { useTempNodeDirs(t) if err := os.MkdirAll(ateletpath.ActorPath(trackedActorUID), 0o700); err != nil { @@ -265,9 +607,13 @@ func TestTrackerSerializesEachActor(t *testing.T) { wg.Go(func() { tr.CommitLocal(trackedActorUID, local) }) wg.Go(func() { tr.CommitDurable(trackedActorUID, durable) }) wg.Wait() - if got, want := uuidOf(tr.State(trackedActorUID).GetLatestDurable()), durable.Snapshot.GetSnapshotUuid(); got != want { + st := tr.State(trackedActorUID) + if got, want := uuidOf(st.GetLatestDurable()), durable.Snapshot.GetSnapshotUuid(); got != want { t.Fatalf("round %d: latest_durable = %q, want %q", i, got, want) } + if got := st.GetLatestLocal(); got != nil { + t.Fatalf("round %d: latest_local = %v, want nil when durable has a higher generation", i, got) + } } if len(tr.actors) != 0 { t.Errorf("in-memory actors = %d, want 0 once every commit is written", len(tr.actors)) @@ -307,3 +653,220 @@ func TestTrackerSkipsUnreadableSnapshotState(t *testing.T) { t.Errorf("state with a broken file = %v, want empty", got) } } + +// BeginCheckpoint and BeginPromote accept any generation when atelet holds no +// snapshot for the actor, and otherwise reject a generation below the actor's +// newest snapshot or equal to it under another snapshot_uuid with +// FailedPrecondition. +func TestTrackerCheckGeneration(t *testing.T) { + useTempNodeDirs(t) + if err := os.MkdirAll(ateletpath.ActorPath(trackedActorUID), 0o700); err != nil { + t.Fatal(err) + } + tr := newTracker(t) + t0 := time.Now() + ref := func(uuid string, generation int32) SnapshotRef { + return testRef(uuid, generation, t0) + } + snap := func(uuid string, generation int32) *ateletpb.Snapshot { + return ref(uuid, generation).Snapshot + } + checkBegin := func(t *testing.T, tr *Tracker, r SnapshotRef, wantCode codes.Code) { + t.Helper() + end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, r.Snapshot) + if err == nil { + end() + } + if got := apierror.Code(err); got != wantCode { + t.Errorf("BeginCheckpoint(%v) code = %v, want %v", r.Snapshot, got, wantCode) + } + _, end, err = tr.BeginPromote(t.Context(), trackedActorUID, r) + if err == nil { + end() + } + if got := apierror.Code(err); got != wantCode { + t.Errorf("BeginPromote(%v) code = %v, want %v", r.Snapshot, got, wantCode) + } + } + + checkBegin(t, tr, ref("s1", 1), codes.OK) + + // Only a local snapshot is tracked (generation 2). + tr.CommitLocal(trackedActorUID, ref("s1", 2)) + for _, tc := range []struct { + name string + ref SnapshotRef + wantCode codes.Code + }{ + {name: "same snapshot at newest generation", ref: ref("s1", 2), wantCode: codes.OK}, + {name: "higher generation", ref: ref("s2", 3), wantCode: codes.OK}, + {name: "generation below newest", ref: ref("s0", 1), wantCode: codes.FailedPrecondition}, + {name: "same uuid at lower generation", ref: ref("s1", 1), wantCode: codes.FailedPrecondition}, + {name: "different uuid at newest generation", ref: ref("other", 2), wantCode: codes.FailedPrecondition}, + } { + t.Run("localNewest/"+tc.name, func(t *testing.T) { + checkBegin(t, tr, tc.ref, tc.wantCode) + checkBegin(t, newTracker(t), tc.ref, tc.wantCode) + }) + } + + // Only a durable snapshot is tracked (generation 4); CommitDurable drops + // the older local snapshot. + tr.CommitDurable(trackedActorUID, ref("s2", 4)) + for _, tc := range []struct { + name string + ref SnapshotRef + wantCode codes.Code + }{ + {name: "same snapshot at newest generation", ref: ref("s2", 4), wantCode: codes.OK}, + {name: "higher generation", ref: ref("s3", 5), wantCode: codes.OK}, + {name: "generation below newest", ref: ref("s1", 3), wantCode: codes.FailedPrecondition}, + {name: "different uuid at newest generation", ref: ref("other", 4), wantCode: codes.FailedPrecondition}, + } { + t.Run("durableNewest/"+tc.name, func(t *testing.T) { + checkBegin(t, tr, tc.ref, tc.wantCode) + }) + } + + // Both local (generation 6) and durable (generation 4) are tracked: local is newest. + tr.CommitLocal(trackedActorUID, ref("s3", 6)) + checkBegin(t, tr, ref("s2", 4), codes.FailedPrecondition) + checkBegin(t, tr, ref("s3", 6), codes.OK) + + // While a Checkpoint or PromoteSnapshot holds an uncommitted snapshot at + // generation 8, it is newest until end() runs. + endCp, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, snap("s4", 8)) + if err != nil { + t.Fatalf("BeginCheckpoint(s4@8): %v", err) + } + checkBegin(t, tr, ref("s3", 6), codes.FailedPrecondition) + checkBegin(t, tr, ref("between", 7), codes.FailedPrecondition) + checkBegin(t, tr, ref("other", 8), codes.FailedPrecondition) + checkBegin(t, tr, ref("s4", 8), codes.OK) + checkBegin(t, tr, ref("s5", 9), codes.OK) + endCp() + // Ending without committing drops the in-flight generation, so s3@6 is newest again. + checkBegin(t, tr, ref("s3", 6), codes.OK) + checkBegin(t, tr, ref("between", 7), codes.OK) + + _, endPromote, err := tr.BeginPromote(t.Context(), trackedActorUID, ref("s4", 8)) + if err != nil { + t.Fatalf("BeginPromote(s4@8): %v", err) + } + checkBegin(t, tr, ref("between", 7), codes.FailedPrecondition) + checkBegin(t, tr, ref("other", 8), codes.FailedPrecondition) + checkBegin(t, tr, ref("s4", 8), codes.OK) + endPromote() + checkBegin(t, tr, ref("between", 7), codes.OK) + + // If durable has a higher generation than local, durable is newest. + a := &actorSnapshots{ + actorUID: trackedActorUID, + actorSnapshotsRecord: actorSnapshotsRecord{ + LatestLocal: ref("local", 3).clone(), + LatestDurable: ref("durable", 5).clone(), + }, + } + if got := apierror.Code(a.checkGeneration(snap("local", 3))); got != codes.FailedPrecondition { + t.Errorf("checkGeneration(older local@3) code = %v, want FailedPrecondition", got) + } + if got := apierror.Code(a.checkGeneration(snap("other", 5))); got != codes.FailedPrecondition { + t.Errorf("checkGeneration(other@5) code = %v, want FailedPrecondition", got) + } + if got := apierror.Code(a.checkGeneration(snap("durable", 5))); got != codes.OK { + t.Errorf("checkGeneration(durable@5) code = %v, want OK", got) + } + + // If Uploading holds a newer snapshot, it is newest even while no op runs. + a.Uploading = ref("saved-uploading", 6).clone() + if got := apierror.Code(a.checkGeneration(snap("durable", 5))); got != codes.FailedPrecondition { + t.Errorf("checkGeneration(older durable@5) code = %v, want FailedPrecondition", got) + } + if got := apierror.Code(a.checkGeneration(snap("other", 6))); got != codes.FailedPrecondition { + t.Errorf("checkGeneration(other@6) code = %v, want FailedPrecondition", got) + } + if got := apierror.Code(a.checkGeneration(snap("saved-uploading", 6))); got != codes.OK { + t.Errorf("checkGeneration(saved-uploading@6) code = %v, want OK", got) + } + + // If an operation in progress acts on a newer snapshot, its target is newest. + a.op = &actorOp{target: ref("uploading", 7)} + if got := apierror.Code(a.checkGeneration(snap("saved-uploading", 6))); got != codes.FailedPrecondition { + t.Errorf("checkGeneration(older saved-uploading@6) code = %v, want FailedPrecondition", got) + } + if got := apierror.Code(a.checkGeneration(snap("other", 7))); got != codes.FailedPrecondition { + t.Errorf("checkGeneration(other@7) code = %v, want FailedPrecondition", got) + } + if got := apierror.Code(a.checkGeneration(snap("uploading", 7))); got != codes.OK { + t.Errorf("checkGeneration(uploading@7) code = %v, want OK", got) + } +} + +// CommitLocal and CommitDurable skip older generations, and an older +// CommitDurable leaves a newer LatestLocal's upload schedule and backoff intact. +func TestTrackerCommitsSkipOlderGenerations(t *testing.T) { + store := newFakeSnapshotStore() + tr := newIdleTestTracker(t, store) + t0 := tr.clock.Now() + + st := tr.CommitLocal(trackedActorUID, testRef("s3", 3, t0)) + for _, stale := range []SnapshotRef{ + testRef("s2", 2, t0.Add(time.Minute)), + testRef("other", 3, t0.Add(time.Minute)), + testRef("s3", 3, t0.Add(time.Minute)), + } { + got := tr.CommitLocal(trackedActorUID, stale) + if !proto.Equal(got, st) { + t.Errorf("CommitLocal(%v) = %v, want unchanged %v", stale.Snapshot, got, st) + } + if keep := tr.LocalSnapshotsToKeep(trackedActorUID); !slices.Equal(keep, []string{"s3"}) { + t.Errorf("LocalSnapshotsToKeep after CommitLocal(%v) = %v, want [s3]", stale.Snapshot, keep) + } + } + + s4 := testRef("s4", 4, t0.Add(2*time.Minute)) + s4.UploadTime = t0.Add(2 * time.Minute) + st = tr.CommitDurable(trackedActorUID, s4) + for _, stale := range []SnapshotRef{ + testRef("s2", 2, t0.Add(3*time.Minute)), + testRef("other", 4, t0.Add(3*time.Minute)), + testRef("s4", 4, t0.Add(3*time.Minute)), + } { + if got := tr.CommitDurable(trackedActorUID, stale); !proto.Equal(got, st) { + t.Errorf("CommitDurable(%v) = %v, want unchanged %v", stale.Snapshot, got, st) + } + } + for _, staleLocal := range []SnapshotRef{ + testRef("s3", 3, t0.Add(3*time.Minute)), + testRef("other", 4, t0.Add(3*time.Minute)), + } { + if got := tr.CommitLocal(trackedActorUID, staleLocal); !proto.Equal(got, st) { + t.Errorf("CommitLocal(%v) below durable = %v, want unchanged %v", staleLocal.Snapshot, got, st) + } + } + + // CommitLocal(s6) sets UndurableSince; CommitDurable(s5) between LatestDurable + // (s4) and LatestLocal (s6) advances LatestDurable without clearing + // LatestLocal, UndurableSince, or retry backoff. + t6 := t0.Add(4 * time.Minute) + tr.CommitLocal(trackedActorUID, testRef("s6", 6, t6)) + stepTo(tr, t6.Add(testUploadDelay)) + store.setErr(errors.New("boom")) + if _, err := tr.Detach(t.Context(), trackedActorUID); err == nil { + t.Fatal("detach succeeded, want the store's error") + } + wantRetry := tr.clock.Now().Add(testRetryDelay) + + s5 := testRef("s5", 5, t6.Add(time.Minute)) + s5.UploadTime = tr.clock.Now() + st = tr.CommitDurable(trackedActorUID, s5) + if got := uuidOf(st.GetLatestDurable()); got != "s5" { + t.Errorf("latest_durable after CommitDurable(s5) = %q, want s5", got) + } + if got := uuidOf(st.GetLatestLocal()); got != "s6" { + t.Errorf("latest_local after CommitDurable(s5) = %q, want s6", got) + } + if got := timeOf(st.GetUploadDueTime()); !got.Equal(wantRetry) { + t.Errorf("upload_due_time after CommitDurable(s5) during backoff = %v, want %v", got, wantRetry) + } +} diff --git a/cmd/atelet/internal/snapshottracker/upload.go b/cmd/atelet/internal/snapshottracker/upload.go new file mode 100644 index 0000000000..0b7864ab26 --- /dev/null +++ b/cmd/atelet/internal/snapshottracker/upload.go @@ -0,0 +1,169 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package snapshottracker + +import ( + "context" + "log/slog" + "time" +) + +// beginUploadLocked begins a background upload of the actor's nextUploadTarget, +// under a context derived from ctx, and returns the func that runs it once the +// caller releases the actor's lock. Any superseded snapshot in a.Uploading is +// replaced in the same state-file save, and its local copy removed (see +// removeSupersededLocalCopy) before the upload runs. +func (t *Tracker) beginUploadLocked(ctx context.Context, a *actorSnapshots) func() { + superseded, _ := a.clearSupersededUploading() + target := *a.nextUploadTarget() + wasUploading := sameUUID(a.Uploading, &target) + opCtx, op := t.beginOpLocked(ctx, a, opUpload, target) + // beginOpLocked records target in a.Uploading. Save only when target was + // not already in a.Uploading, as it already is on disk when retrying or + // resuming an earlier upload. + if !wasUploading { + t.saveLocked(a) + } + actorUID := a.actorUID + return func() { + if superseded != nil { + t.removeSupersededLocalCopy(actorUID, *superseded) + } + t.backgroundUpload(opCtx, actorUID, op) + } +} + +// cleanupSupersededUploadingLocked clears a.Uploading when no upload is +// running and a newer commit has superseded it, so backgroundUpload will never +// run for it. It saves the change and returns a func that removes the local +// copy kept for that upload (see removeSupersededLocalCopy), or nil. +func (t *Tracker) cleanupSupersededUploadingLocked(a *actorSnapshots) func() { + superseded, cleared := a.clearSupersededUploading() + if !cleared { + return nil + } + t.saveLocked(a) + if superseded == nil { + return nil + } + actorUID := a.actorUID + return func() { + t.removeSupersededLocalCopy(actorUID, *superseded) + } +} + +// uploadSnapshot uploads target from its local copy to object storage and, +// when background is true, records the background upload metric. +func (t *Tracker) uploadSnapshot(ctx context.Context, actorUID string, target SnapshotRef, background bool) (uploaded bool, elapsed time.Duration, err error) { + start := t.clock.Now() + uploaded, err = t.store.UploadLocalSnapshot(ctx, actorUID, target) + elapsed = t.clock.Since(start) + if background { + t.instruments.recordUpload(ctx, target, elapsed, uploaded, err) + } + return uploaded, elapsed, err +} + +// backgroundUpload runs op, a background upload of the actor's nextUploadTarget +// when it began, on a worker. Once the upload succeeds or a newer snapshot +// replaces target, it removes target's local copy if LatestLocal moved on (see +// removeSupersededLocalCopy); when the upload fails and LatestLocal cannot +// replace target, it keeps target in Uploading to retry. An upload Forget +// abandoned leaves the actor alone. +func (t *Tracker) backgroundUpload(ctx context.Context, actorUID string, op *actorOp) { + defer close(op.done) + target := op.target + uploaded, elapsed, err := t.uploadSnapshot(ctx, actorUID, target, true) + logOp(ctx, actorUID, op.kind, target, uploaded, true, elapsed, err) + + t.lock(actorUID) + a := t.endOpLocked(actorUID, op) + if a == nil { + t.unlock(actorUID) + return + } + removeOld := false + switch { + case err == nil: + if t.recordUploadLocked(a, target, uploaded) { + t.markReportDueLocked(a) + } + a.Uploading = nil + a.uploadingFailed = false + t.succeededLocked(a) + t.queueReportLocked(a) + removeOld = a.localSuperseded(&target) + case !a.uploadNeeded(target): + // A newer snapshot became durable while the upload ran, as an + // immediate checkpoint or a promote makes one, so the failure costs + // nothing. + a.Uploading = nil + a.uploadingFailed = false + t.saveLocked(a) + t.requeueLocked(a) + removeOld = a.localSuperseded(&target) + default: + if t.ctx.Err() == nil { + a.uploadingFailed = true + if superseded, cleared := a.clearSupersededUploading(); cleared { + t.saveLocked(a) + removeOld = superseded != nil + } + if _, ok := t.nextDueLocked(a); ok { + t.backOffLocked(a) + } + } + t.requeueLocked(a) + } + t.unlock(actorUID) + if removeOld { + t.removeSupersededLocalCopy(actorUID, target) + } +} + +// recordUploadLocked records that an upload of target succeeded, and reports +// whether target is now LatestDurable, which it is not when that generation or +// a newer snapshot was already durable. uploaded reports whether the upload +// wrote the manifest rather than finding it committed already. +func (t *Tracker) recordUploadLocked(a *actorSnapshots, target SnapshotRef, uploaded bool) bool { + now := t.clock.Now() + d := target.clone() + if uploaded { + d.UploadTime = now + } + if !a.recordDurable(d) { + return false + } + a.uploadSucceeded(now) + return true +} + +// removeSupersededLocalCopy deletes the local copy of target, an upload's +// snapshot that a newer snapshot superseded, once no upload needs it: the copy +// was kept only for the upload. It leaves the copy while a Run, Restore, +// Checkpoint, or PromoteSnapshot of the actor is in progress, since a Restore +// or PromoteSnapshot may be reading it, and deletes it under the actor's lock +// so that none begins reading it meanwhile. Best-effort: the actor's next +// checkpoint, detach, or terminate prunes a copy left behind. +func (t *Tracker) removeSupersededLocalCopy(actorUID string, target SnapshotRef) { + t.lock(actorUID) + defer t.unlock(actorUID) + if t.heldLocked(actorUID) { + return + } + if err := removeLocalCheckpoint(t.ctx, actorUID, target.uuid()); err != nil { + slog.WarnContext(t.ctx, "Failed to delete a superseded local checkpoint", slog.String("actorUID", actorUID), slog.Any("err", err)) + } +} diff --git a/cmd/atelet/internal/snapshottracker/upload_test.go b/cmd/atelet/internal/snapshottracker/upload_test.go new file mode 100644 index 0000000000..995a8431dc --- /dev/null +++ b/cmd/atelet/internal/snapshottracker/upload_test.go @@ -0,0 +1,1235 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package snapshottracker + +import ( + "context" + "errors" + "os" + "slices" + "testing" + "time" + + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/proto/ateletpb" + "google.golang.org/grpc/codes" + "google.golang.org/protobuf/proto" +) + +// startUpload advances the clock to when the tracked actor's upload is due, +// and waits for a worker to start it. It returns a channel closed once the +// upload has ended. +func startUpload(t *testing.T, tr *Tracker, store *fakeSnapshotStore) <-chan struct{} { + t.Helper() + due := timeOf(tr.State(trackedActorUID).GetUploadDueTime()) + if due.IsZero() { + t.Fatal("no upload scheduled") + } + stepTo(tr, due) + waitStarted(t, store) + return opDone(t, tr, trackedActorUID) +} + +// uploadNow runs the tracked actor's upload once it is due, and ends it on the +// blocking store with err. +func uploadNow(t *testing.T, tr *Tracker, store *fakeSnapshotStore, err error) { + t.Helper() + done := startUpload(t, tr, store) + store.release <- err + waitClosed(t, done, "the upload") +} + +// Deferred checkpoints keep their snapshots local, and atelet uploads the +// latest one once the oldest snapshot not yet durable has waited the upload +// delay. The local copy stays after the upload. +func TestTrackerUploadsLatestLocalSnapshot(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newTestTracker(t, store) + if err := os.MkdirAll(ateletpath.LocalSnapshotDir(trackedActorUID, "s2"), 0o700); err != nil { + t.Fatal(err) + } + t0 := tr.clock.Now() + + st := tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) + if got := st.GetLatestLocal(); uuidOf(got) != "s1" || !timeOf(got.GetLocalCommitTime()).Equal(t0) { + t.Errorf("latest_local = %v, want s1 committed at %v", got, t0) + } + if got, want := timeOf(st.GetUploadDueTime()), t0.Add(testUploadDelay); !got.Equal(want) { + t.Errorf("upload_due_time = %v, want %v", got, want) + } + + st = tr.CommitLocal(trackedActorUID, testRef("s2", 2, t0.Add(10*time.Minute))) + if got, want := timeOf(st.GetUploadDueTime()), t0.Add(testUploadDelay); !got.Equal(want) { + t.Errorf("upload_due_time = %v, want %v: when the first snapshot was due", got, want) + } + + uploadNow(t, tr, store, nil) + if got, want := store.uploads(), []string{"s2"}; !slices.Equal(got, want) { + t.Errorf("uploads = %v, want %v", got, want) + } + if _, err := os.Stat(ateletpath.LocalSnapshotDir(trackedActorUID, "s2")); err != nil { + t.Errorf("s2 local dir after upload: %v, want it kept", err) + } + st = tr.State(trackedActorUID) + if got := st.GetLatestDurable(); uuidOf(got) != "s2" || got.GetDurableCompleteTime() == nil { + t.Errorf("latest_durable = %v, want s2 with its upload time", got) + } + if got := uuidOf(st.GetLatestLocal()); got != "s2" { + t.Errorf("latest_local = %q, want s2 kept after its upload", got) + } + if st.GetUploading() != nil || st.GetUploadDueTime() != nil { + t.Errorf("state = %v, want no upload in progress or due", st) + } +} + +// A deferred checkpoint during an older snapshot's upload keeps that snapshot +// until the upload ends: LocalSnapshotsToKeep names it beside the new one. The +// upload's end deletes it and schedules the new snapshot's upload, which the +// older upload does not cover, an upload delay after the older upload +// succeeded. +func TestTrackerCheckpointDuringUpload(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newTestTracker(t, store) + for _, name := range []string{"s1", "s2"} { + if err := os.MkdirAll(ateletpath.LocalSnapshotDir(trackedActorUID, name), 0o700); err != nil { + t.Fatal(err) + } + } + t0 := tr.clock.Now() + tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) + done := startUpload(t, tr, store) + + st := tr.CommitLocal(trackedActorUID, testRef("s2", 2, tr.clock.Now())) + if keep := tr.LocalSnapshotsToKeep(trackedActorUID); !slices.Equal(keep, []string{"s2", "s1"}) { + t.Errorf("LocalSnapshotsToKeep = %v, want [s2 s1]: s1 is being uploaded", keep) + } + if got := uuidOf(st.GetUploading()); got != "s1" { + t.Errorf("uploading = %q, want s1", got) + } + if got, want := timeOf(st.GetUploadDueTime()), t0.Add(testUploadDelay); !got.Equal(want) { + t.Errorf("upload_due_time = %v, want %v: s1's, which the upload in progress delays", got, want) + } + + stepTo(tr, tr.clock.Now().Add(10*time.Minute)) + store.release <- nil + waitClosed(t, done, "the upload of s1") + if _, err := os.Stat(ateletpath.LocalSnapshotDir(trackedActorUID, "s1")); !os.IsNotExist(err) { + t.Errorf("s1 after its upload: stat = %v, want it deleted", err) + } + if _, err := os.Stat(ateletpath.LocalSnapshotDir(trackedActorUID, "s2")); err != nil { + t.Errorf("s2 after the upload of s1: %v, want it kept", err) + } + st = tr.State(trackedActorUID) + if got := uuidOf(st.GetLatestDurable()); got != "s1" { + t.Errorf("latest_durable = %q, want s1", got) + } + if got, want := timeOf(st.GetUploadDueTime()), tr.clock.Now().Add(testUploadDelay); !got.Equal(want) { + t.Errorf("upload_due_time = %v, want %v: an upload delay after the upload of s1 succeeded", got, want) + } + + uploadNow(t, tr, store, nil) + if got := uuidOf(tr.State(trackedActorUID).GetLatestDurable()); got != "s2" { + t.Errorf("latest_durable = %q, want s2", got) + } +} + +// An immediate checkpoint makes the actor's newest snapshot durable, which +// drops the older local snapshot, then records its own local copy. An older +// snapshot's upload in progress runs on: LocalSnapshotsToKeep keeps that +// snapshot, and the upload deletes it when it ends without holding the actor +// back when it fails, though the new local copy has a detach scheduled. +func TestTrackerImmediateCheckpointDuringUpload(t *testing.T) { + for _, tc := range []struct { + name string + uploadErr error + }{ + {name: "older upload succeeds"}, + {name: "older upload fails", uploadErr: errors.New("boom")}, + } { + t.Run(tc.name, func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newTestTracker(t, store) + if err := os.MkdirAll(ateletpath.LocalSnapshotDir(trackedActorUID, "s1"), 0o700); err != nil { + t.Fatal(err) + } + t0 := tr.clock.Now() + tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) + done := startUpload(t, tr, store) + + s2 := testRef("s2", 2, t0.Add(time.Minute)) + s2.UploadTime = s2.CommitTime.Add(time.Second) + st := tr.CommitDurable(trackedActorUID, s2) + if st.GetLatestLocal() != nil || st.GetUploadDueTime() != nil || uuidOf(st.GetUploading()) != "s1" { + t.Errorf("state = %v, want nothing local or due, and s1 uploading", st) + } + if keep := tr.LocalSnapshotsToKeep(trackedActorUID); !slices.Equal(keep, []string{"s1"}) { + t.Errorf("LocalSnapshotsToKeep = %v, want [s1]: s1 is being uploaded", keep) + } + if got := st.GetLatestDurable(); uuidOf(got) != "s2" || !timeOf(got.GetDurableCompleteTime()).Equal(s2.UploadTime) { + t.Errorf("latest_durable = %v, want s2 uploaded at %v", got, s2.UploadTime) + } + tr.CommitLocal(trackedActorUID, testRef("s2", 2, s2.CommitTime)) + + store.release <- tc.uploadErr + waitClosed(t, done, "the upload of s1") + if _, err := os.Stat(ateletpath.LocalSnapshotDir(trackedActorUID, "s1")); !os.IsNotExist(err) { + t.Errorf("s1 after its upload: stat = %v, want it deleted", err) + } + if got := uuidOf(tr.State(trackedActorUID).GetLatestDurable()); got != "s2" { + t.Errorf("latest_durable = %q after the upload of s1, want s2", got) + } + if inMemory(tr) { + t.Error("entry stayed in memory after the older upload ended, want it dropped") + } + + // The next deferred checkpoint is due a full delay after it was taken. + s3Time := tr.clock.Now() + st = tr.CommitLocal(trackedActorUID, testRef("s3", 3, s3Time)) + if got, want := timeOf(st.GetUploadDueTime()), s3Time.Add(testUploadDelay); !got.Equal(want) { + t.Errorf("upload_due_time = %v, want %v", got, want) + } + }) + } +} + +// An older upload finishing after a newer snapshot became durable and another +// local checkpoint was taken does not push the new local checkpoint's upload +// due time back. +func TestTrackerLateUploadDoesNotPushBackNextUpload(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newTestTracker(t, store) + t0 := tr.clock.Now() + tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) + done := startUpload(t, tr, store) + + s2 := testRef("s2", 2, t0.Add(time.Minute)) + s2.UploadTime = s2.CommitTime.Add(time.Second) + tr.CommitDurable(trackedActorUID, s2) + t3 := t0.Add(2 * time.Minute) + tr.CommitLocal(trackedActorUID, testRef("s3", 3, t3)) + + stepTo(tr, t3.Add(10*time.Minute)) + store.release <- nil + waitClosed(t, done, "the upload of s1") + + st := tr.State(trackedActorUID) + if got := uuidOf(st.GetLatestDurable()); got != "s2" { + t.Errorf("latest_durable = %q, want s2", got) + } + if got, want := timeOf(st.GetUploadDueTime()), t3.Add(testUploadDelay); !got.Equal(want) { + t.Errorf("upload_due_time = %v, want %v: an upload delay after s3 was taken", got, want) + } +} + +// A failed background upload is retried once the retry delay has passed, and +// the delay doubles with each further failure. +func TestTrackerRetriesFailedUpload(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newTestTracker(t, store) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + + for _, delay := range []time.Duration{testRetryDelay, 2 * testRetryDelay} { + uploadNow(t, tr, store, errors.New("boom")) + st := tr.State(trackedActorUID) + if st.GetLatestDurable() != nil { + t.Errorf("latest_durable = %v after a failed upload, want unset", st.GetLatestDurable()) + } + if got, want := timeOf(st.GetUploadDueTime()), tr.clock.Now().Add(delay); !got.Equal(want) { + t.Errorf("upload_due_time = %v, want %v: %v after the failure", got, want, delay) + } + } + + uploadNow(t, tr, store, nil) + if got := uuidOf(tr.State(trackedActorUID).GetLatestDurable()); got != "s1" { + t.Errorf("latest_durable = %q after the retry, want s1", got) + } +} + +// A failed upload leaves the actor undurable since its oldest snapshot not yet +// durable, rather than since the snapshot the upload read, and a checkpoint +// taken during the upload does not move that. +func TestTrackerFailedUploadKeepsUndurableSince(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newTestTracker(t, store) + t0 := tr.clock.Now() + tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) + tr.CommitLocal(trackedActorUID, testRef("s2", 2, t0.Add(50*time.Minute))) + done := startUpload(t, tr, store) + tr.CommitLocal(trackedActorUID, testRef("s3", 3, tr.clock.Now())) + + store.release <- errors.New("boom") + waitClosed(t, done, "the upload of s2") + if got, want := timeOf(tr.State(trackedActorUID).GetUploadDueTime()), tr.clock.Now().Add(testRetryDelay); !got.Equal(want) { + t.Errorf("upload_due_time = %v, want %v: the retry delay after the failure, as s1 has waited over an upload delay", got, want) + } +} + +// Forget does not wait for a background upload in progress: it cancels the +// upload, which then leaves the forgotten actor alone. The upload records no +// state for the actor, and deletes none of its local snapshots, not even one +// a newer checkpoint superseded: Terminate prunes them. Its metric records +// error.type Canceled. +func TestTrackerForgetCancelsUpload(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newIdleTestTracker(t, store) + inst, reader := newTestInstruments(t) + tr.instruments = inst + tr.Start() + if err := os.MkdirAll(ateletpath.LocalSnapshotDir(trackedActorUID, "s1"), 0o700); err != nil { + t.Fatal(err) + } + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + done := startUpload(t, tr, store) + tr.CommitLocal(trackedActorUID, testRef("s2", 2, tr.clock.Now())) + + returnsPromptly(t, "Forget during an upload", func() { + if err := tr.Forget(trackedActorUID); err != nil { + t.Errorf("Forget: %v", err) + } + }) + waitClosed(t, done, "the cancelled upload") + if st := tr.State(trackedActorUID); !proto.Equal(st, &ateletpb.ActorSnapshotState{}) { + t.Errorf("state after forget = %v, want empty", st) + } + if _, err := os.Stat(ateletpath.ActorSnapshotStateFile(trackedActorUID)); !os.IsNotExist(err) { + t.Errorf("state file once the cancelled upload ended: stat = %v, want none", err) + } + if _, err := os.Stat(ateletpath.LocalSnapshotDir(trackedActorUID, "s1")); err != nil { + t.Errorf("s1 once the cancelled upload ended: %v, want it left for Terminate to prune", err) + } + if got, want := recordedErrorTypes(t, reader), []string{codes.Canceled.String()}; !slices.Equal(got, want) { + t.Errorf("recorded upload error types = %v, want %v", got, want) + } +} + +// The upload workers bound how many background uploads run at once, and an +// actor that checkpoints again while it waits for one uploads only its newest +// snapshot. +func TestTrackerWorkersBoundBackgroundWork(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newIdleTestTracker(t, store) + // The idle detach worker leaves actor-3's upload to the upload workers. + setWorkers(tr, 2, 1) + tr.Start() + + // Snapshots taken an upload delay ago are due for upload straight away. + taken := tr.clock.Now().Add(-testUploadDelay) + for _, id := range []string{"actor-1", "actor-2", "actor-3"} { + if err := os.MkdirAll(ateletpath.ActorPath(id), 0o700); err != nil { + t.Fatal(err) + } + tr.CommitLocal(id, testRef(id+"-s1", 1, taken)) + } + started := []string{waitStarted(t, store), waitStarted(t, store)} + slices.Sort(started) + if want := []string{"actor-1-s1", "actor-2-s1"}; !slices.Equal(started, want) { + t.Fatalf("first started = %v, want %v", started, want) + } + select { + case got := <-store.started: + t.Fatalf("a third upload (%q) started while both upload workers are busy", got) + default: + } + if st := tr.State("actor-3"); st.GetUploading() != nil || st.GetUploadDueTime() == nil { + t.Errorf("actor-3 state while it waits = %v, want an upload due and none in progress", st) + } + + // actor-3 checkpoints again while it waits for an upload worker. + tr.CommitLocal("actor-3", testRef("actor-3-s2", 2, taken.Add(time.Minute))) + + done1 := opDone(t, tr, "actor-1") + done2 := opDone(t, tr, "actor-2") + // Ending one upload frees an upload worker, which takes actor-3. + store.release <- nil + if got := waitStarted(t, store); got != "actor-3-s2" { + t.Fatalf("started = %q, want actor-3-s2", got) + } + done3 := opDone(t, tr, "actor-3") + store.release <- nil + store.release <- nil + waitClosed(t, done1, "actor-1's upload") + waitClosed(t, done2, "actor-2's upload") + waitClosed(t, done3, "actor-3's upload") + + if got := uuidOf(tr.State("actor-3").GetLatestDurable()); got != "actor-3-s2" { + t.Errorf("actor-3 latest_durable = %q, want actor-3-s2", got) + } + if got := len(store.uploads()); got != 3 { + t.Errorf("uploads = %v, want 3: actor-3-s1 superseded before it was uploaded", store.uploads()) + } +} + +// The queue holds an actor until the earliest time it was queued for. A worker +// that takes the actor before its upload is due, as after an immediate +// checkpoint made the snapshot it was queued for durable, does nothing and +// queues it again for when its upload is due. +func TestTrackerRequeuesUntilWorkIsDue(t *testing.T) { + store := newFakeSnapshotStore() + tr := newIdleTestTracker(t, store) + t0 := tr.clock.Now() + // Queued for the upload of s1, due an upload delay out. The immediate + // checkpoint makes s2 durable, so the upload of s3 is due an upload delay + // after s3 was taken, later. + tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) + tr.CommitDurable(trackedActorUID, testRef("s2", 2, t0.Add(10*time.Minute))) + tr.CommitLocal(trackedActorUID, testRef("s3", 3, t0.Add(20*time.Minute))) + + stepTo(tr, t0.Add(testUploadDelay)) + processNext(t, tr, opUpload) + if got := store.uploads(); len(got) != 0 { + t.Errorf("uploads = %v before the upload of s3 is due, want none", got) + } + + stepTo(tr, t0.Add(20*time.Minute+testUploadDelay)) + processNext(t, tr, opUpload) + if got, want := store.uploads(), []string{"s3"}; !slices.Equal(got, want) { + t.Errorf("uploads = %v, want %v", got, want) + } +} + +// Start queues each actor with a snapshot state file, so a tracker started +// after an atelet restart uploads the newest snapshot its predecessor left +// due, using the oldest undurable snapshot's commit time. +func TestTrackerResumesUploadAfterRestart(t *testing.T) { + useActorDir(t) + store := newFakeSnapshotStore() + before := newStoreTracker(t, store) + t0 := before.clock.Now().Add(-testUploadDelay) + before.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) + before.CommitLocal(trackedActorUID, testRef("s2", 2, t0.Add(50*time.Minute))) + + after := newStoreTracker(t, store) + if got, want := timeOf(after.State(trackedActorUID).GetUploadDueTime()), t0.Add(testUploadDelay); !got.Equal(want) { + t.Fatalf("upload_due_time after restart = %v, want %v from s1's commit time", got, want) + } + after.Start() + if got := waitStarted(t, store); got != "s2" { + t.Fatalf("started = %q after the restart, want s2", got) + } +} + +// A tracker without a store schedules no upload or detach, and reports no +// upload due. +func TestTrackerWithoutStoreSchedulesNoUploads(t *testing.T) { + useActorDir(t) + tr := newTracker(t) + taken := tr.clock.Now().Add(-time.Hour) + if st := tr.CommitLocal(trackedActorUID, testRef("s1", 1, taken)); st.GetUploadDueTime() != nil { + t.Errorf("upload_due_time = %v without a store, want unset", st.GetUploadDueTime()) + } + if n := queuedNow(tr); n != 0 { + t.Errorf("queued actors = %d without a store, want none", n) + } + // The queues' lengths leave out actors queued for later, as a detach is. + tr.lock(trackedActorUID) + due, ok := tr.nextDueLocked(tr.findLocked(trackedActorUID)) + tr.unlock(trackedActorUID) + if ok { + t.Errorf("background work due at %v without a store, want none", due) + } +} + +// A local snapshot no upload can make durable, as a pause takes today, never +// comes due for a background upload, rather than failing it again and again. +// A checkpoint that replaces it schedules an upload again. +func TestTrackerSkipsUnuploadableSnapshots(t *testing.T) { + for _, tc := range []struct { + name string + commit func(tr *Tracker, taken time.Time) + }{ + { + name: "no object storage location", + commit: func(tr *Tracker, taken time.Time) { + local := testRef("s1", 1, taken) + local.Snapshot.Object = nil + tr.CommitLocal(trackedActorUID, local) + }, + }, + { + name: "durable as another generation", + commit: func(tr *Tracker, taken time.Time) { + tr.CommitDurable(trackedActorUID, testRef("s1", 1, taken)) + tr.CommitLocal(trackedActorUID, testRef("s1", 2, taken)) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + tr := newIdleTestTracker(t, newFakeSnapshotStore()) + // Taken an upload delay ago, so an upload would be due at once. + taken := tr.clock.Now().Add(-testUploadDelay) + tc.commit(tr, taken) + if due := tr.State(trackedActorUID).GetUploadDueTime(); due != nil { + t.Errorf("upload_due_time = %v, want unset", due) + } + if n := queuedNow(tr); n != 0 { + t.Errorf("queued actors = %d, want none", n) + } + + tr.CommitLocal(trackedActorUID, testRef("s2", 3, taken)) + if due := tr.State(trackedActorUID).GetUploadDueTime(); due == nil { + t.Error("upload_due_time unset once a checkpoint replaced the snapshot, want set") + } + }) + } +} + +// A Run or Restore unschedules detach, but leaves failed upload backoff intact +// while an upload is still due. +func TestTrackerStartKeepsUploadBackoff(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newTestTracker(t, store) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + + uploadNow(t, tr, store, errors.New("boom")) + retryAt := tr.clock.Now().Add(testRetryDelay) + + end, err := tr.BeginStart(t.Context(), trackedActorUID) + if err != nil { + t.Fatalf("BeginStart: %v", err) + } + end() + + if got := timeOf(tr.State(trackedActorUID).GetUploadDueTime()); !got.Equal(retryAt) { + t.Fatalf("upload_due_time after start = %v, want %v kept", got, retryAt) + } + uploadNow(t, tr, store, errors.New("boom")) + if got, want := timeOf(tr.State(trackedActorUID).GetUploadDueTime()), retryAt.Add(2*testRetryDelay); !got.Equal(want) { + t.Errorf("upload_due_time after second failure = %v, want %v (doubled)", got, want) + } +} + +// BeginStart and BeginCheckpoint wait only for a detach in progress, not for a +// background upload. +func TestTrackerStartAndCheckpointDoNotWaitForUpload(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newTestTracker(t, store) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + done := startUpload(t, tr, store) + + for _, tc := range []struct { + name string + begin func(context.Context, string) (func(), error) + }{ + {"BeginStart", tr.BeginStart}, + {"BeginCheckpoint", func(ctx context.Context, uid string) (func(), error) { + return tr.BeginCheckpoint(ctx, uid, testRef("s2", 2, time.Time{}).Snapshot) + }}, + } { + returnsPromptly(t, tc.name+" during a background upload", func() { + end, err := tc.begin(t.Context(), trackedActorUID) + if err != nil { + t.Errorf("%s: %v", tc.name, err) + return + } + end() + }) + } + + store.release <- nil + waitClosed(t, done, "the upload") +} + +// A background upload runs even while the actor is held by a Run, Restore, or +// Checkpoint, since an upload keeps the local snapshot intact. +func TestTrackerUploadRunsWhileActorHeld(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newTestTracker(t, store) + t0 := tr.clock.Now() + tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) + + end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, testRef("s2", 2, t0).Snapshot) + if err != nil { + t.Fatalf("BeginCheckpoint: %v", err) + } + defer end() + + done := startUpload(t, tr, store) + store.release <- nil + waitClosed(t, done, "the upload while held") + if got := uuidOf(tr.State(trackedActorUID).GetLatestDurable()); got != "s1" { + t.Errorf("latest_durable = %q, want s1", got) + } +} + +// BeginPromote waits for any detach in progress and any background upload of +// the same snapshot, reports whether the snapshot is already durable, and +// fails with FailedPrecondition when the durable snapshot with that UUID has a +// different generation, fidelity, or URI. +func TestTrackerBeginPromote(t *testing.T) { + t.Run("waits for detach", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newTestTracker(t, store) + s1 := testRef("s1", 1, tr.clock.Now()) + tr.CommitLocal(trackedActorUID, s1) + detached := startDetach(t, tr) + waitStarted(t, store) + + type promoteRes struct { + durable bool + err error + } + resCh := make(chan promoteRes, 1) + go func() { + durable, end, err := tr.BeginPromote(t.Context(), trackedActorUID, s1) + if err == nil { + end() + } + resCh <- promoteRes{durable, err} + }() + select { + case res := <-resCh: + t.Fatalf("BeginPromote returned (%v, %v) during a detach", res.durable, res.err) + case <-time.After(50 * time.Millisecond): + } + + store.release <- nil + if err := <-detached; err != nil { + t.Fatalf("detach: %v", err) + } + res := <-resCh + if res.err != nil || !res.durable { + t.Errorf("BeginPromote = (%v, %v), want (true, nil)", res.durable, res.err) + } + }) + + t.Run("waits for upload of same snapshot", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newTestTracker(t, store) + s1 := testRef("s1", 1, tr.clock.Now()) + tr.CommitLocal(trackedActorUID, s1) + done := startUpload(t, tr, store) + + type promoteRes struct { + durable bool + err error + } + resCh := make(chan promoteRes, 1) + go func() { + durable, end, err := tr.BeginPromote(t.Context(), trackedActorUID, s1) + if err == nil { + end() + } + resCh <- promoteRes{durable, err} + }() + select { + case res := <-resCh: + t.Fatalf("BeginPromote returned (%v, %v) during an upload of the same snapshot", res.durable, res.err) + case <-time.After(50 * time.Millisecond): + } + + store.release <- nil + waitClosed(t, done, "the upload") + res := <-resCh + if res.err != nil || !res.durable { + t.Errorf("BeginPromote = (%v, %v), want (true, nil)", res.durable, res.err) + } + }) + + t.Run("rejects snapshot superseded during upload", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newTestTracker(t, store) + s1 := testRef("s1", 1, tr.clock.Now()) + tr.CommitLocal(trackedActorUID, s1) + done := startUpload(t, tr, store) + + errCh := make(chan error, 1) + go func() { + _, end, err := tr.BeginPromote(t.Context(), trackedActorUID, s1) + if err == nil { + end() + } + errCh <- err + }() + select { + case err := <-errCh: + t.Fatalf("BeginPromote returned (%v) during an upload of the same snapshot", err) + case <-time.After(50 * time.Millisecond): + } + + tr.CommitLocal(trackedActorUID, testRef("s2", 2, tr.clock.Now())) + store.release <- nil + waitClosed(t, done, "the upload") + if err := <-errCh; apierror.Code(err) != codes.FailedPrecondition { + t.Errorf("BeginPromote = %v, want FailedPrecondition", err) + } + }) + + t.Run("does not wait for upload of older snapshot", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newTestTracker(t, store) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + done := startUpload(t, tr, store) + s2 := testRef("s2", 2, tr.clock.Now()) + tr.CommitLocal(trackedActorUID, s2) + + returnsPromptly(t, "BeginPromote of a newer snapshot during an older upload", func() { + durable, end, err := tr.BeginPromote(t.Context(), trackedActorUID, s2) + if err != nil { + t.Errorf("BeginPromote: %v", err) + return + } + end() + if durable { + t.Errorf("BeginPromote durable = true, want false") + } + }) + + store.release <- nil + waitClosed(t, done, "the upload") + }) + + for _, tc := range []struct { + name string + mutate func(s *ateletpb.Snapshot) + }{ + { + name: "refuses generation mismatch", + mutate: func(s *ateletpb.Snapshot) { s.Generation = 2 }, + }, + { + name: "refuses fidelity mismatch", + mutate: func(s *ateletpb.Snapshot) { s.Fidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES }, + }, + { + name: "refuses URI mismatch", + mutate: func(s *ateletpb.Snapshot) { s.Object.SnapshotUri += "-other" }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + tr := newIdleTestTracker(t, newFakeSnapshotStore()) + tr.CommitDurable(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + req := testRef("s1", 1, time.Time{}) + tc.mutate(req.Snapshot) + if _, _, err := tr.BeginPromote(t.Context(), trackedActorUID, req); apierror.Code(err) != codes.FailedPrecondition { + t.Errorf("BeginPromote = %v, want FailedPrecondition", err) + } + }) + } +} + +// When a background upload of s1 fails after a pause replaced LatestLocal with +// s2 (which has no object storage location), s1 stays in Uploading on disk and +// is retried. Further pause checkpoints keep s1 on disk and preserve its +// backoff, and once s1's retry succeeds, s1's local copy is deleted. +func TestTrackerRetriesUploadingWhenLatestLocalHasNoObject(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newTestTracker(t, store) + for _, name := range []string{"s1", "s2", "s3"} { + if err := os.MkdirAll(ateletpath.LocalSnapshotDir(trackedActorUID, name), 0o700); err != nil { + t.Fatal(err) + } + } + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + done := startUpload(t, tr, store) + + s2 := testRef("s2", 2, tr.clock.Now()) + s2.Snapshot.Object = nil + tr.CommitLocal(trackedActorUID, s2) + if keep := tr.LocalSnapshotsToKeep(trackedActorUID); !slices.Equal(keep, []string{"s2", "s1"}) { + t.Errorf("LocalSnapshotsToKeep during upload = %v, want [s2 s1]", keep) + } + + store.release <- errors.New("boom") + waitClosed(t, done, "the failed upload of s1") + retryAt := tr.clock.Now().Add(testRetryDelay) + + if _, err := os.Stat(ateletpath.LocalSnapshotDir(trackedActorUID, "s1")); err != nil { + t.Fatalf("s1 after failed upload: %v, want it kept to retry", err) + } + st := tr.State(trackedActorUID) + if uuidOf(st.GetLatestLocal()) != "s2" || uuidOf(st.GetUploading()) != "s1" || !timeOf(st.GetUploadDueTime()).Equal(retryAt) { + t.Fatalf("state after failed upload = %v, want s2 local, s1 uploading, due at %v", st, retryAt) + } + + // Another pause checkpoint during backoff still keeps s1 and its backoff. + s3 := testRef("s3", 3, tr.clock.Now()) + s3.Snapshot.Object = nil + st = tr.CommitLocal(trackedActorUID, s3) + if keep := tr.LocalSnapshotsToKeep(trackedActorUID); !slices.Equal(keep, []string{"s3", "s1"}) { + t.Errorf("LocalSnapshotsToKeep during backoff = %v, want [s3 s1]", keep) + } + if uuidOf(st.GetUploading()) != "s1" || !timeOf(st.GetUploadDueTime()).Equal(retryAt) { + t.Errorf("state after s3 = %v, want s1 still uploading and due at %v", st, retryAt) + } + + uploadNow(t, tr, store, nil) + if got, want := store.uploads(), []string{"s1", "s1"}; !slices.Equal(got, want) { + t.Errorf("uploads = %v, want %v", got, want) + } + if _, err := os.Stat(ateletpath.LocalSnapshotDir(trackedActorUID, "s1")); !os.IsNotExist(err) { + t.Errorf("s1 after retry succeeded: stat = %v, want it deleted", err) + } + st = tr.State(trackedActorUID) + if uuidOf(st.GetLatestDurable()) != "s1" || uuidOf(st.GetLatestLocal()) != "s3" || st.GetUploading() != nil || st.GetUploadDueTime() != nil { + t.Errorf("state after retry = %v, want s1 durable, s3 local, and no upload left", st) + } +} + +// When s1 failed and sits in Uploading waiting for retry, a newer durable +// snapshot supersedes s1: a worker clears Uploading and deletes s1's local +// copy without waiting for the retry timer. A promote that makes s1 itself +// durable clears Uploading too, but keeps s1's copy, which is still the latest +// local snapshot. +func TestTrackerClearsIdleUploadingWhenSuperseded(t *testing.T) { + for _, tc := range []struct { + name string + commit func(tr *Tracker) + keepS1 bool + }{ + { + name: "CommitDurable", + commit: func(tr *Tracker) { + tr.CommitDurable(trackedActorUID, testRef("s2", 2, tr.clock.Now())) + }, + }, + { + name: "CommitPromoted of s1", + commit: func(tr *Tracker) { + s1 := testRef("s1", 1, time.Time{}) + s1.UploadTime = tr.clock.Now() + tr.CommitPromoted(trackedActorUID, s1) + }, + keepS1: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + store := newFakeSnapshotStore() + store.setErr(errors.New("boom")) + tr := newIdleTestTracker(t, store) + s1Dir := ateletpath.LocalSnapshotDir(trackedActorUID, "s1") + if err := os.MkdirAll(s1Dir, 0o700); err != nil { + t.Fatal(err) + } + t0 := tr.clock.Now() + tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) + stepTo(tr, t0.Add(testUploadDelay)) + processNext(t, tr, opUpload) // s1's upload fails, and s1 waits in Uploading to retry. + + store.setErr(nil) + tc.commit(tr) + if got := tr.queues[opDetach].Len(); got != 0 { + t.Errorf("detach queue len after %s = %d, want 0", tc.name, got) + } + processNext(t, tr, opUpload) + // State hides an Uploading it no longer keeps, so read the file. + if rec := readSnapshotState(trackedActorUID); rec == nil || rec.Uploading != nil { + t.Errorf("state file after %s = %+v, want Uploading cleared", tc.name, rec) + } + _, err := os.Stat(s1Dir) + switch { + case tc.keepS1 && err != nil: + t.Errorf("s1 after %s: stat = %v, want it kept as the latest local snapshot", tc.name, err) + case !tc.keepS1 && !os.IsNotExist(err): + t.Errorf("s1 after %s: stat = %v, want it deleted", tc.name, err) + } + }) + } +} + +// When s1 failed and sits in Uploading waiting for retry, CommitLocal of a +// newer snapshot that can be uploaded drops s1 from the snapshot state file in +// the same save, and leaves s1's local copy for the Checkpoint's prune. An +// atelet restart before any worker runs then uploads s2, rather than resuming +// s1 from a copy the prune has deleted. +func TestTrackerCommitLocalDropsFailedUploading(t *testing.T) { + store := newFakeSnapshotStore() + store.setErr(errors.New("boom")) + tr := newIdleTestTracker(t, store) + // Taken an upload delay ago, so an upload is due at once, both here and + // after the restart. + taken := tr.clock.Now().Add(-testUploadDelay) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, taken)) + processNext(t, tr, opUpload) // s1's upload fails, and s1 waits in Uploading to retry. + + tr.CommitLocal(trackedActorUID, testRef("s2", 2, tr.clock.Now())) + if rec := readSnapshotState(trackedActorUID); rec == nil || rec.Uploading != nil { + t.Errorf("state file after CommitLocal(s2) = %+v, want Uploading cleared", rec) + } + if keep := tr.LocalSnapshotsToKeep(trackedActorUID); !slices.Equal(keep, []string{"s2"}) { + t.Errorf("LocalSnapshotsToKeep = %v, want [s2]", keep) + } + + store2 := newFakeSnapshotStore() + after := newStoreTracker(t, store2) + after.Start() + if got := waitStarted(t, store2); got != "s2" { + t.Errorf("first upload after the restart = %q, want s2", got) + } +} + +// A superseded local copy stays on disk while a Run, Restore, Checkpoint, or +// PromoteSnapshot of the actor is in progress, since a Restore may be reading +// it in place. It is left for the actor's next checkpoint to prune. +func TestTrackerKeepsSupersededCopyWhileHeld(t *testing.T) { + t.Run("upload ends", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newTestTracker(t, store) + s1Dir := ateletpath.LocalSnapshotDir(trackedActorUID, "s1") + if err := os.MkdirAll(s1Dir, 0o700); err != nil { + t.Fatal(err) + } + tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + done := startUpload(t, tr, store) + tr.CommitLocal(trackedActorUID, testRef("s2", 2, tr.clock.Now())) + + end, err := tr.BeginStart(t.Context(), trackedActorUID) + if err != nil { + t.Fatalf("BeginStart: %v", err) + } + store.release <- nil + waitClosed(t, done, "the upload of s1") + end() + if _, err := os.Stat(s1Dir); err != nil { + t.Errorf("s1 after its upload ended during a Restore: stat = %v, want it kept", err) + } + if keep := tr.LocalSnapshotsToKeep(trackedActorUID); !slices.Equal(keep, []string{"s2"}) { + t.Errorf("LocalSnapshotsToKeep = %v, want [s2], so the next checkpoint prunes s1", keep) + } + }) + + t.Run("worker clears a superseded Uploading", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setErr(errors.New("boom")) + tr := newIdleTestTracker(t, store) + s1Dir := ateletpath.LocalSnapshotDir(trackedActorUID, "s1") + if err := os.MkdirAll(s1Dir, 0o700); err != nil { + t.Fatal(err) + } + taken := tr.clock.Now().Add(-testUploadDelay) + tr.CommitLocal(trackedActorUID, testRef("s1", 1, taken)) + processNext(t, tr, opUpload) // s1's upload fails, and s1 waits in Uploading to retry. + tr.CommitDurable(trackedActorUID, testRef("s2", 2, tr.clock.Now())) + + end, err := tr.BeginStart(t.Context(), trackedActorUID) + if err != nil { + t.Fatalf("BeginStart: %v", err) + } + processNext(t, tr, opUpload) + end() + if rec := readSnapshotState(trackedActorUID); rec == nil || rec.Uploading != nil { + t.Errorf("state file = %+v, want Uploading cleared", rec) + } + if _, err := os.Stat(s1Dir); err != nil { + t.Errorf("s1 after a worker cleared it during a Restore: stat = %v, want it kept", err) + } + }) +} + +// Uploading is saved to the snapshot state file, so an atelet restart mid-upload +// resumes s1 unless a newer snapshot was already durable before the restart. +func TestTrackerResumesUploadingAfterRestart(t *testing.T) { + t.Run("continues uploading across pause checkpoint", func(t *testing.T) { + useActorDir(t) + store := newFakeSnapshotStore() + store.setBlocking(true) + ctx, cancel := context.WithCancel(t.Context()) + before := New(ctx, store, newStoreTracker(t, nil).clock, WithUploadDelay(testUploadDelay), WithDetachDelay(testDetachDelay)) + before.Start() + + t0 := before.clock.Now().Add(-testUploadDelay) + before.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) + if got := waitStarted(t, store); got != "s1" { + t.Fatalf("before started = %q, want s1", got) + } + s2 := testRef("s2", 2, t0.Add(time.Minute)) + s2.Snapshot.Object = nil + before.CommitLocal(trackedActorUID, s2) + cancel() + if err := before.Wait(t.Context()); err != nil { + t.Fatalf("before.Wait: %v", err) + } + + store2 := newFakeSnapshotStore() + after := newStoreTracker(t, store2) + if got := uuidOf(after.State(trackedActorUID).GetUploading()); got != "s1" { + t.Fatalf("uploading after restart = %q, want s1 from state file", got) + } + after.Start() + if got := waitStarted(t, store2); got != "s1" { + t.Fatalf("after started = %q, want s1", got) + } + }) + + t.Run("continues uploading across uploadable local checkpoint", func(t *testing.T) { + useActorDir(t) + for _, name := range []string{"s1", "s2"} { + if err := os.MkdirAll(ateletpath.LocalSnapshotDir(trackedActorUID, name), 0o700); err != nil { + t.Fatal(err) + } + } + store := newFakeSnapshotStore() + store.setBlocking(true) + ctx, cancel := context.WithCancel(t.Context()) + before := New(ctx, store, newStoreTracker(t, nil).clock, WithUploadDelay(testUploadDelay), WithDetachDelay(testDetachDelay)) + before.Start() + + t0 := before.clock.Now().Add(-testUploadDelay) + before.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) + if got := waitStarted(t, store); got != "s1" { + t.Fatalf("before started = %q, want s1", got) + } + before.CommitLocal(trackedActorUID, testRef("s2", 2, t0.Add(time.Minute))) + cancel() + if err := before.Wait(t.Context()); err != nil { + t.Fatalf("before.Wait: %v", err) + } + + store2 := newFakeSnapshotStore() + store2.setBlocking(true) + after := newStoreTracker(t, store2) + if got := uuidOf(after.State(trackedActorUID).GetUploading()); got != "s1" { + t.Fatalf("uploading after restart = %q, want s1", got) + } + after.Start() + if got := waitStarted(t, store2); got != "s1" { + t.Fatalf("after started = %q, want s1 continued before s2", got) + } + done := opDone(t, after, trackedActorUID) + store2.release <- nil + waitClosed(t, done, "the resumed upload of s1") + + if _, err := os.Stat(ateletpath.LocalSnapshotDir(trackedActorUID, "s1")); !os.IsNotExist(err) { + t.Errorf("s1 after resumed upload: stat = %v, want it deleted", err) + } + st := after.State(trackedActorUID) + if uuidOf(st.GetLatestDurable()) != "s1" || st.GetUploading() != nil { + t.Errorf("state after s1 finished = %v, want s1 durable and uploading nil", st) + } + if got, want := timeOf(st.GetUploadDueTime()), after.clock.Now().Add(testUploadDelay); !got.Equal(want) { + t.Errorf("s2 upload_due_time = %v, want %v", got, want) + } + }) + + t.Run("drops uploading when newer snapshot was already durable", func(t *testing.T) { + useActorDir(t) + s1Dir := ateletpath.LocalSnapshotDir(trackedActorUID, "s1") + if err := os.MkdirAll(s1Dir, 0o700); err != nil { + t.Fatal(err) + } + store := newFakeSnapshotStore() + store.setBlocking(true) + ctx, cancel := context.WithCancel(t.Context()) + before := New(ctx, store, newStoreTracker(t, nil).clock, WithUploadDelay(testUploadDelay), WithDetachDelay(testDetachDelay)) + before.Start() + + t0 := before.clock.Now().Add(-testUploadDelay) + before.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) + if got := waitStarted(t, store); got != "s1" { + t.Fatalf("before started = %q, want s1", got) + } + before.CommitDurable(trackedActorUID, testRef("s2", 2, t0.Add(time.Minute))) + // Capture the state file as a hard crash mid-upload leaves it. + stateFile := ateletpath.ActorSnapshotStateFile(trackedActorUID) + crashedState, err := os.ReadFile(stateFile) + if err != nil { + t.Fatal(err) + } + cancel() + if err := before.Wait(t.Context()); err != nil { + t.Fatalf("before.Wait: %v", err) + } + if err := os.WriteFile(stateFile, crashedState, 0o600); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(s1Dir, 0o700); err != nil { + t.Fatal(err) + } + + store2 := newFakeSnapshotStore() + after := newStoreTracker(t, store2) + if got := after.State(trackedActorUID).GetUploading(); got != nil { + t.Errorf("uploading after restart = %v, want nil because s2 is durable", got) + } + setWorkers(after, 0, 0) + after.Start() + if got := after.queues[opDetach].Len(); got != 0 { + t.Errorf("detach queue len after restart = %d, want 0", got) + } + processNext(t, after, opUpload) + if got := store2.uploads(); len(got) != 0 { + t.Errorf("uploads after restart = %v, want none", got) + } + if _, err := os.Stat(s1Dir); !os.IsNotExist(err) { + t.Errorf("s1 after startup cleanup: stat = %v, want it deleted", err) + } + }) +} + +// TestTrackerUploadActsForTheActor checks that a background upload names the +// actor the snapshot was committed for, including an upload of Uploading that +// an atelet restart resumes from the snapshot state file. +func TestTrackerUploadActsForTheActor(t *testing.T) { + t.Run("latest local", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newTestTracker(t, store) + s1 := testRef("s1", 1, tr.clock.Now()) + s1.Actor = testActor + tr.CommitLocal(trackedActorUID, s1) + + uploadNow(t, tr, store, nil) + if got := store.actorOf("s1"); got != testActor { + t.Errorf("upload acted for %+v, want %+v", got, testActor) + } + }) + + t.Run("resumed after a restart", func(t *testing.T) { + useActorDir(t) + store := newFakeSnapshotStore() + store.setBlocking(true) + ctx, cancel := context.WithCancel(t.Context()) + before := New(ctx, store, newStoreTracker(t, nil).clock, WithUploadDelay(testUploadDelay), WithDetachDelay(testDetachDelay)) + before.Start() + + t0 := before.clock.Now().Add(-testUploadDelay) + s1 := testRef("s1", 1, t0) + s1.Actor = testActor + before.CommitLocal(trackedActorUID, s1) + if got := waitStarted(t, store); got != "s1" { + t.Fatalf("before started = %q, want s1", got) + } + // A pause snapshot no upload can make durable leaves s1 in Uploading, + // so the resumed upload reads its actor from there. + s2 := testRef("s2", 2, t0.Add(time.Minute)) + s2.Snapshot.Object = nil + before.CommitLocal(trackedActorUID, s2) + cancel() + if err := before.Wait(t.Context()); err != nil { + t.Fatalf("before.Wait: %v", err) + } + + store2 := newFakeSnapshotStore() + after := newStoreTracker(t, store2) + after.Start() + if got := waitStarted(t, store2); got != "s1" { + t.Fatalf("after started = %q, want s1", got) + } + if got := store2.actorOf("s1"); got != testActor { + t.Errorf("resumed upload acted for %+v, want %+v", got, testActor) + } + }) +} + +// A background upload that comes due while PromoteSnapshot holds the actor +// waits until the promote ends, then skips if the promote made the snapshot +// durable or runs if the promote failed. +func TestTrackerUploadWaitsForPromote(t *testing.T) { + t.Run("promote succeeds", func(t *testing.T) { + store := newFakeSnapshotStore() + tr := newIdleTestTracker(t, store) + s1 := testRef("s1", 1, tr.clock.Now()) + tr.CommitLocal(trackedActorUID, s1) + + durable, endPromote, err := tr.BeginPromote(t.Context(), trackedActorUID, s1) + if err != nil || durable { + t.Fatalf("BeginPromote = (%v, %v), want (false, nil)", durable, err) + } + stepTo(tr, tr.clock.Now().Add(testUploadDelay)) + processNext(t, tr, opUpload) + if got := store.uploads(); len(got) != 0 { + t.Fatalf("uploads during promote = %v, want none", got) + } + + s1.UploadTime = tr.clock.Now() + tr.CommitPromoted(trackedActorUID, s1) + endPromote() + if n := tr.queues[opUpload].Len(); n != 0 { + processNext(t, tr, opUpload) + } + if got := store.uploads(); len(got) != 0 { + t.Errorf("uploads after promote succeeded = %v, want none", got) + } + }) + + t.Run("promote fails", func(t *testing.T) { + store := newFakeSnapshotStore() + tr := newIdleTestTracker(t, store) + s1 := testRef("s1", 1, tr.clock.Now()) + tr.CommitLocal(trackedActorUID, s1) + + _, endPromote, err := tr.BeginPromote(t.Context(), trackedActorUID, s1) + if err != nil { + t.Fatalf("BeginPromote: %v", err) + } + stepTo(tr, tr.clock.Now().Add(testUploadDelay)) + processNext(t, tr, opUpload) + if got := store.uploads(); len(got) != 0 { + t.Fatalf("uploads during promote = %v, want none", got) + } + if n := tr.queues[opUpload].Len(); n != 0 { + t.Fatalf("queued uploads during promote = %d, want none: the promote's end queues it", n) + } + + endPromote() + processNext(t, tr, opUpload) + if got, want := store.uploads(), []string{"s1"}; !slices.Equal(got, want) { + t.Errorf("uploads after the failed promote ended = %v, want %v", got, want) + } + }) +} + +// An out-of-order CommitLocal with a lower generation than Uploading does not +// replace a failed Uploading target, reset its backoff, or delete Uploading's +// local copy when it finishes. +func TestTrackerOlderLocalDoesNotSupersedeUploading(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newTestTracker(t, store) + s2Dir := ateletpath.LocalSnapshotDir(trackedActorUID, "s2") + if err := os.MkdirAll(s2Dir, 0o700); err != nil { + t.Fatal(err) + } + t0 := tr.clock.Now() + tr.CommitLocal(trackedActorUID, testRef("s2", 2, t0)) + + // Fail s2 once, then record an older generation s1 as LatestLocal. + uploadNow(t, tr, store, errors.New("boom")) + retryAt := tr.clock.Now().Add(testRetryDelay) + st := tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0.Add(time.Minute))) + if got := uuidOf(st.GetUploading()); got != "s2" { + t.Fatalf("uploading after older CommitLocal = %q, want s2 kept", got) + } + if got := timeOf(st.GetUploadDueTime()); !got.Equal(retryAt) { + t.Errorf("upload_due_time after older CommitLocal = %v, want s2's retry at %v", got, retryAt) + } + + // Retry s2 and let it succeed. Because LatestLocal has a lower generation, + // s2 is not treated as superseded and its local directory stays intact. + uploadNow(t, tr, store, nil) + if got, want := store.uploads(), []string{"s2", "s2"}; !slices.Equal(got, want) { + t.Errorf("uploads = %v, want %v", got, want) + } + if _, err := os.Stat(s2Dir); err != nil { + t.Errorf("s2 local dir after upload: %v, want it kept", err) + } +} diff --git a/cmd/atelet/lifecycle_test.go b/cmd/atelet/lifecycle_test.go index 4d2f2e0abf..4df2da5f16 100644 --- a/cmd/atelet/lifecycle_test.go +++ b/cmd/atelet/lifecycle_test.go @@ -16,7 +16,6 @@ package main import ( "context" - "crypto/sha256" "errors" "fmt" "net" @@ -34,14 +33,18 @@ import ( "go.ate.dev/substrate/cmd/atelet/internal/snapshottracker" "go.ate.dev/substrate/cmd/atelet/internal/trustbundle" "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/hardware" "go.ate.dev/substrate/internal/nodepath" "go.ate.dev/substrate/internal/proto/ateletpb" "go.ate.dev/substrate/internal/proto/ateworkerpb" "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/api/v1alpha1" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/protobuf/proto" "k8s.io/utils/clock" + clocktesting "k8s.io/utils/clock/testing" ) // useTempNodeDirs roots atelet's on-node state in temp directories so a test @@ -79,6 +82,9 @@ type fakeWorker struct { // fidelities records the fidelity each checkpoint or restore arrived // with, by RPC name. fidelities map[string]ateworkerpb.SnapshotFidelity + // onCheckpoint, if non-nil, runs inside CheckpointWorkload after snapshot + // files are written. + onCheckpoint func() } func (f *fakeWorker) recordFidelity(rpc string, fidelity ateworkerpb.SnapshotFidelity) { @@ -115,6 +121,9 @@ func (f *fakeWorker) CheckpointWorkload(_ context.Context, req *ateworkerpb.Chec } names = append(names, name) } + if f.onCheckpoint != nil { + f.onCheckpoint() + } return &ateworkerpb.CheckpointWorkloadResponse{SnapshotFiles: names}, nil } @@ -211,26 +220,21 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity) { image := host + "/actor:v1" pinnedImage := pushTestImage(t, image, singleFileLayer(t, "bin/app", "app")) - // A single "runsc" asset served from a fake bucket: enough to exercise the - // content-addressed asset fetch without a gVisor release tarball. - runsc := []byte("runsc binary") + release := newGVisorTestRelease(t) s := &Atelet{ workerDialer: newWorkerDialer(1), imageCache: newImageVolumeStore(t), sandboxAssetCache: newTestAssetCache(t), - anonGCSClient: fakeObjectStorage{data: runsc}, + anonGCSClient: fakeObjectStorage{data: release.tarball}, systemInfoVolumes: newSystemInfoVolumeRefresher(nil, nil), - snapshots: snapshottracker.New(t.Context(), clock.RealClock{}), + snapshots: snapshottracker.New(t.Context(), nil, clock.RealClock{}), } sandboxAssets := &ateletpb.SandboxAssets{ SandboxClass: "gvisor", PauseImage: pinnedImage, Assets: map[string]*ateletpb.ArchAssets{ runtime.GOARCH: {Files: map[string]*ateletpb.AssetFile{ - runscAssetName: { - Url: "gs://test-bucket/runsc", - Sha256: fmt.Sprintf("%x", sha256.Sum256(runsc)), - }, + gvisorAssetName: release.proto(), }}, }, } @@ -238,6 +242,7 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity) { Containers: []*ateletpb.Container{{Name: "app", Image: pinnedImage, Command: []string{"/bin/app"}}}, } + advertiseRuntime(t, s, sandboxAssets) if _, err := s.Run(ctx, &ateletpb.RunRequest{ Atespace: atespace, ActorName: actorName, @@ -245,8 +250,8 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity) { ActorTemplateAtespace: "default", ActorTemplateName: "counter", WorkerPodUid: workerPodUID, - SandboxAssets: sandboxAssets, - Spec: spec, + SandboxAssets: sandboxAssets, SandboxRuntime: runtimeFor(t, sandboxAssets), + Spec: spec, }); err != nil { t.Fatalf("Run: %v", err) } @@ -272,7 +277,7 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity) { } // A paused actor stays resumable on this node, so it keeps its staged // assets, and with them its binaries in the cache. - if _, err := os.Stat(filepath.Join(ateletpath.SandboxAssetsDir(actorUID), runscAssetName)); err != nil { + if _, err := os.Stat(stagedRunsc(actorUID)); err != nil { t.Errorf("paused actor lost its staged runsc: %v", err) } st := resp.GetState() @@ -291,8 +296,8 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity) { ActorTemplateAtespace: "default", ActorTemplateName: "counter", WorkerPodUid: workerPodUID, - SandboxAssets: sandboxAssets, - Spec: spec, + SandboxAssets: sandboxAssets, SandboxRuntime: runtimeFor(t, sandboxAssets), + Spec: spec, Snapshot: &ateletpb.Snapshot{ SnapshotUuid: snapshotName, Generation: 1, @@ -304,7 +309,7 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity) { if got := worker.restored["checkpoint.img"]; got != "guest-memory" { t.Fatalf("restore staged %q for worker, want the pause snapshot's %q", got, "guest-memory") } - if _, err := os.Stat(filepath.Join(ateletpath.SandboxAssetsDir(actorUID), runscAssetName)); err != nil { + if _, err := os.Stat(stagedRunsc(actorUID)); err != nil { t.Errorf("resumed actor has no staged runsc: %v", err) } for _, rpc := range []string{"CheckpointWorkload", "RestoreWorkload"} { @@ -414,14 +419,14 @@ func TestRestoreUsesRequestSandboxAssets(t *testing.T) { checkpointPause := pushTestImage(t, host+"/pause:v1", singleFileLayer(t, "pause", "pause-v1")) restorePause := pushTestImage(t, host+"/pause:v2", singleFileLayer(t, "pause", "pause-v2")) - runsc := []byte("runsc binary") + release := newGVisorTestRelease(t) s := &Atelet{ workerDialer: newWorkerDialer(1), imageCache: newImageVolumeStore(t), sandboxAssetCache: newTestAssetCache(t), - anonGCSClient: fakeObjectStorage{data: runsc}, + anonGCSClient: fakeObjectStorage{data: release.tarball}, systemInfoVolumes: newSystemInfoVolumeRefresher(nil, nil), - snapshots: snapshottracker.New(t.Context(), clock.RealClock{}), + snapshots: snapshottracker.New(t.Context(), nil, clock.RealClock{}), } assetsWithPause := func(pause string) *ateletpb.SandboxAssets { return &ateletpb.SandboxAssets{ @@ -429,10 +434,7 @@ func TestRestoreUsesRequestSandboxAssets(t *testing.T) { PauseImage: pause, Assets: map[string]*ateletpb.ArchAssets{ runtime.GOARCH: {Files: map[string]*ateletpb.AssetFile{ - runscAssetName: { - Url: "gs://test-bucket/runsc", - Sha256: fmt.Sprintf("%x", sha256.Sum256(runsc)), - }, + gvisorAssetName: release.proto(), }}, }, } @@ -441,6 +443,7 @@ func TestRestoreUsesRequestSandboxAssets(t *testing.T) { Containers: []*ateletpb.Container{{Name: "app", Image: pinnedImage, Command: []string{"/bin/app"}}}, } + advertiseRuntime(t, s, assetsWithPause(checkpointPause)) if _, err := s.Run(ctx, &ateletpb.RunRequest{ Atespace: atespace, ActorName: actorName, @@ -448,8 +451,8 @@ func TestRestoreUsesRequestSandboxAssets(t *testing.T) { ActorTemplateAtespace: "default", ActorTemplateName: "counter", WorkerPodUid: workerPodUID, - SandboxAssets: assetsWithPause(checkpointPause), - Spec: spec, + SandboxAssets: assetsWithPause(checkpointPause), SandboxRuntime: runtimeFor(t, assetsWithPause(checkpointPause)), + Spec: spec, }); err != nil { t.Fatalf("Run: %v", err) } @@ -480,6 +483,7 @@ func TestRestoreUsesRequestSandboxAssets(t *testing.T) { t.Fatalf("manifest pause image = %q, want %q", manifestRec.PauseImage, checkpointPause) } + advertiseRuntime(t, s, assetsWithPause(restorePause)) if _, err := s.Restore(ctx, &ateletpb.RestoreRequest{ Atespace: atespace, ActorName: actorName, @@ -487,8 +491,8 @@ func TestRestoreUsesRequestSandboxAssets(t *testing.T) { ActorTemplateAtespace: "default", ActorTemplateName: "counter", WorkerPodUid: workerPodUID, - SandboxAssets: assetsWithPause(restorePause), - Spec: spec, + SandboxAssets: assetsWithPause(restorePause), SandboxRuntime: runtimeFor(t, assetsWithPause(restorePause)), + Spec: spec, Snapshot: &ateletpb.Snapshot{ SnapshotUuid: snapshotName, Generation: 1, @@ -545,12 +549,11 @@ func TestActivationFailureBeforeRegistration(t *testing.T) { t.Fatalf("initial projection = %q, %v; want cert A", got, err) } - content := []byte("runsc binary") - assetHash := fmt.Sprintf("%x", sha256.Sum256(content)) + release := newGVisorTestRelease(t) if tc.restore && tc.failure != "manifest" { writeLocalSnapshot(t, ateletpath.LocalSnapshotDir(actorUID, snapshotName), sandboxAssetsRecord{ SandboxClass: "gvisor", PauseImage: testPauseImage, - Assets: map[string]assetEntry{"runsc": {URL: "gs://test-bucket/runsc", SHA256: assetHash}}, + Assets: map[string]assetEntry{gvisorAssetName: release.entry}, SnapshotFiles: []string{"checkpoint.img"}, }, map[string]string{"checkpoint.img": "guest-memory"}) } @@ -573,28 +576,29 @@ func TestActivationFailureBeforeRegistration(t *testing.T) { // The live activation's staged binary: a Run that fails before // it swaps its own set in must leave it, and a Run that fails // after must still leave nothing half-staged behind. - liveAsset := filepath.Join(ateletpath.SandboxAssetsDir(actorUID), "runsc") + liveAsset := stagedRunsc(actorUID) if err := os.MkdirAll(filepath.Dir(liveAsset), 0o700); err != nil { t.Fatal(err) } if err := os.WriteFile(liveAsset, []byte("live runsc"), 0o555); err != nil { t.Fatal(err) } - storage := fakeObjectStorage{data: content} + storage := fakeObjectStorage{data: release.tarball} if tc.failure == "asset" { storage.err = fmt.Errorf("asset unavailable") } - s := &Atelet{anonGCSClient: storage, systemInfoVolumes: refresher, snapshots: snapshottracker.New(t.Context(), clock.RealClock{}), sandboxAssetCache: newTestAssetCache(t)} + s := &Atelet{anonGCSClient: storage, systemInfoVolumes: refresher, snapshots: snapshottracker.New(t.Context(), nil, clock.RealClock{}), sandboxAssetCache: newTestAssetCache(t)} assets := &ateletpb.SandboxAssets{ SandboxClass: "gvisor", PauseImage: testPauseImage, Assets: map[string]*ateletpb.ArchAssets{runtime.GOARCH: {Files: map[string]*ateletpb.AssetFile{ - runscAssetName: {Url: "gs://test-bucket/runsc", Sha256: assetHash}, + gvisorAssetName: release.proto(), }}}, } + advertiseRuntime(t, s, assets) if tc.restore { _, err = s.Restore(ctx, &ateletpb.RestoreRequest{ Atespace: "team-a", ActorName: "actor-1", ActorUid: actorUID, WorkerPodUid: "5f1c0d2e-3a4b-4c5d-8e9f-0a1b2c3d4e5f", - SandboxAssets: assets, Spec: spec, + SandboxAssets: assets, SandboxRuntime: runtimeFor(t, assets), Spec: spec, Snapshot: &ateletpb.Snapshot{ SnapshotUuid: snapshotName, Generation: 1, @@ -604,7 +608,7 @@ func TestActivationFailureBeforeRegistration(t *testing.T) { } else { _, err = s.Run(ctx, &ateletpb.RunRequest{ Atespace: "team-a", ActorName: "actor-1", ActorUid: actorUID, WorkerPodUid: "5f1c0d2e-3a4b-4c5d-8e9f-0a1b2c3d4e5f", - SandboxAssets: assets, Spec: spec, + SandboxAssets: assets, SandboxRuntime: runtimeFor(t, assets), Spec: spec, }) } if err == nil || !strings.Contains(err.Error(), tc.wantErr) { @@ -636,7 +640,7 @@ func TestActivationFailureBeforeRegistration(t *testing.T) { t.Fatal(err) } for _, e := range entries { - if got, err := os.ReadFile(liveAsset); e.Name() != "runsc" || err != nil || string(got) != "live runsc" { + if got, err := os.ReadFile(liveAsset); e.Name() != gvisorAssetName || err != nil || string(got) != "live runsc" { t.Errorf("failed activation left %q behind in the assets dir", e.Name()) } } @@ -671,15 +675,14 @@ func TestRunFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { store := newCTBStore(t) store.set(t, string(testCertPEM(t))) refresher := newSystemInfoVolumeRefresher(trustbundle.NewSource(store.lister.Get, nil), nil) - content := []byte("runsc binary") - assetHash := fmt.Sprintf("%x", sha256.Sum256(content)) + release := newGVisorTestRelease(t) const actorUID = "0f6c2b0e-1b7e-4b3f-9a6a-1d2c3e4f5a62" s := &Atelet{ sandboxAssetCache: newTestAssetCache(t), - anonGCSClient: fakeObjectStorage{data: content}, + anonGCSClient: fakeObjectStorage{data: release.tarball}, imageCache: newImageVolumeStore(t), systemInfoVolumes: refresher, - snapshots: snapshottracker.New(t.Context(), clock.RealClock{}), + snapshots: snapshottracker.New(t.Context(), nil, clock.RealClock{}), } spec := &ateletpb.WorkloadSpec{ Volumes: []*ateletpb.Volume{{ @@ -687,18 +690,20 @@ func TestRunFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { SystemInfo: trustVolumeSpec("ca.pem"), }}, } + assets := &ateletpb.SandboxAssets{ + SandboxClass: "gvisor", + PauseImage: missingPauseImage(t), + Assets: map[string]*ateletpb.ArchAssets{runtime.GOARCH: {Files: map[string]*ateletpb.AssetFile{ + gvisorAssetName: release.proto(), + }}}, + } + advertiseRuntime(t, s, assets) _, err := s.Run(t.Context(), &ateletpb.RunRequest{ - Atespace: "team-a", - ActorName: "actor-run", - ActorUid: actorUID, - WorkerPodUid: "5f1c0d2e-3a4b-4c5d-8e9f-0a1b2c3d4e5f", - SandboxAssets: &ateletpb.SandboxAssets{ - SandboxClass: "gvisor", - PauseImage: missingPauseImage(t), - Assets: map[string]*ateletpb.ArchAssets{runtime.GOARCH: {Files: map[string]*ateletpb.AssetFile{ - runscAssetName: {Url: "gs://test-bucket/runsc", Sha256: assetHash}, - }}}, - }, + Atespace: "team-a", + ActorName: "actor-run", + ActorUid: actorUID, + WorkerPodUid: "5f1c0d2e-3a4b-4c5d-8e9f-0a1b2c3d4e5f", + SandboxAssets: assets, SandboxRuntime: runtimeFor(t, assets), Spec: spec, }) if err == nil || !strings.Contains(err.Error(), "while creating pause OCI bundle") { @@ -730,12 +735,11 @@ func TestRestoreFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { store := newCTBStore(t) store.set(t, string(testCertPEM(t))) refresher := newSystemInfoVolumeRefresher(trustbundle.NewSource(store.lister.Get, nil), nil) - content := []byte("runsc binary") - assetHash := fmt.Sprintf("%x", sha256.Sum256(content)) + release := newGVisorTestRelease(t) writeLocalSnapshot(t, ateletpath.LocalSnapshotDir(actorUID, snapshotName), sandboxAssetsRecord{ SandboxClass: "gvisor", PauseImage: pauseImage, - Assets: map[string]assetEntry{"runsc": {URL: "gs://test-bucket/runsc", SHA256: assetHash}}, + Assets: map[string]assetEntry{gvisorAssetName: release.entry}, SnapshotFiles: []string{"checkpoint.img"}, }, map[string]string{"checkpoint.img": "guest-memory"}) @@ -745,23 +749,25 @@ func TestRestoreFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { }}} s := &Atelet{ sandboxAssetCache: newTestAssetCache(t), - anonGCSClient: fakeObjectStorage{data: content}, + anonGCSClient: fakeObjectStorage{data: release.tarball}, imageCache: newImageVolumeStore(t), systemInfoVolumes: refresher, - snapshots: snapshottracker.New(t.Context(), clock.RealClock{}), + snapshots: snapshottracker.New(t.Context(), nil, clock.RealClock{}), } + assets := &ateletpb.SandboxAssets{ + SandboxClass: "gvisor", + PauseImage: pauseImage, + Assets: map[string]*ateletpb.ArchAssets{runtime.GOARCH: {Files: map[string]*ateletpb.AssetFile{ + gvisorAssetName: release.proto(), + }}}, + } + advertiseRuntime(t, s, assets) _, err := s.Restore(t.Context(), &ateletpb.RestoreRequest{ - Atespace: "team-a", - ActorName: "actor-restore-after", - ActorUid: actorUID, - WorkerPodUid: "5f1c0d2e-3a4b-4c5d-8e9f-0a1b2c3d4e5f", - SandboxAssets: &ateletpb.SandboxAssets{ - SandboxClass: "gvisor", - PauseImage: pauseImage, - Assets: map[string]*ateletpb.ArchAssets{runtime.GOARCH: {Files: map[string]*ateletpb.AssetFile{ - runscAssetName: {Url: "gs://test-bucket/runsc", Sha256: assetHash}, - }}}, - }, + Atespace: "team-a", + ActorName: "actor-restore-after", + ActorUid: actorUID, + WorkerPodUid: "5f1c0d2e-3a4b-4c5d-8e9f-0a1b2c3d4e5f", + SandboxAssets: assets, SandboxRuntime: runtimeFor(t, assets), Spec: spec, Snapshot: &ateletpb.Snapshot{ SnapshotUuid: snapshotName, @@ -800,17 +806,13 @@ func TestRestorePrefersLocalAndFallsBackToObjectStorage(t *testing.T) { host := imageVolumeTestRegistry(t) pinnedImage := pushTestImage(t, host+"/actor:v1", singleFileLayer(t, "bin/app", "app")) - runsc := []byte("runsc binary") - assetHash := fmt.Sprintf("%x", sha256.Sum256(runsc)) + release := newGVisorTestRelease(t) sandboxAssets := &ateletpb.SandboxAssets{ SandboxClass: "gvisor", PauseImage: pinnedImage, Assets: map[string]*ateletpb.ArchAssets{ runtime.GOARCH: {Files: map[string]*ateletpb.AssetFile{ - runscAssetName: { - Url: "gs://test-bucket/runsc", - Sha256: assetHash, - }, + gvisorAssetName: release.proto(), }}, }, } @@ -820,7 +822,7 @@ func TestRestorePrefersLocalAndFallsBackToObjectStorage(t *testing.T) { manifestRec := sandboxAssetsRecord{ SandboxClass: "gvisor", PauseImage: pinnedImage, - Assets: map[string]assetEntry{runscAssetName: {URL: "gs://test-bucket/runsc", SHA256: assetHash}}, + Assets: map[string]assetEntry{gvisorAssetName: release.entry}, SnapshotFiles: []string{"checkpoint.img"}, } @@ -830,7 +832,7 @@ func TestRestorePrefersLocalAndFallsBackToObjectStorage(t *testing.T) { s := newPluginHerder(t, objStore) s.workerDialer = newWorkerDialer(1) s.imageCache = newImageVolumeStore(t) - s.anonGCSClient = fakeObjectStorage{data: runsc} + s.anonGCSClient = fakeObjectStorage{data: release.tarball} s.systemInfoVolumes = newSystemInfoVolumeRefresher(nil, nil) uri, err := resources.ParseSnapshotURI(pausedSnapshotURI) @@ -839,7 +841,7 @@ func TestRestorePrefersLocalAndFallsBackToObjectStorage(t *testing.T) { } remoteSrcDir := t.TempDir() writeLocalSnapshot(t, remoteSrcDir, manifestRec, map[string]string{"checkpoint.img": "object-storage-memory"}) - if err := s.uploadSnapshot(t.Context(), uri, remoteSrcDir, &manifestRec, "default", "counter"); err != nil { + if err := s.uploadSnapshot(t.Context(), uri, remoteSrcDir, &manifestRec); err != nil { t.Fatalf("uploadSnapshot: %v", err) } @@ -892,6 +894,7 @@ func TestRestorePrefersLocalAndFallsBackToObjectStorage(t *testing.T) { useTempNodeDirs(t) localDir := ateletpath.LocalSnapshotDir(actorUID, snapshotName) tc.prepLocal(t, localDir) + advertiseRuntime(t, s, sandboxAssets) if _, err := s.Restore(t.Context(), &ateletpb.RestoreRequest{ Atespace: atespace, @@ -900,8 +903,8 @@ func TestRestorePrefersLocalAndFallsBackToObjectStorage(t *testing.T) { ActorTemplateAtespace: "default", ActorTemplateName: "counter", WorkerPodUid: workerPodUID, - SandboxAssets: sandboxAssets, - Spec: spec, + SandboxAssets: sandboxAssets, SandboxRuntime: runtimeFor(t, sandboxAssets), + Spec: spec, Snapshot: &ateletpb.Snapshot{ SnapshotUuid: snapshotName, Generation: 1, @@ -945,11 +948,11 @@ func TestRestoreWithoutUsableLocalCopy(t *testing.T) { actorUID = "0f6c2b0e-1b7e-4b3f-9a6a-1d2c3e4f5a65" snapshotName = "pause-snap-1" ) - assetHash := fmt.Sprintf("%x", sha256.Sum256([]byte("runsc binary"))) + release := newGVisorTestRelease(t) assets := &ateletpb.SandboxAssets{ SandboxClass: "gvisor", PauseImage: testPauseImage, Assets: map[string]*ateletpb.ArchAssets{runtime.GOARCH: {Files: map[string]*ateletpb.AssetFile{ - runscAssetName: {Url: "gs://test-bucket/runsc", Sha256: assetHash}, + gvisorAssetName: release.proto(), }}}, } for _, tc := range []struct { @@ -985,11 +988,12 @@ func TestRestoreWithoutUsableLocalCopy(t *testing.T) { // The object store is empty: nothing was ever uploaded. s := newPluginHerder(t, &recordingObjectStorage{}) s.systemInfoVolumes = newSystemInfoVolumeRefresher(nil, nil) + advertiseRuntime(t, s, assets) _, err := s.Restore(t.Context(), &ateletpb.RestoreRequest{ Atespace: "team-a", ActorName: "actor-1", ActorUid: actorUID, WorkerPodUid: "5f1c0d2e-3a4b-4c5d-8e9f-0a1b2c3d4e5f", - SandboxAssets: assets, - Spec: &ateletpb.WorkloadSpec{}, + SandboxAssets: assets, SandboxRuntime: runtimeFor(t, assets), + Spec: &ateletpb.WorkloadSpec{}, Snapshot: &ateletpb.Snapshot{ SnapshotUuid: snapshotName, Generation: 1, @@ -1044,7 +1048,7 @@ func TestTerminateWithoutWorkerPodUID(t *testing.T) { workerDialer: newWorkerDialer(1), sandboxAssetCache: newTestAssetCache(t), systemInfoVolumes: newSystemInfoVolumeRefresher(nil, nil), - snapshots: snapshottracker.New(t.Context(), clock.RealClock{}), + snapshots: snapshottracker.New(t.Context(), nil, clock.RealClock{}), } spec := &ateletpb.WorkloadSpec{ Containers: []*ateletpb.Container{{Name: "app", Image: "example.com/app@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, @@ -1103,22 +1107,19 @@ func runPluginActor(t *testing.T) *pluginActor { image := host + "/actor:v1" pinnedImage := pushTestImage(t, image, singleFileLayer(t, "bin/app", "app")) - runsc := []byte("runsc binary") + release := newGVisorTestRelease(t) a.s = newPluginHerder(t, a.store) a.s.workerDialer = newWorkerDialer(1) a.s.imageCache = newImageVolumeStore(t) a.s.sandboxAssetCache = newTestAssetCache(t) - a.s.anonGCSClient = fakeObjectStorage{data: runsc} + a.s.anonGCSClient = fakeObjectStorage{data: release.tarball} a.s.systemInfoVolumes = newSystemInfoVolumeRefresher(nil, nil) a.assets = &ateletpb.SandboxAssets{ SandboxClass: "gvisor", PauseImage: pinnedImage, Assets: map[string]*ateletpb.ArchAssets{ runtime.GOARCH: {Files: map[string]*ateletpb.AssetFile{ - runscAssetName: { - Url: "gs://test-bucket/runsc", - Sha256: fmt.Sprintf("%x", sha256.Sum256(runsc)), - }, + gvisorAssetName: release.proto(), }}, }, } @@ -1126,6 +1127,7 @@ func runPluginActor(t *testing.T) *pluginActor { Containers: []*ateletpb.Container{{Name: "app", Image: pinnedImage, Command: []string{"/bin/app"}}}, } + advertiseRuntime(t, a.s, a.assets) if _, err := a.s.Run(t.Context(), &ateletpb.RunRequest{ Atespace: "ate-demo", ActorName: "counter", @@ -1133,8 +1135,8 @@ func runPluginActor(t *testing.T) *pluginActor { ActorTemplateAtespace: "default", ActorTemplateName: "counter", WorkerPodUid: pluginWorkerPodUID, - SandboxAssets: a.assets, - Spec: a.spec, + SandboxAssets: a.assets, SandboxRuntime: runtimeFor(t, a.assets), + Spec: a.spec, }); err != nil { t.Fatalf("Run: %v", err) } @@ -1173,9 +1175,9 @@ func (a *pluginActor) restoreLocal(t *testing.T, name string, generation int32) ActorTemplateAtespace: "default", ActorTemplateName: "counter", WorkerPodUid: pluginWorkerPodUID, - SandboxAssets: a.assets, - Spec: a.spec, - Snapshot: &ateletpb.Snapshot{SnapshotUuid: name, Generation: generation, Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY}, + SandboxAssets: a.assets, SandboxRuntime: runtimeFor(t, a.assets), + Spec: a.spec, + Snapshot: &ateletpb.Snapshot{SnapshotUuid: name, Generation: generation, Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY}, }); err != nil { t.Fatalf("Restore: %v", err) } @@ -1204,10 +1206,10 @@ func manifestKey(name string) string { return "test-bucket/atespaces/ate-demo/actors/" + pluginActorUID + "/snapshots/" + name + "/" + sandboxManifestName } -// TestImmediateCheckpointKeepsNoLocalCopy suspends a running actor: the -// checkpoint uploads the snapshot before it returns and leaves none on the -// node. -func TestImmediateCheckpointKeepsNoLocalCopy(t *testing.T) { +// TestImmediateCheckpointKeepsLocalCopy suspends a running actor: the +// checkpoint uploads the snapshot before it returns and keeps a local copy on +// the node. +func TestImmediateCheckpointKeepsLocalCopy(t *testing.T) { a := runPluginActor(t) snap := a.snapshot("suspend-snap-1", 1) @@ -1218,44 +1220,105 @@ func TestImmediateCheckpointKeepsNoLocalCopy(t *testing.T) { if !slices.Contains(a.store.keys(), manifestKey("suspend-snap-1")) { t.Errorf("objects = %v, want the snapshot uploaded", a.store.keys()) } - if _, err := os.Stat(ateletpath.LocalCheckpointsDir(pluginActorUID)); !os.IsNotExist(err) { - t.Errorf("local checkpoints after an immediate checkpoint: stat = %v, want none", err) + if _, err := os.Stat(ateletpath.LocalSnapshotDir(pluginActorUID, "suspend-snap-1")); err != nil { + t.Errorf("local snapshot after an immediate checkpoint: stat = %v, want it kept", err) } st := resp.GetState() - if st.GetLatestLocal() != nil { - t.Errorf("state = %v, want no local snapshot", st) + if got := st.GetLatestLocal(); !proto.Equal(got.GetSnapshot(), snap) || got.GetLocalCommitTime() == nil || got.GetDurableCompleteTime() != nil { + t.Errorf("latest_local = %v, want %v with its commit time and no upload time", got, snap) } - if got := st.GetLatestDurable(); !proto.Equal(got.GetSnapshot(), snap) || got.GetDurableCompleteTime() == nil { - t.Errorf("latest_durable = %v, want %v with its upload time", got, snap) + if got := st.GetLatestDurable(); !proto.Equal(got.GetSnapshot(), snap) || got.GetLocalCommitTime() == nil || got.GetDurableCompleteTime() == nil { + t.Errorf("latest_durable = %v, want %v with its commit and upload times", got, snap) } } -// TestFailedCheckpointForgetsLocalSnapshot suspends a resumed actor whose -// upload fails after the checkpoint has deleted the pause snapshot: the -// tracker no longer reports the deleted snapshot. -func TestFailedCheckpointForgetsLocalSnapshot(t *testing.T) { +// TestImmediateCheckpointWithoutLocalCopy suspends a running actor whose +// snapshot cannot be stored locally: the snapshot is durable, so the +// checkpoint still succeeds, and reports no local snapshot. +func TestImmediateCheckpointWithoutLocalCopy(t *testing.T) { a := runPluginActor(t) - if _, err := a.checkpoint(t, a.snapshot("pause-snap-1", 1), ateletpb.UploadMode_UPLOAD_MODE_DEFERRED); err != nil { - t.Fatalf("Checkpoint (pause): %v", err) + snap := a.snapshot("suspend-snap-1", 1) + // A file where the local checkpoint directory belongs fails the move. + if err := os.WriteFile(ateletpath.LocalCheckpointsDir(pluginActorUID), nil, 0o600); err != nil { + t.Fatal(err) } - a.restoreLocal(t, "pause-snap-1", 1) - a.store.putErr = errors.New("object store unavailable") - if _, err := a.checkpoint(t, a.snapshot("suspend-snap-2", 2), ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE); err == nil { - t.Fatal("Checkpoint (suspend) succeeded, want the upload to fail") + resp, err := a.checkpoint(t, snap, ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE) + if err != nil { + t.Fatalf("Checkpoint: %v", err) } - if _, err := os.Stat(ateletpath.LocalSnapshotDir(pluginActorUID, "pause-snap-1")); !os.IsNotExist(err) { - t.Fatalf("pause snapshot after the failed suspend: stat = %v, want it deleted", err) + if !slices.Contains(a.store.keys(), manifestKey("suspend-snap-1")) { + t.Errorf("objects = %v, want the snapshot uploaded", a.store.keys()) } - if st := a.s.snapshots.State(pluginActorUID); st.GetLatestLocal() != nil || st.GetLatestDurable() != nil { - t.Errorf("state = %v, want neither a local nor a durable snapshot", st) + st := resp.GetState() + if st.GetLatestLocal() != nil { + t.Errorf("latest_local = %v, want unset without a local copy", st.GetLatestLocal()) + } + if got := st.GetLatestDurable(); !proto.Equal(got.GetSnapshot(), snap) || got.GetDurableCompleteTime() == nil { + t.Errorf("latest_durable = %v, want %v with its upload time", got, snap) + } + if got := a.s.snapshots.State(pluginActorUID); !proto.Equal(got, st) { + t.Errorf("tracked state = %v, want %v, as Checkpoint returned", got, st) + } +} + +// TestFailedCheckpointKeepsLocalSnapshot checkpoints a resumed actor again, and +// fails before the new snapshot commits: the pause snapshot stays on disk and +// recorded as the actor's latest local snapshot, so the actor can still be +// restored from it. +func TestFailedCheckpointKeepsLocalSnapshot(t *testing.T) { + for _, tc := range []struct { + name string + snap string + mode ateletpb.UploadMode + fail func(t *testing.T, a *pluginActor) + }{ + { + name: "suspend upload fails", + snap: "suspend-snap-2", + mode: ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE, + fail: func(t *testing.T, a *pluginActor) { + a.store.putErr = errors.New("object store unavailable") + }, + }, + { + name: "pause move fails", + snap: "pause-snap-2", + mode: ateletpb.UploadMode_UPLOAD_MODE_DEFERRED, + fail: func(t *testing.T, a *pluginActor) { + // A file where the new snapshot's directory belongs fails the move. + if err := os.WriteFile(ateletpath.LocalSnapshotDir(pluginActorUID, "pause-snap-2"), nil, 0o600); err != nil { + t.Fatal(err) + } + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + a := runPluginActor(t) + if _, err := a.checkpoint(t, a.snapshot("pause-snap-1", 1), ateletpb.UploadMode_UPLOAD_MODE_DEFERRED); err != nil { + t.Fatalf("Checkpoint (pause): %v", err) + } + a.restoreLocal(t, "pause-snap-1", 1) + + tc.fail(t, a) + if _, err := a.checkpoint(t, a.snapshot(tc.snap, 2), tc.mode); err == nil { + t.Fatal("Checkpoint succeeded, want it to fail") + } + if _, err := os.Stat(ateletpath.LocalSnapshotDir(pluginActorUID, "pause-snap-1")); err != nil { + t.Errorf("pause snapshot after the failed checkpoint: stat = %v, want it kept", err) + } + if got := a.s.snapshots.State(pluginActorUID).GetLatestLocal().GetSnapshot().GetSnapshotUuid(); got != "pause-snap-1" { + t.Errorf("latest_local = %q, want pause-snap-1", got) + } + a.restoreLocal(t, "pause-snap-1", 1) + }) } } // TestPromoteSnapshotRecordsUpload suspends a paused actor: the promote // records the uploaded snapshot as the latest durable one, with the pause's -// commit time, and the deleted local copy is no longer reported. A retry, -// which finds the snapshot already uploaded, reports the same state. +// commit time, and keeps the local copy on the node. A retry reports the same +// state. func TestPromoteSnapshotRecordsUpload(t *testing.T) { a := runPluginActor(t) snap := a.snapshot("pause-snap-1", 1) @@ -1266,8 +1329,11 @@ func TestPromoteSnapshotRecordsUpload(t *testing.T) { commitTime := resp.GetState().GetLatestLocal().GetLocalCommitTime() st := a.promote(t, snap) - if st.GetLatestLocal() != nil { - t.Errorf("latest_local = %v after the promote deleted it, want unset", st.GetLatestLocal()) + if _, err := os.Stat(ateletpath.LocalSnapshotDir(pluginActorUID, "pause-snap-1")); err != nil { + t.Errorf("local snapshot after promote: stat = %v, want it kept", err) + } + if got := st.GetLatestLocal(); !proto.Equal(got.GetSnapshot(), snap) || !proto.Equal(got.GetLocalCommitTime(), commitTime) || got.GetDurableCompleteTime() != nil { + t.Errorf("latest_local = %v, want %v committed at %v, with no upload time", got, snap, commitTime) } if got := st.GetLatestDurable(); !proto.Equal(got.GetSnapshot(), snap) || !proto.Equal(got.GetLocalCommitTime(), commitTime) || got.GetDurableCompleteTime() == nil { t.Errorf("latest_durable = %v, want %v committed at %v, with its upload time", got, snap, commitTime) @@ -1276,9 +1342,23 @@ func TestPromoteSnapshotRecordsUpload(t *testing.T) { t.Errorf("tracked state = %v, want %v, as PromoteSnapshot returned", got, st) } + a.store.putErr = errors.New("retried promote should not upload") if got := a.promote(t, snap); !proto.Equal(got, st) { t.Errorf("state after a retried promote = %v, want %v", got, st) } + + mismatch := proto.CloneOf(snap) + mismatch.Generation = snap.GetGeneration() + 1 + if _, err := a.s.PromoteSnapshot(t.Context(), &ateletpb.PromoteSnapshotRequest{ + Atespace: "ate-demo", + ActorName: "counter", + ActorUid: pluginActorUID, + ActorTemplateAtespace: "default", + ActorTemplateName: "counter", + Snapshot: mismatch, + }); apierror.Code(err) != codes.FailedPrecondition { + t.Errorf("PromoteSnapshot with mismatched generation = %v, want FailedPrecondition", err) + } } // TestConcurrentRunsShareOneAssetDownload runs two actors that need the same @@ -1294,21 +1374,22 @@ func TestConcurrentRunsShareOneAssetDownload(t *testing.T) { host := imageVolumeTestRegistry(t) pinnedImage := pushTestImage(t, host+"/actor:v1", singleFileLayer(t, "bin/app", "app")) - runsc := []byte("runsc binary") + release := newGVisorTestRelease(t) var gets atomic.Int32 s := &Atelet{ workerDialer: newWorkerDialer(2), imageCache: newImageVolumeStore(t), sandboxAssetCache: newTestAssetCache(t), - anonGCSClient: fakeObjectStorage{data: runsc, gets: &gets, delay: 200 * time.Millisecond}, + anonGCSClient: fakeObjectStorage{data: release.tarball, gets: &gets, delay: 200 * time.Millisecond}, systemInfoVolumes: newSystemInfoVolumeRefresher(nil, nil), + snapshots: snapshottracker.New(t.Context(), nil, clock.RealClock{}), } sandboxAssets := &ateletpb.SandboxAssets{ SandboxClass: "gvisor", PauseImage: pinnedImage, Assets: map[string]*ateletpb.ArchAssets{ runtime.GOARCH: {Files: map[string]*ateletpb.AssetFile{ - runscAssetName: {Url: "gs://test-bucket/runsc", Sha256: fmt.Sprintf("%x", sha256.Sum256(runsc))}, + gvisorAssetName: release.proto(), }}, }, } @@ -1316,6 +1397,7 @@ func TestConcurrentRunsShareOneAssetDownload(t *testing.T) { Containers: []*ateletpb.Container{{Name: "app", Image: pinnedImage, Command: []string{"/bin/app"}}}, } + advertiseRuntime(t, s, sandboxAssets) actorUIDs := []string{"01234567-89ab-cdef-0123-456789abcde1", "01234567-89ab-cdef-0123-456789abcde2"} var wg sync.WaitGroup errs := make([]error, len(actorUIDs)) @@ -1328,8 +1410,8 @@ func TestConcurrentRunsShareOneAssetDownload(t *testing.T) { ActorTemplateAtespace: "default", ActorTemplateName: "counter", WorkerPodUid: "5f1c0d2e-3a4b-4c5d-8e9f-0a1b2c3d4e5f", - SandboxAssets: sandboxAssets, - Spec: spec, + SandboxAssets: sandboxAssets, SandboxRuntime: runtimeFor(t, sandboxAssets), + Spec: spec, }) }) } @@ -1345,10 +1427,10 @@ func TestConcurrentRunsShareOneAssetDownload(t *testing.T) { var infos []os.FileInfo for _, uid := range actorUIDs { - staged := filepath.Join(ateletpath.SandboxAssetsDir(uid), runscAssetName) + staged := stagedRunsc(uid) got, err := os.ReadFile(staged) - if err != nil || string(got) != string(runsc) { - t.Fatalf("actor %s staged runsc = %q, %v; want the asset bytes", uid, got, err) + if err != nil || string(got) != "runsc bytes" { + t.Fatalf("actor %s staged runsc = %q, %v; want the release's runsc", uid, got, err) } fi, err := os.Stat(staged) if err != nil { @@ -1374,28 +1456,29 @@ func TestCommitSandboxAssetsChecksPathsBeforeMoving(t *testing.T) { useTempNodeDirs(t) const actorUID = "01234567-89ab-cdef-0123-456789abcdef" staging, final := ateletpath.SandboxAssetsStagingDir(actorUID), ateletpath.SandboxAssetsDir(actorUID) - for _, dir := range []string{staging, final} { + runsc := filepath.Join(gvisorAssetName, "runsc") + for _, dir := range []string{filepath.Join(staging, gvisorAssetName), filepath.Join(final, gvisorAssetName)} { if err := os.MkdirAll(dir, 0o700); err != nil { t.Fatal(err) } } - if err := os.WriteFile(filepath.Join(staging, "runsc"), []byte("new"), 0o555); err != nil { + if err := os.WriteFile(filepath.Join(staging, runsc), []byte("new"), 0o555); err != nil { t.Fatal(err) } - if err := os.WriteFile(filepath.Join(final, "runsc"), []byte("previous"), 0o555); err != nil { + if err := os.WriteFile(filepath.Join(final, runsc), []byte("previous"), 0o555); err != nil { t.Fatal(err) } - paths := map[string]string{runscAssetName: filepath.Join(staging, "runsc"), "stray": "/elsewhere/kernel"} + paths := map[string]string{gvisorAssetName: filepath.Join(staging, runsc), "stray": "/elsewhere/kernel"} if err := commitSandboxAssets(actorUID, staging, paths); err == nil { t.Fatal("commitSandboxAssets accepted a path outside the staging dir") } - if got, err := os.ReadFile(filepath.Join(final, "runsc")); err != nil || string(got) != "previous" { + if got, err := os.ReadFile(filepath.Join(final, runsc)); err != nil || string(got) != "previous" { t.Errorf("assets dir changed by a refused commit: %q, %v", got, err) } - if _, err := os.Stat(filepath.Join(staging, "runsc")); err != nil { + if _, err := os.Stat(filepath.Join(staging, runsc)); err != nil { t.Errorf("staged set gone after a refused commit: %v", err) } - if paths[runscAssetName] != filepath.Join(staging, "runsc") { + if paths[gvisorAssetName] != filepath.Join(staging, runsc) { t.Errorf("paths rewritten by a refused commit: %v", paths) } @@ -1403,13 +1486,682 @@ func TestCommitSandboxAssetsChecksPathsBeforeMoving(t *testing.T) { if err := commitSandboxAssets(actorUID, staging, paths); err != nil { t.Fatalf("commitSandboxAssets: %v", err) } - if got, err := os.ReadFile(filepath.Join(final, "runsc")); err != nil || string(got) != "new" { + if got, err := os.ReadFile(filepath.Join(final, runsc)); err != nil || string(got) != "new" { t.Errorf("assets dir after commit = %q, %v; want the staged set", got, err) } - if paths[runscAssetName] != filepath.Join(final, "runsc") { + if paths[gvisorAssetName] != filepath.Join(final, runsc) { t.Errorf("paths after commit = %v, want them under %s", paths, final) } if _, err := os.Lstat(staging); !os.IsNotExist(err) { t.Errorf("staging dir remains after commit (lstat err = %v)", err) } } + +func TestTrackerStoreUploadLocalSnapshot(t *testing.T) { + a := runPluginActor(t) + snap := a.snapshot("pause-snap-1", 1) + if _, err := a.checkpoint(t, snap, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED); err != nil { + t.Fatalf("Checkpoint: %v", err) + } + st := &trackerStore{s: a.s} + + // An upload asking for a fidelity other than the pause captured is + // rejected before uploading. + noFidelity := proto.CloneOf(snap) + noFidelity.Fidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED + if _, err := st.UploadLocalSnapshot(t.Context(), pluginActorUID, snapshottracker.SnapshotRef{Snapshot: noFidelity}); apierror.Code(err) != codes.FailedPrecondition { + t.Fatalf("UploadLocalSnapshot with another fidelity: err = %v, want FailedPrecondition", err) + } + + // First upload writes to object storage and reports uploaded = true. + uploaded, err := st.UploadLocalSnapshot(t.Context(), pluginActorUID, snapshottracker.SnapshotRef{Snapshot: snap}) + if err != nil || !uploaded { + t.Fatalf("UploadLocalSnapshot = (%v, %v), want (true, nil)", uploaded, err) + } + + // Once the local copy is pruned, a retry sees the remote manifest and + // reports uploaded = false. + if err := snapshottracker.PruneLocalCheckpoints(t.Context(), pluginActorUID); err != nil { + t.Fatalf("PruneLocalCheckpoints: %v", err) + } + uploaded, err = st.UploadLocalSnapshot(t.Context(), pluginActorUID, snapshottracker.SnapshotRef{Snapshot: snap}) + if err != nil || uploaded { + t.Fatalf("UploadLocalSnapshot after prune = (%v, %v), want (false, nil)", uploaded, err) + } +} + +// TestRestoredActorIsNotDetached restores a paused actor from its local copy: +// once the actor runs on the node, atelet refuses to detach it. +func TestRestoredActorIsNotDetached(t *testing.T) { + a := runPluginActor(t) + a.s.snapshots = snapshottracker.New(t.Context(), &trackerStore{s: a.s}, clock.RealClock{}) + if _, err := a.checkpoint(t, a.snapshot("pause-snap-1", 1), ateletpb.UploadMode_UPLOAD_MODE_DEFERRED); err != nil { + t.Fatalf("Checkpoint (pause): %v", err) + } + a.restoreLocal(t, "pause-snap-1", 1) + if _, err := a.s.snapshots.Detach(t.Context(), pluginActorUID); apierror.Code(err) != codes.Aborted { + t.Errorf("Detach of the restored actor = %v, want Aborted", err) + } +} + +// TestFailedRestoreUnschedulesDetach fails a restore of a paused actor after +// it began: as after a restore that succeeds, atelet keeps the actor's local +// snapshot, and does not detach the actor until its next checkpoint. +func TestFailedRestoreUnschedulesDetach(t *testing.T) { + a := runPluginActor(t) + a.s.snapshots = snapshottracker.New(t.Context(), &trackerStore{s: a.s}, clock.RealClock{}) + if _, err := a.checkpoint(t, a.snapshot("pause-snap-1", 1), ateletpb.UploadMode_UPLOAD_MODE_DEFERRED); err != nil { + t.Fatalf("Checkpoint (pause): %v", err) + } + // A snapshot with neither a local copy nor an object storage location + // fails the restore once it has begun. + _, err := a.s.Restore(t.Context(), &ateletpb.RestoreRequest{ + Atespace: "ate-demo", + ActorName: "counter", + ActorUid: pluginActorUID, + ActorTemplateAtespace: "default", + ActorTemplateName: "counter", + WorkerPodUid: pluginWorkerPodUID, + SandboxAssets: a.assets, SandboxRuntime: runtimeFor(t, a.assets), + Spec: a.spec, + Snapshot: &ateletpb.Snapshot{SnapshotUuid: "missing-snap", Generation: 2, Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY}, + }) + if apierror.Code(err) != codes.NotFound { + t.Fatalf("Restore of a missing snapshot = %v, want NotFound", err) + } + if got := a.s.snapshots.State(pluginActorUID).GetLatestLocal().GetSnapshot().GetSnapshotUuid(); got != "pause-snap-1" { + t.Errorf("latest_local = %q after the failed restore, want pause-snap-1 kept", got) + } + if _, err := a.s.snapshots.Detach(t.Context(), pluginActorUID); apierror.Code(err) != codes.Aborted { + t.Errorf("Detach after the failed restore = %v, want Aborted", err) + } +} + +// TestInvalidRunKeepsDetachScheduled runs an actor with a request that +// names no sandbox runtime: Run fails before it begins the start, so the +// actor's detach stays scheduled. +func TestInvalidRunKeepsDetachScheduled(t *testing.T) { + a := runPluginActor(t) + a.s.snapshots = snapshottracker.New(t.Context(), &trackerStore{s: a.s}, clock.RealClock{}) + if _, err := a.checkpoint(t, a.snapshot("pause-snap-1", 1), ateletpb.UploadMode_UPLOAD_MODE_DEFERRED); err != nil { + t.Fatalf("Checkpoint (pause): %v", err) + } + _, err := a.s.Run(t.Context(), &ateletpb.RunRequest{ + Atespace: "ate-demo", + ActorName: "counter", + ActorUid: pluginActorUID, + ActorTemplateAtespace: "default", + ActorTemplateName: "counter", + WorkerPodUid: pluginWorkerPodUID, + SandboxAssets: a.assets, + Spec: a.spec, + }) + if apierror.Code(err) != codes.InvalidArgument { + t.Fatalf("Run without sandbox_runtime = %v, want InvalidArgument", err) + } + if _, err := a.s.snapshots.Detach(t.Context(), pluginActorUID); err != nil { + t.Errorf("Detach after the invalid Run = %v, want the detach still scheduled", err) + } +} + +// blockingTrackerStore is a snapshottracker.Store whose uploads report on +// started, then block until their context ends. +type blockingTrackerStore struct{ started chan struct{} } + +func (s *blockingTrackerStore) UploadLocalSnapshot(ctx context.Context, _ string, _ snapshottracker.SnapshotRef) (bool, error) { + s.started <- struct{}{} + <-ctx.Done() + return false, ctx.Err() +} + +// startBlockedDetach pauses the actor as pause-snap-1, and starts a Detach of +// it whose upload blocks until the test ends. +func startBlockedDetach(t *testing.T, a *pluginActor) { + t.Helper() + store := &blockingTrackerStore{started: make(chan struct{}, 1)} + a.s.snapshots = snapshottracker.New(t.Context(), store, clock.RealClock{}) + if _, err := a.checkpoint(t, a.snapshot("pause-snap-1", 1), ateletpb.UploadMode_UPLOAD_MODE_DEFERRED); err != nil { + t.Fatalf("Checkpoint (pause): %v", err) + } + ctx, cancel := context.WithCancel(t.Context()) + detached := make(chan struct{}) + go func() { + defer close(detached) + _, _ = a.s.snapshots.Detach(ctx, pluginActorUID) + }() + t.Cleanup(func() { + cancel() + <-detached + }) + select { + case <-store.started: + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for the detach's upload to start") + } +} + +// TestCheckpointRecordsDetachWait gives up a Checkpoint while it waits for the +// actor's detach in progress: the checkpoint's duration, the wait included, is +// still recorded. +func TestCheckpointRecordsDetachWait(t *testing.T) { + a := runPluginActor(t) + startBlockedDetach(t, a) + inst, reader := newTestInstruments(t) + a.s.instruments = inst + + gaveUp, cancel := context.WithCancel(t.Context()) + cancel() + _, err := a.s.Checkpoint(gaveUp, &ateletpb.CheckpointRequest{ + Atespace: "ate-demo", + ActorName: "counter", + ActorUid: pluginActorUID, + ActorTemplateAtespace: "default", + ActorTemplateName: "counter", + WorkerPodUid: pluginWorkerPodUID, + Spec: a.spec, + Snapshot: a.snapshot("pause-snap-2", 2), + UploadConfig: &ateletpb.UploadConfig{Mode: ateletpb.UploadMode_UPLOAD_MODE_DEFERRED}, + }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("Checkpoint that gave up waiting for the detach = %v, want %v", err, context.Canceled) + } + if _, ok := phaseValues(t, collectHistogram(t, reader, checkpointDurationMetric))[ateattr.SnapshotPhaseTotal]; !ok { + t.Error("no total duration recorded for the checkpoint") + } +} + +// TestPromoteSnapshotRecordsDetachWait gives up a PromoteSnapshot while it +// waits for the actor's detach in progress: the promote's duration, the wait +// included, is still recorded. +func TestPromoteSnapshotRecordsDetachWait(t *testing.T) { + a := runPluginActor(t) + startBlockedDetach(t, a) + inst, reader := newTestInstruments(t) + a.s.instruments = inst + + gaveUp, cancel := context.WithCancel(t.Context()) + cancel() + _, err := a.s.PromoteSnapshot(gaveUp, &ateletpb.PromoteSnapshotRequest{ + Atespace: "ate-demo", + ActorName: "counter", + ActorUid: pluginActorUID, + ActorTemplateAtespace: "default", + ActorTemplateName: "counter", + Snapshot: a.snapshot("pause-snap-1", 1), + }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("PromoteSnapshot that gave up waiting for the detach = %v, want %v", err, context.Canceled) + } + if _, ok := phaseValues(t, collectHistogram(t, reader, checkpointDurationMetric))[ateattr.SnapshotPhaseTotal]; !ok { + t.Error("no total duration recorded for the promote") + } +} + +// TestPromoteSnapshotFoundAlreadyUploaded promotes a paused actor's snapshot +// whose upload committed before the promote, unrecorded, and whose local copy +// is gone since: the promote records the snapshot as durable, without an +// upload time, since it wrote nothing. +func TestPromoteSnapshotFoundAlreadyUploaded(t *testing.T) { + a := runPluginActor(t) + snap := a.snapshot("pause-snap-1", 1) + if _, err := a.checkpoint(t, snap, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED); err != nil { + t.Fatalf("Checkpoint: %v", err) + } + if _, err := (&trackerStore{s: a.s}).UploadLocalSnapshot(t.Context(), pluginActorUID, snapshottracker.SnapshotRef{Snapshot: snap}); err != nil { + t.Fatalf("UploadLocalSnapshot: %v", err) + } + if err := snapshottracker.PruneLocalCheckpoints(t.Context(), pluginActorUID); err != nil { + t.Fatalf("PruneLocalCheckpoints: %v", err) + } + + if got := a.promote(t, snap).GetLatestDurable(); !proto.Equal(got.GetSnapshot(), snap) || got.GetDurableCompleteTime() != nil { + t.Errorf("latest_durable = %v, want %v with no upload time", got, snap) + } +} + +// advertiseRuntime makes the runtime that sa names one a worker on this +// node advertises, as the syncer would have after planning that worker's +// pod, so a Run or Restore carrying sa resolves. It gives s an index if it +// has none. +func advertiseRuntime(t *testing.T, s *Atelet, sa *ateletpb.SandboxAssets) { + t.Helper() + if s.runtimes == nil { + s.runtimes = newRuntimeIndex() + } + rec, err := recordFromRequest(sa) + if err != nil { + t.Fatalf("recordFromRequest: %v", err) + } + hash, err := versionBundleHash(rec) + if err != nil { + t.Fatalf("versionBundleHash: %v", err) + } + s.runtimes.register("test-worker-pod", advertisedClass{ + class: v1alpha1.SandboxClass(rec.SandboxClass), + config: "test-config", + hashed: []hashedVersion{{versionPlan: versionPlan{name: "v1", isDefault: true, rec: rec}, hash: hash}}, + }) +} + +// TestRunRejectsUnadvertisedRuntime pins that an actor placed for a runtime +// no worker on this node advertises is refused before anything is fetched +// or staged, rather than run on whatever the request describes. +func TestRunRejectsUnadvertisedRuntime(t *testing.T) { + useTempNodeDirs(t) + const actorUID = "01234567-89ab-cdef-0123-456789abcdef" + release := newGVisorTestRelease(t) + var gets atomic.Int32 + s := &Atelet{ + runtimes: newRuntimeIndex(), + sandboxAssetCache: newTestAssetCache(t), + anonGCSClient: fakeObjectStorage{data: release.tarball, gets: &gets}, + systemInfoVolumes: newSystemInfoVolumeRefresher(nil, nil), + } + assets := &ateletpb.SandboxAssets{ + SandboxClass: "gvisor", + PauseImage: testPauseImage, + Assets: map[string]*ateletpb.ArchAssets{runtime.GOARCH: {Files: map[string]*ateletpb.AssetFile{ + gvisorAssetName: release.proto(), + }}}, + } + _, err := s.Run(t.Context(), &ateletpb.RunRequest{ + Atespace: "team-a", ActorName: "actor-1", ActorUid: actorUID, WorkerPodUid: "5f1c0d2e-3a4b-4c5d-8e9f-0a1b2c3d4e5f", + SandboxAssets: assets, SandboxRuntime: runtimeFor(t, assets), Spec: &ateletpb.WorkloadSpec{}, + }) + if apierror.Code(err) != codes.FailedPrecondition || !strings.Contains(err.Error(), "not advertised") { + t.Fatalf("Run for an unadvertised runtime = %v (code %s), want %s naming the runtime", err, apierror.Code(err), codes.FailedPrecondition) + } + if gets.Load() != 0 { + t.Errorf("assets fetched %d times for a refused Run, want 0", gets.Load()) + } + if _, err := os.Lstat(ateletpath.ActorPath(actorUID)); !os.IsNotExist(err) { + t.Errorf("refused Run created the actor dir (lstat err = %v)", err) + } + + // A request that names no runtime is invalid, whatever its assets say. + _, err = s.Run(t.Context(), &ateletpb.RunRequest{ + Atespace: "team-a", ActorName: "actor-1", ActorUid: actorUID, WorkerPodUid: "5f1c0d2e-3a4b-4c5d-8e9f-0a1b2c3d4e5f", + SandboxAssets: assets, Spec: &ateletpb.WorkloadSpec{}, + }) + if apierror.Code(err) != codes.InvalidArgument { + t.Errorf("Run without sandbox_runtime = %v (code %s), want %s", err, apierror.Code(err), codes.InvalidArgument) + } + + // Once a worker advertises it, the same request resolves. + advertiseRuntime(t, s, assets) + if _, err := s.advertisedRuntime(t.Context(), runtimeFor(t, assets)); err != nil { + t.Errorf("advertisedRuntime after advertising: %v", err) + } +} + +// runtimeFor returns the SandboxRuntime ate-api-server sends for an actor +// placed on a worker that advertises the runtime sa names: the worker's +// registration, whose sandbox_bundle_hash is the hash of sa's record. +func runtimeFor(t *testing.T, sa *ateletpb.SandboxAssets) *ateletpb.SandboxRuntime { + t.Helper() + rec, err := recordFromRequest(sa) + if err != nil { + t.Fatalf("recordFromRequest: %v", err) + } + hash, err := versionBundleHash(rec) + if err != nil { + t.Fatalf("versionBundleHash: %v", err) + } + return &ateletpb.SandboxRuntime{ + SandboxClass: sa.GetSandboxClass(), + Version: &ateletpb.VersionedSandboxCompat{ + SchemaVersion: hardware.SchemaVersionV1, + Attributes: []*ateletpb.AttributeEntry{ + {Key: hardware.AttrArchitecture, Value: runtime.GOARCH}, + {Key: hardware.AttrSandboxBundleHash, Value: hash}, + }, + }, + } +} + +type gatingTrackerStore struct { + inner snapshottracker.Store + objects *recordingObjectStorage + started chan struct{} + release chan struct{} +} + +func (s *gatingTrackerStore) UploadLocalSnapshot(ctx context.Context, actorUID string, ref snapshottracker.SnapshotRef) (bool, error) { + s.started <- struct{}{} + select { + case <-s.release: + case <-ctx.Done(): + return false, ctx.Err() + } + uploaded, err := s.inner.UploadLocalSnapshot(ctx, actorUID, ref) + // Fail any second upload attempt so PromoteSnapshot proves it skips + // re-uploading once the background upload made the snapshot durable. + s.objects.mu.Lock() + delete(s.objects.objects, manifestKey(ref.Snapshot.GetSnapshotUuid())) + s.objects.putErr = errors.New("promote re-uploaded a snapshot already uploaded in the background") + s.objects.mu.Unlock() + return uploaded, err +} + +// stepToUpload advances clk to due and waits for the background upload to +// start. The tracker's queue arms its timer from a clock reading taken just +// before, so a step that lands in between leaves the timer late; further +// one-second steps get past it. +func stepToUpload(t *testing.T, clk *clocktesting.FakeClock, due time.Time, started <-chan struct{}) { + t.Helper() + clk.Step(due.Sub(clk.Now())) + waitFor(t, "the background upload to start", func() bool { + select { + case <-started: + return true + case <-time.After(10 * time.Millisecond): + clk.Step(time.Second) + return false + } + }) +} + +// TestPromoteSnapshotWaitsForBackgroundUpload promotes a snapshot while its +// background upload is in flight: PromoteSnapshot waits for the background +// upload to finish and does not upload the snapshot a second time. +func TestPromoteSnapshotWaitsForBackgroundUpload(t *testing.T) { + a := runPluginActor(t) + snap := a.snapshot("pause-snap-1", 1) + clk := clocktesting.NewFakeClock(time.Now()) + store := &gatingTrackerStore{ + inner: &trackerStore{s: a.s}, + objects: a.store, + started: make(chan struct{}, 1), + release: make(chan struct{}), + } + a.s.snapshots = snapshottracker.New(t.Context(), store, clk, + snapshottracker.WithUploadDelay(time.Second), + snapshottracker.WithDetachDelay(time.Hour)) + a.s.snapshots.Start() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if err := a.s.snapshots.Wait(ctx); err != nil { + t.Errorf("Wait: %v", err) + } + }) + + cpResp, err := a.checkpoint(t, snap, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) + if err != nil { + t.Fatalf("Checkpoint: %v", err) + } + stepToUpload(t, clk, cpResp.GetState().GetUploadDueTime().AsTime(), store.started) + + type promoteResult struct { + state *ateletpb.ActorSnapshotState + err error + } + resCh := make(chan promoteResult, 1) + go func() { + resp, err := a.s.PromoteSnapshot(t.Context(), &ateletpb.PromoteSnapshotRequest{ + Atespace: "ate-demo", + ActorName: "counter", + ActorUid: pluginActorUID, + ActorTemplateAtespace: "default", + ActorTemplateName: "counter", + Snapshot: snap, + }) + resCh <- promoteResult{resp.GetState(), err} + }() + select { + case res := <-resCh: + t.Fatalf("PromoteSnapshot returned (%v, %v) while background upload was in progress", res.state, res.err) + case <-time.After(50 * time.Millisecond): + } + + close(store.release) + res := <-resCh + if res.err != nil { + t.Fatalf("PromoteSnapshot: %v", res.err) + } + if got := res.state.GetLatestDurable(); !proto.Equal(got.GetSnapshot(), snap) || got.GetDurableCompleteTime() == nil { + t.Errorf("latest_durable = %v, want %v with the background upload's time", got, snap) + } +} + +// TestCheckpointKeepsSnapshotBeingUploaded pauses a resumed actor while the +// background upload of its previous pause snapshot still runs: the +// checkpoint's prune keeps that snapshot for the upload, which deletes it once +// it ends. +func TestCheckpointKeepsSnapshotBeingUploaded(t *testing.T) { + a := runPluginActor(t) + clk := clocktesting.NewFakeClock(time.Now()) + store := &gatingTrackerStore{ + inner: &trackerStore{s: a.s}, + objects: a.store, + started: make(chan struct{}, 1), + release: make(chan struct{}), + } + a.s.snapshots = snapshottracker.New(t.Context(), store, clk, + snapshottracker.WithUploadDelay(time.Second), + snapshottracker.WithDetachDelay(time.Hour)) + a.s.snapshots.Start() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if err := a.s.snapshots.Wait(ctx); err != nil { + t.Errorf("Wait: %v", err) + } + }) + + cpResp, err := a.checkpoint(t, a.snapshot("pause-snap-1", 1), ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) + if err != nil { + t.Fatalf("Checkpoint (pause-snap-1): %v", err) + } + stepToUpload(t, clk, cpResp.GetState().GetUploadDueTime().AsTime(), store.started) + + a.restoreLocal(t, "pause-snap-1", 1) + if _, err := a.checkpoint(t, a.snapshot("pause-snap-2", 2), ateletpb.UploadMode_UPLOAD_MODE_DEFERRED); err != nil { + t.Fatalf("Checkpoint (pause-snap-2): %v", err) + } + snap1Dir := ateletpath.LocalSnapshotDir(pluginActorUID, "pause-snap-1") + if _, err := os.Stat(snap1Dir); err != nil { + t.Fatalf("pause-snap-1 during its upload: stat = %v, want it kept", err) + } + + close(store.release) + waitFor(t, "pause-snap-1 to be deleted once its upload ends", func() bool { + _, err := os.Stat(snap1Dir) + return os.IsNotExist(err) + }) + if _, err := os.Stat(ateletpath.LocalSnapshotDir(pluginActorUID, "pause-snap-2")); err != nil { + t.Errorf("pause-snap-2 after the upload of pause-snap-1: stat = %v, want it kept", err) + } +} + +// TestCheckpointRejectsStaleGeneration checks that Checkpoint fails with +// FailedPrecondition when given a generation below the actor's newest snapshot +// or equal to it under another snapshot_uuid, before calling worker, pruning +// local checkpoints, or changing tracked state. +func TestCheckpointRejectsStaleGeneration(t *testing.T) { + a := runPluginActor(t) + snap2 := a.snapshot("pause-snap-2", 2) + resp, err := a.checkpoint(t, snap2, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) + if err != nil { + t.Fatalf("Checkpoint: %v", err) + } + wantState := resp.GetState() + a.restoreLocal(t, "pause-snap-2", 2) + delete(a.worker.actorDirs, "CheckpointWorkload") + + for _, stale := range []*ateletpb.Snapshot{ + a.snapshot("stale-snap-1", 1), + a.snapshot("other-snap-2", 2), + } { + _, err := a.checkpoint(t, stale, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) + if code := apierror.Code(err); code != codes.FailedPrecondition { + t.Errorf("Checkpoint(%v) error = %v (code %v), want FailedPrecondition", stale, err, code) + } + if got := a.worker.actorDirs["CheckpointWorkload"]; got != nil { + t.Errorf("Checkpoint(%v) called worker.CheckpointWorkload (%v), want rejected before worker", stale, got) + } + if _, err := os.Stat(ateletpath.LocalSnapshotDir(pluginActorUID, "pause-snap-2")); err != nil { + t.Errorf("local snapshot pause-snap-2 after rejected Checkpoint(%v): stat = %v, want preserved", stale, err) + } + if got := a.s.snapshots.State(pluginActorUID); !proto.Equal(got, wantState) { + t.Errorf("tracked state after rejected Checkpoint(%v) = %v, want %v", stale, got, wantState) + } + } +} + +// TestPromoteSnapshotRejectsStaleGeneration checks that PromoteSnapshot fails +// with FailedPrecondition when given a generation below the actor's newest +// snapshot or equal to it under another snapshot_uuid, before uploading to +// object storage, pruning local checkpoints, or changing tracked state. +func TestPromoteSnapshotRejectsStaleGeneration(t *testing.T) { + a := runPluginActor(t) + snap2 := a.snapshot("pause-snap-2", 2) + resp, err := a.checkpoint(t, snap2, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) + if err != nil { + t.Fatalf("Checkpoint: %v", err) + } + wantState := resp.GetState() + + for _, stale := range []*ateletpb.Snapshot{ + a.snapshot("stale-snap-1", 1), + a.snapshot("other-snap-2", 2), + } { + writeLocalSnapshot(t, ateletpath.LocalSnapshotDir(pluginActorUID, stale.GetSnapshotUuid()), sandboxAssetsRecord{ + SandboxClass: "gvisor", + PauseImage: a.assets.GetPauseImage(), + SnapshotFiles: []string{"checkpoint.img"}, + Fidelity: "MEMORY", + }, map[string]string{"checkpoint.img": "stale-memory"}) + + _, err := a.s.PromoteSnapshot(t.Context(), &ateletpb.PromoteSnapshotRequest{ + Atespace: "ate-demo", + ActorName: "counter", + ActorUid: pluginActorUID, + ActorTemplateAtespace: "default", + ActorTemplateName: "counter", + Snapshot: stale, + }) + if code := apierror.Code(err); code != codes.FailedPrecondition { + t.Errorf("PromoteSnapshot(%v) error = %v (code %v), want FailedPrecondition", stale, err, code) + } + if got := a.store.keys(); len(got) != 0 { + t.Errorf("objects after rejected PromoteSnapshot(%v) = %v, want nothing uploaded", stale, got) + } + if _, err := os.Stat(ateletpath.LocalSnapshotDir(pluginActorUID, "pause-snap-2")); err != nil { + t.Errorf("local snapshot pause-snap-2 after rejected PromoteSnapshot(%v): stat = %v, want preserved", stale, err) + } + if got := a.s.snapshots.State(pluginActorUID); !proto.Equal(got, wantState) { + t.Errorf("tracked state after rejected PromoteSnapshot(%v) = %v, want %v", stale, got, wantState) + } + } +} + +// TestCheckpointInFlightRejectsOlderGeneration checks that while Checkpoint is +// running on the worker for generation 3, an out-of-order Checkpoint or +// PromoteSnapshot for generation 2 fails with FailedPrecondition. +func TestCheckpointInFlightRejectsOlderGeneration(t *testing.T) { + a := runPluginActor(t) + started := make(chan struct{}) + release := make(chan struct{}) + var once sync.Once + a.worker.onCheckpoint = func() { + once.Do(func() { + close(started) + <-release + }) + } + + cpErr := make(chan error, 1) + go func() { + _, err := a.checkpoint(t, a.snapshot("pause-snap-3", 3), ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) + cpErr <- err + }() + select { + case <-started: + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for Checkpoint(pause-snap-3) to reach worker") + } + + stale := a.snapshot("pause-snap-2", 2) + if _, err := a.checkpoint(t, stale, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED); apierror.Code(err) != codes.FailedPrecondition { + t.Errorf("Checkpoint(%v) while generation 3 is in flight = %v, want FailedPrecondition", stale, err) + } + if _, err := a.s.PromoteSnapshot(t.Context(), &ateletpb.PromoteSnapshotRequest{ + Atespace: "ate-demo", + ActorName: "counter", + ActorUid: pluginActorUID, + ActorTemplateAtespace: "default", + ActorTemplateName: "counter", + Snapshot: stale, + }); apierror.Code(err) != codes.FailedPrecondition { + t.Errorf("PromoteSnapshot(%v) while generation 3 is in flight = %v, want FailedPrecondition", stale, err) + } + + close(release) + if err := <-cpErr; err != nil { + t.Fatalf("Checkpoint(pause-snap-3): %v", err) + } + if got := a.s.snapshots.State(pluginActorUID).GetLatestLocal().GetSnapshot().GetSnapshotUuid(); got != "pause-snap-3" { + t.Errorf("latest_local = %q, want pause-snap-3", got) + } +} + +// TestCheckpointSkipsStaleCommitAndKeepsNewerLocal checks that when a stalled +// Checkpoint(gen=2) reaches CommitLocal and PruneLocalCheckpoints after a newer +// snapshot (gen=3) has already committed locally, gen=3 stays tracked and its +// local directory stays on disk while gen=2's local copy is pruned. +func TestCheckpointSkipsStaleCommitAndKeepsNewerLocal(t *testing.T) { + a := runPluginActor(t) + started := make(chan struct{}) + release := make(chan struct{}) + var once sync.Once + a.worker.onCheckpoint = func() { + once.Do(func() { + close(started) + <-release + }) + } + + type cpResult struct { + resp *ateletpb.CheckpointResponse + err error + } + resCh := make(chan cpResult, 1) + go func() { + resp, err := a.checkpoint(t, a.snapshot("pause-snap-2", 2), ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) + resCh <- cpResult{resp, err} + }() + select { + case <-started: + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for Checkpoint(pause-snap-2) to reach worker") + } + + snap3 := a.snapshot("pause-snap-3", 3) + snap3Dir := ateletpath.LocalSnapshotDir(pluginActorUID, snap3.GetSnapshotUuid()) + writeLocalSnapshot(t, snap3Dir, sandboxAssetsRecord{ + SandboxClass: "gvisor", + PauseImage: a.assets.GetPauseImage(), + SnapshotFiles: []string{"checkpoint.img"}, + Fidelity: "MEMORY", + }, map[string]string{"checkpoint.img": "newer-memory"}) + wantState := a.s.snapshots.CommitLocal(pluginActorUID, snapshottracker.SnapshotRef{ + Snapshot: snap3, + CommitTime: time.Now(), + }) + + close(release) + res := <-resCh + if res.err != nil { + t.Fatalf("Checkpoint(pause-snap-2): %v", res.err) + } + if !proto.Equal(res.resp.GetState(), wantState) { + t.Errorf("Checkpoint(pause-snap-2) state = %v, want %v (keeping pause-snap-3)", res.resp.GetState(), wantState) + } + if _, err := os.Stat(snap3Dir); err != nil { + t.Errorf("newer local snapshot pause-snap-3: stat = %v, want kept", err) + } + if _, err := os.Stat(ateletpath.LocalSnapshotDir(pluginActorUID, "pause-snap-2")); !os.IsNotExist(err) { + t.Errorf("stale local snapshot pause-snap-2: stat = %v, want pruned", err) + } +} diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 037d0188c9..4384e43f0c 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -122,7 +122,13 @@ var ( drainDelay = pflag.Duration("drain-delay", 0, "How long to keep accepting new RPCs after SIGTERM before starting the gRPC drain.") drainTimeout = pflag.Duration("drain-timeout", 5*time.Minute, "Deadline for the graceful gRPC drain on shutdown. In-flight RPCs still running past it are forcefully cancelled.") - objectStorePluginSocket = pflag.String("objectstore-plugin-socket", "/run/objectstore-plugin/node.sock", "Unix socket of the node object-store plugin that external snapshots are fetched and uploaded through, and every sandbox asset is fetched through. A gs:// asset the plugin cannot serve is then read anonymously.") + objectStorePluginSocket = pflag.String("objectstore-plugin-socket", "/run/objectstore-plugin/node.sock", "Unix socket of the node object-store plugin that external snapshots are fetched and uploaded through, and every sandbox asset is fetched through. A gs:// asset the plugin cannot serve is then read anonymously.") + snapshotUploadDelay = pflag.Duration("snapshot-upload-delay", snapshottracker.DefaultUploadDelay, "How long after an actor's oldest snapshot not yet in object storage was taken atelet uploads the actor's latest local snapshot in the background; later while an upload is in progress or failing.") + snapshotDetachDelay = pflag.Duration("snapshot-detach-delay", snapshottracker.DefaultDetachDelay, "How long an actor stays suspended on this node, its latest snapshot kept locally, before atelet detaches it in the background.") + snapshotUploadWorkers = pflag.Int("snapshot-upload-workers", snapshottracker.DefaultUploadWorkers, "How many background snapshot uploads atelet runs at once.") + snapshotDetachWorkers = pflag.Int("snapshot-detach-workers", snapshottracker.DefaultDetachWorkers, "How many background detaches atelet runs at once.") + snapshotReportWorkers = pflag.Int("snapshot-report-workers", snapshottracker.DefaultReportWorkers, "How many background snapshot state reports atelet runs at once. 0 disables background reporting.") + snapshotReportGiveUpAfter = pflag.Duration("snapshot-report-give-up-after", snapshottracker.DefaultReportGiveUpAfter, "How long after a background upload or detach atelet keeps retrying a failed snapshot state report before giving up on it.") ) var _ imagecache.CandidateKeychain = (*credentialprovider.Keychain)(nil) @@ -145,6 +151,12 @@ func main() { if err := rejectStorageEnv(); err != nil { serverboot.Fatal(ctx, "Invalid environment", err) } + if err := validateSnapshotDelayFlags(); err != nil { + serverboot.Fatal(ctx, "Invalid snapshot delay flags", err) + } + if err := validateSnapshotWorkerFlags(); err != nil { + serverboot.Fatal(ctx, "Invalid snapshot worker flags", err) + } // Kept separate from ctx so in-flight work (e.g. a Checkpoint/Restore // streaming a multi-GiB snapshot) is not cancelled the moment SIGTERM @@ -175,10 +187,19 @@ func main() { if err != nil { serverboot.Fatal(ctx, "Failed to create atelet metrics", err) } + trackerInstruments, err := snapshottracker.NewInstruments(otel.Meter("atelet")) + if err != nil { + serverboot.Fatal(ctx, "Failed to create snapshot tracker metrics", err) + } // readiness flips to not-ready on SIGTERM so /readyz reports 503 while the - // pod drains, while /healthz stays 200 for liveness. + // pod drains, while /healthz stays 200 for liveness. It is not ready + // until the gRPC port is open either: everything from here to net.Listen, + // the trust bundle sync, the plugin socket, the cache open and the worker + // runtimes gate, is startup, and the health server already answers. readiness := &serverboot.Readiness{} + serving := make(chan struct{}) + readiness.AddCheck("serving", servingCheck(serving)) go serverboot.StartMetricsServer(ctx, *metricsListenAddr) go serverboot.StartHealthServer(ctx, *healthListenAddr, readiness) @@ -314,11 +335,51 @@ func main() { serverboot.Fatal(ctx, "Object-store plugin is not serving", err) } + var ateapiConn *grpc.ClientConn + var reporter snapshottracker.Reporter + if !*grpcInsecure { + dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ + K8sClient: k8sClient, + CAFile: *ateapiCAFile, + ServerName: *ateapiServerName, + ClientCredBundle: *grpcServerCredBundle, + }) + if err != nil { + serverboot.Fatal(ctx, "Failed to build ateapi client credentials", err) + } + // atelet acts for the system, not a user, so it always opts in to the + // preview features the server enables. + dialOpts = append(dialOpts, preview.DialOptionsWithOptIn()...) + ateapiConn, err = grpc.NewClient(*ateapiAddress, dialOpts...) + if err != nil { + serverboot.Fatal(ctx, "Failed to create ateapi client", err) + } + defer ateapiConn.Close() + reporter = &trackerReporter{workers: ateapipb.NewWorkerServiceClient(ateapiConn)} + } + // The snapshot tracker's workers get a context of their own, cancelled only // once the gRPC server has drained, so background snapshot work keeps // running while in-flight RPCs finish. trackerCtx, stopTracker := context.WithCancel(ctx) - snapshots := snapshottracker.New(trackerCtx, clock.RealClock{}) + store := &trackerStore{} + snapshots := snapshottracker.New(trackerCtx, store, clock.RealClock{}, + snapshottracker.WithUploadDelay(*snapshotUploadDelay), + snapshottracker.WithDetachDelay(*snapshotDetachDelay), + snapshottracker.WithUploadWorkers(*snapshotUploadWorkers), + snapshottracker.WithDetachWorkers(*snapshotDetachWorkers), + snapshottracker.WithInstruments(trackerInstruments), + snapshottracker.WithReporter(reporter), + snapshottracker.WithReportWorkers(*snapshotReportWorkers), + snapshottracker.WithReportGiveUpAfter(*snapshotReportGiveUpAfter)) + // The runtimes this node's workers advertise, read by Run and Restore. + // Loaded from disk before anything serves, so a runtime a worker + // registered before a restart resolves at once; the worker sandbox + // syncer below keeps it current. + runtimes, err := openRuntimeIndex(ctx, ateletpath.SandboxRuntimeIndexFile) + if err != nil { + serverboot.Fatal(ctx, "Failed to open the sandbox runtime index", err) + } wmService := NewService( ctx, workerDialer, @@ -328,42 +389,22 @@ func main() { *objectStorePluginSocket, imageCache, sandboxAssetCache, + runtimes, instruments, volPlugins, csiDriverConfigGetter, systemInfoVolumes, snapshots, ) + store.s = wmService snapshots.Start() go systemInfoVolumes.run(ctx) - lis, err := net.Listen("tcp", ":"+strconv.Itoa(*port)) - if err != nil { - serverboot.Fatal(ctx, "Failed to listen", err) - } - serverOpts := ateletServerOptions(insecure.NewCredentials()) nodeName := os.Getenv("NODE_NAME") if *grpcInsecure { slog.WarnContext(ctx, "Serving atelet gRPC without transport security") } else { - dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ - K8sClient: k8sClient, - CAFile: *ateapiCAFile, - ServerName: *ateapiServerName, - ClientCredBundle: *grpcServerCredBundle, - }) - if err != nil { - serverboot.Fatal(ctx, "Failed to build ateapi client credentials", err) - } - // atelet acts for the system, so it opts in to enabled preview features. - dialOpts = append(dialOpts, preview.DialOptionsWithOptIn()...) - ateapiConn, err := grpc.NewClient(*ateapiAddress, dialOpts...) - if err != nil { - serverboot.Fatal(ctx, "Failed to create ateapi client", err) - } - defer ateapiConn.Close() - tlsCfg, err := ateletServerTLSConfig(*grpcServerCredBundle, *clientCACerts) if err != nil { serverboot.Fatal(ctx, "Failed to build server TLS config", err) @@ -413,14 +454,14 @@ func main() { serverboot.Fatal(ctx, "Failed to determine local node", fmt.Errorf("NODE_NAME must be set when --grpc-insecure is enabled")) } - // SandboxConfigs feed the asset prewarmer and workerSandboxSyncer. - // WorkerPools tell the syncer which SandboxConfig each local worker's pool - // uses. + // SandboxConfigs and WorkerPools feed the workerSandboxSyncer: the pools + // tell it which SandboxConfig each local worker uses. // - // We intentionally never WaitForCacheSync on these factories, so a failing + // Startup does not wait on these factories: the runtime index was loaded + // from disk above, so the workers already on this node resolve at once, + // and the syncer catches up with changes in the background. A failing // list/watch (e.g. Forbidden while the ClusterRole rollout lags the - // binary) cannot hold up atelet startup: the reflector retries in the - // background and the prewarmer and syncer wait until it recovers. + // binary) is logged by the informer and leaves the files as they are. // // 0 disables periodic resync. Both react to watch events and retry on their own. ateFactory := externalversions.NewSharedInformerFactory(ateClient, 0) @@ -432,32 +473,38 @@ func main() { o.FieldSelector = "spec.nodeName=" + nodeName o.LabelSelector = workerPoolLabel })) - // Pre-download sandbox assets as SandboxConfigs appear/change so the first - // Run/Restore on this node hits the cache. Best-effort: on failure the - // on-demand fetch in ensureSandboxAssets still covers correctness. - // - // TODO: remove once the worker sandbox syncer fetches assets through - // cmd/atelet/internal/filecache. - sandboxConfigInformer := ateFactory.Api().V1alpha1().SandboxConfigs().Informer() - if _, err := startSandboxAssetPrewarm(ctx, sandboxConfigInformer, wmService, imageCache, microvmNodeCapable(hostDevRoot)); err != nil { - slog.ErrorContext(ctx, "Sandbox asset prewarm disabled", slog.Any("err", err)) - } + // Pre-download each local worker's default runtime, as the syncer plans + // the worker, so the first Run/Restore on this node hits the cache. + // Best-effort: on failure the on-demand fetch in ensureSandboxAssets + // still covers correctness. + prewarmer := newRuntimePrewarmer(runtimes, wmService, imageCache) + go prewarmer.run(ctx) // Keep a sandbox runtimes file in each local worker's directory: the // runtimes of the enabled versions of its pool's SandboxConfig, // rewritten when the config changes. - if _, err := startWorkerSandboxSyncer(ctx, + _, err = startWorkerSandboxSyncer(ctx, workerPodFactory.Core().V1().Pods(), ateFactory.Api().V1alpha1().WorkerPools(), ateFactory.Api().V1alpha1().SandboxConfigs(), wmService, - nodepath.WorkersDir()); err != nil { - slog.ErrorContext(ctx, "Worker sandbox runtimes syncer disabled", slog.Any("err", err)) + nodepath.WorkersDir(), runtimes, prewarmer.enqueue) + if err != nil { + // Without the syncer no runtime is ever indexed, so no actor could run. + serverboot.Fatal(ctx, "Failed to start the worker sandbox runtimes syncer", err) } // Start after the informers are registered: a factory only runs // informers that exist when Start is called. ateFactory.Start(stopCh) workerPodFactory.Start(stopCh) + // Workers already registered with the control plane may be sent actors + // the moment atelet is reachable; the runtime index loaded above is what + // makes that safe after a restart. + lis, err := net.Listen("tcp", ":"+strconv.Itoa(*port)) + if err != nil { + serverboot.Fatal(ctx, "Failed to listen", err) + } + close(serving) svr := grpc.NewServer(serverOpts...) ateletpb.RegisterAteletServer(svr, wmService) reflection.Register(svr) @@ -479,6 +526,20 @@ func main() { slog.InfoContext(ctx, "Shutdown complete") } +// servingCheck is the readiness check that holds /readyz at 503 until served +// is closed, which main does once the gRPC port is open. An atelet that +// stops before then never closes it, and so is never ready. +func servingCheck(served <-chan struct{}) func(context.Context) error { + return func(context.Context) error { + select { + case <-served: + return nil + default: + return errors.New("atelet is starting; the gRPC port is not open yet") + } + } +} + // ateletServerOptions are the options of the gRPC server that serves // Atelet to ate-api-server. Its interceptor logs every request; fields // marked debug_redact (env values, asset URLs) are masked in that log. @@ -501,6 +562,42 @@ func rejectStorageEnv() error { return nil } +// validateSnapshotDelayFlags checks --snapshot-upload-delay, +// --snapshot-detach-delay, and --snapshot-report-give-up-after: all three must +// be positive, and the upload delay must not exceed the detach delay, since a +// detach uploads the latest local snapshot anyway. +func validateSnapshotDelayFlags() error { + if *snapshotUploadDelay <= 0 { + return fmt.Errorf("--snapshot-upload-delay %v must be > 0", *snapshotUploadDelay) + } + if *snapshotDetachDelay <= 0 { + return fmt.Errorf("--snapshot-detach-delay %v must be > 0", *snapshotDetachDelay) + } + if *snapshotUploadDelay > *snapshotDetachDelay { + return fmt.Errorf("--snapshot-upload-delay %v must not exceed --snapshot-detach-delay %v", *snapshotUploadDelay, *snapshotDetachDelay) + } + if *snapshotReportGiveUpAfter <= 0 { + return fmt.Errorf("--snapshot-report-give-up-after %v must be > 0", *snapshotReportGiveUpAfter) + } + return nil +} + +// validateSnapshotWorkerFlags checks --snapshot-upload-workers, +// --snapshot-detach-workers, and --snapshot-report-workers: upload and detach +// must be at least 1, and report must be at least 0. +func validateSnapshotWorkerFlags() error { + if *snapshotUploadWorkers < 1 { + return fmt.Errorf("--snapshot-upload-workers %d must be >= 1", *snapshotUploadWorkers) + } + if *snapshotDetachWorkers < 1 { + return fmt.Errorf("--snapshot-detach-workers %d must be >= 1", *snapshotDetachWorkers) + } + if *snapshotReportWorkers < 0 { + return fmt.Errorf("--snapshot-report-workers %d must be >= 0", *snapshotReportWorkers) + } + return nil +} + // drainOnShutdown drives graceful shutdown when ctx is cancelled (SIGTERM or // interrupt): it marks the process not-ready, waits drain-delay while still // accepting work, then GracefulStop()s the gRPC server so in-flight RPCs finish. @@ -552,6 +649,9 @@ type Atelet struct { // sandboxAssetCache holds every sandbox asset on the node once; actors get hard // links into their own directories (see ensureSandboxAssets). sandboxAssetCache *filecache.Store + // runtimes resolves the runtime a Run or Restore names to the assets to + // stage, among those this node's workers advertise. + runtimes *runtimeIndex // anonGCSClient reads gs:// assets the plugin cannot serve, without credentials. anonGCSClient objectstorage.ObjectStorage // objectStorePlugin moves external snapshot files between the node and storage. @@ -586,6 +686,7 @@ func NewService( pluginSocket string, imageCache *imagecache.Store, sandboxAssetCache *filecache.Store, + runtimes *runtimeIndex, instruments *Instruments, volumePlugins map[string]volume.VolumePluginWorkerPlane, csiDriverConfigGetter csi.CSIDriverConfigGetter, @@ -596,6 +697,7 @@ func NewService( workerDialer: workerDialer, imageCache: imageCache, sandboxAssetCache: sandboxAssetCache, + runtimes: runtimes, anonGCSClient: anonGCSClient, objectStorePlugin: objectStorePlugin, assetPlugin: assetPlugin, @@ -618,10 +720,20 @@ func (s *Atelet) Run(ctx context.Context, req *ateletpb.RunRequest) (resp *atele actorUID := req.GetActorUid() actorRef := resources.ActorRef{Atespace: req.GetAtespace(), Name: req.GetActorName()} - sandboxRec, err := recordFromRequest(req.GetSandboxAssets()) + // The sandbox (binaries + pause image) is the runtime the control plane + // assigned the actor to, resolved through the index. Resolved before any + // on-node work, BeginStart included, so an invalid request changes + // nothing. + sandboxRec, err := s.advertisedRuntime(ctx, req.GetSandboxRuntime()) if err != nil { - return nil, apierror.InvalidArgument("%v", err) + return nil, err + } + endStart, err := s.snapshots.BeginStart(ctx, actorUID) + if err != nil { + return nil, err } + defer endStart() + // Fetched before the reset below, into a staging dir, so a failed fetch // leaves a live activation as it was. The staged set is swapped in only // after the reset, together with the record that describes it, so the @@ -766,7 +878,7 @@ func (s *Atelet) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest // Per-phase timing, recorded on the way out so a failed checkpoint still // reports the phases it completed. Phases left at zero never ran. tStart := time.Now() - var dAssets, dWorker, dPersist time.Duration + var dAssets, dWorker, dPersist, dCleanup time.Duration op := snapshotOp{ templateNamespace: req.GetActorTemplateAtespace(), templateName: req.GetActorTemplateName(), @@ -787,6 +899,7 @@ func (s *Atelet) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest {ateattr.SnapshotPhaseSandboxAssets, dAssets}, {ateattr.SnapshotPhaseWorkerCheckpoint, dWorker}, {ateattr.SnapshotPhasePersist, dPersist}, + {ateattr.SnapshotPhaseCleanup, dCleanup}, {ateattr.SnapshotPhaseTotal, time.Since(tStart)}, } s.instruments.recordCheckpoint(ctx, op, phases...) @@ -794,6 +907,14 @@ func (s *Atelet) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest snapshotLogAttrs(attribution, op, checkpointDurationMetric, err, phases)...) }() + // A detach in progress finishes first, and none starts until the + // checkpoint returns, so it cannot delete the snapshot being written. + endCheckpoint, err := s.snapshots.BeginCheckpoint(ctx, actorUID, req.GetSnapshot()) + if err != nil { + return nil, err + } + defer endCheckpoint() + // Checkpoint requests no longer carry the sandbox config; recover the // version this actor was started with from the on-node record and re-fetch // it (a cache hit) so worker can drive runsc, and so we can pin it into the @@ -833,7 +954,6 @@ func (s *Atelet) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest ActorTemplateAtespace: req.GetActorTemplateAtespace(), ActorTemplateName: req.GetActorTemplateName(), RunscPath: runscPathFor(assetPaths), - RuntimeAssetPaths: assetPaths, Spec: spec, Fidelity: toWorkerFidelity(req.GetSnapshot().GetFidelity()), ActorUid: actorUID, @@ -847,7 +967,7 @@ func (s *Atelet) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest } // The snapshot commits on this node once worker has written it, in either // upload mode. - snap := snapshottracker.SnapshotRef{Snapshot: req.GetSnapshot(), CommitTime: time.Now()} + snap := snapshottracker.SnapshotRef{Snapshot: req.GetSnapshot(), CommitTime: time.Now(), Actor: actorRef} s.systemInfoVolumes.Deregister(actorUID) @@ -862,36 +982,6 @@ func (s *Atelet) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest sandboxRec.ActorTemplateName = req.GetActorTemplateName() sandboxRec.Fidelity = ateattr.SnapshotFidelityValue(req.GetSnapshot().GetFidelity()) - // Forget the local snapshot before the prune below deletes it, so the - // tracker never names a deleted snapshot, even if this checkpoint fails. - // - // TODO: Revisit this once the durable case can keep its local snapshot too - // (see UPLOAD_MODE_IMMEDIATE). Today the prune deletes every local - // snapshot, so the tracker forgets the latest one unconditionally. The - // snapshot this checkpoint just took is safe: it stays in - // CheckpointStateDir until after the prune, and only moveLocalCheckpoint - // moves it into LocalCheckpointsDir, the directory the prune empties. - s.snapshots.ForgetLocal(actorUID) - - // No earlier pause snapshot can ever be restored again, so remove them - // all: the actor's current state was just captured by CheckpointWorkload, - // and the control plane tracks only a single local snapshot, which this - // checkpoint either overwrites (pause) or clears (suspend). - // - // Do not move this above CheckpointWorkload to keep MergeDeltaIntoBase on its - // in-place path: that leaves the whole checkpoint window with no local snapshot - // while LocalSnapshotInfo still names the pruned one, and a crash there strands - // the actor for good (a paused actor has no uploaded copy for resume to fall - // back to, RequiredNodes pins it to this node, nothing clears the field). - // - // Best-effort: if this fail, the actor's terminate prunes again. - if err := pruneLocalCheckpoints(ctx, actorUID); err != nil { - slog.WarnContext(ctx, "failed to prune superseded local checkpoints", slog.Any("actor", actorRef), slog.Any("err", err)) - } - - // Pruning stays outside the persist window: it collects superseded - // snapshots on both paths, so timing it as part of an external upload would - // mix local disk deletion into the object-storage measurement. tPersist := time.Now() var state *ateletpb.ActorSnapshotState switch mode := req.GetUploadConfig().GetMode(); mode { @@ -911,21 +1001,55 @@ func (s *Atelet) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest dPersist = time.Since(tPersist) return nil, fmt.Errorf("while uploading external snapshot: %w", err) } - snap.UploadTime = time.Now() - state = s.snapshots.CommitDurable(actorUID, snap) + durable := snap + durable.UploadTime = time.Now() + state = s.snapshots.CommitDurable(actorUID, durable) + // The snapshot is durable now, so the local copy only saves a + // download: failing to keep it must not fail the checkpoint. The prune + // below deletes a partial copy. + if err := s.moveLocalCheckpoint(ctx, req, sandboxRec); err != nil { + slog.WarnContext(ctx, "Failed to keep a local copy of the uploaded snapshot", slog.Any("actor", actorRef), slog.Any("err", err)) + } else { + state = s.snapshots.CommitLocal(actorUID, snap) + } default: return nil, fmt.Errorf("unexpected upload mode: %v", mode) } dPersist = time.Since(tPersist) + // Delete the actor's earlier local snapshots only now that this checkpoint + // has committed its own. A checkpoint that fails before then leaves the + // previous local snapshot in place and recorded as the actor's latest, so + // the actor can still be restored from it on this node. + // + // The prune keeps what LocalSnapshotsToKeep names: the snapshot this + // checkpoint kept, and an older one a background upload is still reading or + // will retry, which is deleted once the upload no longer needs it. + // + // Do not move this above CheckpointWorkload to keep MergeDeltaIntoBase on its + // in-place path: that leaves the whole checkpoint window with no local snapshot + // while LocalSnapshotInfo still names the pruned one, and a crash there strands + // the actor for good (a paused actor has no uploaded copy for resume to fall + // back to, RequiredNodes pins it to this node, nothing clears the field). + // + // Best-effort: if this fails, the actor's next checkpoint or terminate + // prunes again. + tCleanup := time.Now() + if err := snapshottracker.PruneLocalCheckpoints(ctx, actorUID, s.snapshots.LocalSnapshotsToKeep(actorUID)...); err != nil { + slog.WarnContext(ctx, "failed to prune superseded local checkpoints", slog.Any("actor", actorRef), slog.Any("err", err)) + } + if err := s.unmountExternalVolumes(ctx, actorUID, req.GetSpec().GetVolumes()); err != nil { + dCleanup += time.Since(tCleanup) return nil, fmt.Errorf("while unmounting external volumes: %w", err) } // Note: we do not crash the actor if resetting the directory fails. if err := resetActorDirs(actorUID); err != nil { + dCleanup += time.Since(tCleanup) return nil, fmt.Errorf("while resetting actor dirs: %w", err) } + dCleanup += time.Since(tCleanup) return &ateletpb.CheckpointResponse{State: state}, nil } @@ -1038,7 +1162,7 @@ func (s *Atelet) uploadExternalCheckpoint(ctx context.Context, req *ateletpb.Che if err != nil { return err } - return s.uploadSnapshot(ctx, uri, checkpointDir, rec, req.GetActorTemplateAtespace(), req.GetActorTemplateName(), grpc.WaitForReady(true)) + return s.uploadSnapshot(ctx, uri, checkpointDir, rec, grpc.WaitForReady(true)) } // uploadSnapshot uploads rec's snapshot files from srcDir to uri through the @@ -1048,20 +1172,7 @@ func (s *Atelet) uploadExternalCheckpoint(ctx context.Context, req *ateletpb.Che // files, never a manifest pointing at files that never landed; retries // overwrite the deterministic object names. opts are passed to every plugin // call, and plugin errors are returned as uploadSnapshotFiles returns them. -func (s *Atelet) uploadSnapshot(ctx context.Context, uri resources.SnapshotURI, srcDir string, rec *sandboxAssetsRecord, templateAtespace, templateName string, opts ...grpc.CallOption) error { - root, err := os.OpenRoot(srcDir) - if err != nil { - return fmt.Errorf("while opening snapshot directory: %w", err) - } - defer root.Close() - - for _, fileName := range rec.SnapshotFiles { - info, err := root.Lstat(fileName) - if err != nil { - return fmt.Errorf("while inspecting %s in snapshot directory: %w", fileName, err) - } - recordSnapshotSize(ctx, fileName, allocatedBytes(info), templateAtespace, templateName) - } +func (s *Atelet) uploadSnapshot(ctx context.Context, uri resources.SnapshotURI, srcDir string, rec *sandboxAssetsRecord, opts ...grpc.CallOption) error { if err := s.uploadSnapshotFiles(ctx, uri.String(), srcDir, rec.SnapshotFiles, opts...); err != nil { return fmt.Errorf("while uploading snapshot files: %w", err) } @@ -1091,8 +1202,7 @@ func (s *Atelet) PromoteSnapshot(ctx context.Context, req *ateletpb.PromoteSnaps op := snapshotOp{ templateNamespace: req.GetActorTemplateAtespace(), templateName: req.GetActorTemplateName(), - // Always the actor's durable latest: golden actors are never paused - // (validation above rejects the golden atespace). + // Uploading a paused checkpoint creates a durable latest snapshot. kind: ateattr.SnapshotKindLatest, fidelity: ateattr.SnapshotFidelityValue(req.GetSnapshot().GetFidelity()), } @@ -1108,75 +1218,100 @@ func (s *Atelet) PromoteSnapshot(ctx context.Context, req *ateletpb.PromoteSnaps } localDir := ateletpath.LocalSnapshotDir(req.GetActorUid(), req.GetSnapshot().GetSnapshotUuid()) - tPersist := time.Now() - sandboxClass, err := s.uploadLocalCheckpointDir(ctx, req, localDir, uri) - dPersist = time.Since(tPersist) - op.sandboxClass = sandboxClass + // A detach or a background upload of this snapshot in progress finishes + // first, and no detach starts until the promote returns. + snap := snapshottracker.SnapshotRef{Snapshot: req.GetSnapshot(), Actor: resources.ActorRef{Atespace: req.GetAtespace(), Name: req.GetActorName()}} + alreadyDurable, endPromote, err := s.snapshots.BeginPromote(ctx, req.GetActorUid(), snap) if err != nil { return nil, err } + defer endPromote() - // Record the upload before the prune below deletes the local copy, so the - // tracker never names a deleted snapshot. - state := s.snapshots.CommitPromoted(req.GetActorUid(), snapshottracker.SnapshotRef{Snapshot: req.GetSnapshot(), UploadTime: time.Now()}) - - // Best-effort: the snapshot is in object storage now and the actor will - // not resume from this node's local copy. Kept outside the persist window - // so local disk deletion does not mix into the upload measurement. - if err := pruneLocalCheckpoints(ctx, req.GetActorUid()); err != nil { - slog.WarnContext(ctx, "failed to prune uploaded local checkpoints", slog.String("actorUID", req.GetActorUid()), slog.Any("err", err)) + if !alreadyDurable { + tPersist := time.Now() + sandboxClass, uploaded, err := s.uploadLocalCheckpointDir(ctx, req, localDir, uri) + dPersist = time.Since(tPersist) + op.sandboxClass = sandboxClass + if err != nil { + return nil, err + } + // The upload time is known only when this call wrote the manifest, not + // when it found the snapshot committed already. + if uploaded { + snap.UploadTime = time.Now() + } } + state := s.snapshots.CommitPromoted(req.GetActorUid(), snap) return &ateletpb.PromoteSnapshotResponse{State: state}, nil } // uploadLocalCheckpointDir uploads the local checkpoint in localDir to uri. It // returns the sandbox class recorded in the snapshot manifest (empty when the -// manifest was not read). Parameterized by localDir for tests. -func (s *Atelet) uploadLocalCheckpointDir(ctx context.Context, req *ateletpb.PromoteSnapshotRequest, localDir string, uri resources.SnapshotURI) (string, error) { +// manifest was not read), and whether it uploaded: it does not when the local +// copy is gone but already uploaded. Parameterized by localDir for tests. +func (s *Atelet) uploadLocalCheckpointDir(ctx context.Context, req *ateletpb.PromoteSnapshotRequest, localDir string, uri resources.SnapshotURI) (sandboxClass string, uploaded bool, err error) { localSnapshotName := req.GetSnapshot().GetSnapshotUuid() manifest, err := readSnapshotManifest(localDir) if errors.Is(err, os.ErrNotExist) { - // The local snapshot is gone. A previous invocation may have uploaded - // and pruned it: the remote manifest is uploaded last, so its presence - // means the whole snapshot is committed and this retry already - // succeeded. Absent on both sides, the paused actor's state is - // unrecoverable. + // The local snapshot is gone. A detach may have uploaded and pruned + // it: the remote manifest is uploaded last, so its presence means the + // whole snapshot is committed. Absent on both sides, the local + // snapshot is unrecoverable. _, fetchErr := s.fetchManifest(ctx, uri.String()) if fetchErr == nil { slog.InfoContext(ctx, "Local snapshot already uploaded and pruned; nothing to do", slog.String("snapshot_uri", uri.String())) - return "", nil + return "", false, nil } if status.Code(fetchErr) == codes.NotFound { - return "", fmt.Errorf("local snapshot %q is gone and no uploaded copy exists: %w", + return "", false, fmt.Errorf("local snapshot %q is gone and no uploaded copy exists: %w", localSnapshotName, fetchErr) } - return "", fmt.Errorf("while probing for an already-uploaded snapshot manifest: %w", fetchErr) + return "", false, fmt.Errorf("while probing for an already-uploaded snapshot manifest: %w", fetchErr) } if err != nil { - return "", wrapFileSystemErr("while reading local snapshot manifest", err) + return "", false, wrapFileSystemErr("while reading local snapshot manifest", err) } rec, err := unmarshalSandboxRecord(manifest) if err != nil { - return "", err + return "", false, err } capturedFidelity := rec.Fidelity if capturedFidelity == "" { - return rec.SandboxClass, apierror.FailedPrecondition("local snapshot %q has no fidelity recorded in its manifest; resume and pause the actor again before suspending it", localSnapshotName) + return rec.SandboxClass, false, apierror.FailedPrecondition("local snapshot %q has no fidelity recorded in its manifest; resume and pause the actor again before suspending it", localSnapshotName) } // Pause and suspend capture the template's one preferred fidelity, and a // paused actor's template cannot change, so the upload is always of what // the pause captured. A mismatch means the store or the control plane // drifted; refuse rather than guess. if desiredFidelity := ateattr.SnapshotFidelityValue(req.GetSnapshot().GetFidelity()); capturedFidelity != desiredFidelity { - return rec.SandboxClass, apierror.FailedPrecondition("pause snapshot captured %s; upload asks for %s", capturedFidelity, desiredFidelity) + return rec.SandboxClass, false, apierror.FailedPrecondition("pause snapshot captured %s; upload asks for %s", capturedFidelity, desiredFidelity) } // The local snapshot stays until the upload succeeds, so a retry can // upload it again. - return rec.SandboxClass, pluginsocket.CallError(s.uploadSnapshot(ctx, uri, localDir, rec, req.GetActorTemplateAtespace(), req.GetActorTemplateName())) + if err := pluginsocket.CallError(s.uploadSnapshot(ctx, uri, localDir, rec)); err != nil { + return rec.SandboxClass, false, err + } + return rec.SandboxClass, true, nil +} + +// trackerStore is the snapshottracker.Store that uploads through s. +type trackerStore struct{ s *Atelet } + +// UploadLocalSnapshot uploads ref's local checkpoint directory with +// uploadLocalCheckpointDir. +func (st *trackerStore) UploadLocalSnapshot(ctx context.Context, actorUID string, ref snapshottracker.SnapshotRef) (bool, error) { + uri, err := resources.ParseSnapshotURI(ref.Snapshot.GetObject().GetSnapshotUri()) + if err != nil { + return false, apierror.InvalidArgument("%v", err) + } + localDir := ateletpath.LocalSnapshotDir(actorUID, ref.Snapshot.GetSnapshotUuid()) + req := &ateletpb.PromoteSnapshotRequest{Atespace: ref.Actor.Atespace, ActorName: ref.Actor.Name, ActorUid: actorUID, Snapshot: ref.Snapshot} + _, uploaded, err := st.s.uploadLocalCheckpointDir(ctx, req, localDir, uri) + return uploaded, err } // errUnusableLocalSnapshot marks a node-local snapshot that is present on disk @@ -1223,14 +1358,14 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res actorUID := req.GetActorUid() actorRef := resources.ActorRef{Atespace: req.GetAtespace(), Name: req.GetActorName()} - // The sandbox (binaries + pause image) that runs the restored workload - // comes from the request, resolved by the control plane from the - // ActorTemplate's SandboxConfig. The snapshot manifests only supply the - // files to restore and the actor identity. Resolved before any on-node - // work so an invalid request changes nothing. - runtimeRec, err := recordFromRequest(req.GetSandboxAssets()) + // The sandbox (binaries + pause image) that runs the restored workload is + // the runtime the control plane assigned the actor to, resolved through + // the index. The snapshot manifests only supply the files to restore and + // the actor identity. Resolved before any on-node work so an invalid + // request changes nothing. + runtimeRec, err := s.advertisedRuntime(ctx, req.GetSandboxRuntime()) if err != nil { - return nil, apierror.InvalidArgument("invalid sandbox_assets: %v", err) + return nil, err } // Per-step timing so we can attribute resume latency between the rustfs @@ -1268,6 +1403,14 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res snapshotLogAttrs(attribution, op, restoreDurationMetric, err, phases)...) }() + // A detach in progress finishes first; its local copy is then gone, and + // the restore reads snapshot.object. + endStart, err := s.snapshots.BeginStart(ctx, actorUID) + if err != nil { + return nil, err + } + defer endStart() + // Not crashing the actor, because terminal errors here indicate problems with atelet, // node or the disk itself. s.systemInfoVolumes.Deregister(actorUID) @@ -1345,8 +1488,7 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res dManifest = time.Since(tManifest) manifestDone = true - // The manifest is what tells a golden restore from a latest one, so the - // snapshot kind only becomes knowable here. + // The snapshot kind is recorded once the manifest has been read. op.kind = restoreSnapshotKind(directLocal, sandboxRec) var registration *registeredActor @@ -1525,6 +1667,9 @@ func (s *Atelet) Terminate(ctx context.Context, req *ateletpb.TerminateRequest) } // The actor is gone, so no pause snapshot of it can ever be restored again. + // Forget cancels the actor's background upload or detach in progress + // without waiting for it. The cancelled operation leaves the actor's files + // alone, unless it was already deleting them, as this does too. // TODO(#664): this only removes local snapshots in one node. We should clean // up the copies on any other NodeVmsWithLocalSnapshots. This is fine *as of // the day this was written* because today NodeVmsWithLocalSnapshots has at @@ -1532,7 +1677,7 @@ func (s *Atelet) Terminate(ctx context.Context, req *ateletpb.TerminateRequest) if err := s.snapshots.Forget(actorUID); err != nil { return nil, fmt.Errorf("failed to delete snapshot state during terminate (actor: %s, actorUID: %s): %w", actorRef, actorUID, err) } - if err := pruneLocalCheckpoints(ctx, actorUID); err != nil { + if err := snapshottracker.PruneLocalCheckpoints(ctx, actorUID); err != nil { return nil, fmt.Errorf("failed to prune local checkpoints during terminate (actor: %s, actorUID: %s): %w", actorRef, actorUID, err) } diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index d040c0bf2d..97c4099583 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -37,6 +37,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/spf13/pflag" "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/cmd/atelet/internal/snapshottracker" "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/ateattr" "go.ate.dev/substrate/internal/atelet" @@ -79,6 +80,87 @@ func TestPortFlagDefault(t *testing.T) { } } +// The snapshot delay flags must be positive, and the upload delay must not +// exceed the detach delay, which the defaults satisfy too. +func TestValidateSnapshotDelayFlags(t *testing.T) { + setFlags := func(upload, detach, giveUp time.Duration) { + *snapshotUploadDelay, *snapshotDetachDelay, *snapshotReportGiveUpAfter = upload, detach, giveUp + } + t.Cleanup(func() { + setFlags(snapshottracker.DefaultUploadDelay, snapshottracker.DefaultDetachDelay, snapshottracker.DefaultReportGiveUpAfter) + }) + + for _, tc := range []struct { + name string + upload, detach, giveUp time.Duration + wantErr bool + }{ + {"defaults", snapshottracker.DefaultUploadDelay, snapshottracker.DefaultDetachDelay, snapshottracker.DefaultReportGiveUpAfter, false}, + {"upload sooner than detach", time.Minute, time.Hour, time.Minute, false}, + {"upload as late as detach", time.Hour, time.Hour, time.Minute, false}, + {"upload later than detach", time.Hour, time.Minute, time.Minute, true}, + {"zero upload delay", 0, time.Hour, time.Minute, true}, + {"negative detach delay", time.Minute, -time.Minute, time.Minute, true}, + {"zero report give-up delay", time.Minute, time.Hour, 0, true}, + {"negative report give-up delay", time.Minute, time.Hour, -time.Minute, true}, + } { + t.Run(tc.name, func(t *testing.T) { + setFlags(tc.upload, tc.detach, tc.giveUp) + if err := validateSnapshotDelayFlags(); (err != nil) != tc.wantErr { + t.Errorf("upload=%v detach=%v giveUp=%v: err = %v, wantErr %v", tc.upload, tc.detach, tc.giveUp, err, tc.wantErr) + } + }) + } +} + +// Upload and detach worker flags must be at least 1, report workers must be at +// least 0, and each flag defaults to the tracker's default count. +func TestValidateSnapshotWorkerFlags(t *testing.T) { + setFlags := func(upload, detach, report int) { + *snapshotUploadWorkers, *snapshotDetachWorkers, *snapshotReportWorkers = upload, detach, report + } + t.Cleanup(func() { + setFlags(snapshottracker.DefaultUploadWorkers, snapshottracker.DefaultDetachWorkers, snapshottracker.DefaultReportWorkers) + }) + + for _, tc := range []struct { + name string + upload, detach, report int + wantErr bool + }{ + {"defaults", snapshottracker.DefaultUploadWorkers, snapshottracker.DefaultDetachWorkers, snapshottracker.DefaultReportWorkers, false}, + {"single worker each", 1, 1, 1, false}, + {"zero report workers", 2, 2, 0, false}, + {"zero upload workers", 0, 2, 1, true}, + {"negative upload workers", -1, 2, 1, true}, + {"zero detach workers", 2, 0, 1, true}, + {"negative detach workers", 2, -1, 1, true}, + {"negative report workers", 2, 2, -1, true}, + } { + t.Run(tc.name, func(t *testing.T) { + setFlags(tc.upload, tc.detach, tc.report) + if err := validateSnapshotWorkerFlags(); (err != nil) != tc.wantErr { + t.Errorf("upload=%d detach=%d report=%d: err = %v, wantErr %v", tc.upload, tc.detach, tc.report, err, tc.wantErr) + } + }) + } + + for name, want := range map[string]int{ + "snapshot-upload-workers": snapshottracker.DefaultUploadWorkers, + "snapshot-detach-workers": snapshottracker.DefaultDetachWorkers, + "snapshot-report-workers": snapshottracker.DefaultReportWorkers, + } { + f := pflag.Lookup(name) + if f == nil { + t.Errorf("no --%s flag registered", name) + continue + } + if f.DefValue != strconv.Itoa(want) { + t.Errorf("--%s default = %q, want %d", name, f.DefValue, want) + } + } +} + // TestCopyLocalCheckpointLinks covers staging a local checkpoint into the restore // dir: the files must land as extra links to the cached snapshot rather than // copies, so a resume does not rewrite the image's working set. Sharing the inode @@ -391,7 +473,7 @@ func TestUploadSnapshotRejectsSymlinkOutsideRoot(t *testing.T) { } store := &recordingObjectStorage{} err = newPluginHerder(t, store).uploadSnapshot(context.Background(), uri, checkpointDir, - &sandboxAssetsRecord{SnapshotFiles: []string{"checkpoint.img"}}, "test", "test") + &sandboxAssetsRecord{SnapshotFiles: []string{"checkpoint.img"}}) if err == nil { t.Fatal("uploadSnapshot() followed a symlink outside the checkpoint directory") } @@ -1368,7 +1450,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { "config.json": "cfg", "memory-ranges": "mem", "data.tar": "data", }) - if _, err := s.uploadLocalCheckpointDir(ctx, validPromoteSnapshotRequest(), dir, uri); err != nil { + if _, _, err := s.uploadLocalCheckpointDir(ctx, validPromoteSnapshotRequest(), dir, uri); err != nil { t.Fatalf("uploadLocalCheckpointDir: %v", err) } want := []string{ @@ -1399,7 +1481,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { req := validPromoteSnapshotRequest() req.Snapshot.Fidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS - if _, err := s.uploadLocalCheckpointDir(ctx, req, dir, uri); err != nil { + if _, _, err := s.uploadLocalCheckpointDir(ctx, req, dir, uri); err != nil { t.Fatalf("uploadLocalCheckpointDir: %v", err) } want := []string{ @@ -1439,7 +1521,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { req := validPromoteSnapshotRequest() req.Snapshot.Fidelity = tc.desired - _, err := s.uploadLocalCheckpointDir(ctx, req, dir, uri) + _, _, err := s.uploadLocalCheckpointDir(ctx, req, dir, uri) if got := apierror.Code(err); got != codes.FailedPrecondition { t.Fatalf("status.Code = %v (err %v), want FailedPrecondition", got, err) } @@ -1461,7 +1543,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { req := validPromoteSnapshotRequest() req.Snapshot.Fidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES - _, err := s.uploadLocalCheckpointDir(ctx, req, dir, uri) + _, _, err := s.uploadLocalCheckpointDir(ctx, req, dir, uri) if got := apierror.Code(err); got != codes.FailedPrecondition { t.Fatalf("status.Code = %v (err %v), want FailedPrecondition for a scope-less manifest", got, err) } @@ -1476,7 +1558,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { }} s := newPluginHerder(t, store) - if _, err := s.uploadLocalCheckpointDir(ctx, validPromoteSnapshotRequest(), filepath.Join(t.TempDir(), "never-created"), uri); err != nil { + if _, _, err := s.uploadLocalCheckpointDir(ctx, validPromoteSnapshotRequest(), filepath.Join(t.TempDir(), "never-created"), uri); err != nil { t.Fatalf("uploadLocalCheckpointDir: %v", err) } }) @@ -1484,7 +1566,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { t.Run("gone locally and remotely crashes the actor", func(t *testing.T) { s := newPluginHerder(t, &recordingObjectStorage{}) - _, err := s.uploadLocalCheckpointDir(ctx, validPromoteSnapshotRequest(), filepath.Join(t.TempDir(), "never-created"), uri) + _, _, err := s.uploadLocalCheckpointDir(ctx, validPromoteSnapshotRequest(), filepath.Join(t.TempDir(), "never-created"), uri) if err == nil { t.Fatal("uploadLocalCheckpointDir succeeded, want error") } @@ -1500,7 +1582,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { "config.json": "cfg", "memory-ranges": "mem", "data.tar": "data", }) - _, err := s.uploadLocalCheckpointDir(ctx, validPromoteSnapshotRequest(), dir, uri) + _, _, err := s.uploadLocalCheckpointDir(ctx, validPromoteSnapshotRequest(), dir, uri) if err == nil { t.Fatal("uploadLocalCheckpointDir succeeded, want error") } @@ -1582,3 +1664,17 @@ func TestShouldHaveSnapshots(t *testing.T) { }) } } + +// TestServingCheck pins the readiness contract at startup: /readyz holds +// 503 until main has opened the gRPC port, whatever else is already up. +func TestServingCheck(t *testing.T) { + served := make(chan struct{}) + check := servingCheck(served) + if err := check(context.Background()); err == nil { + t.Error("servingCheck passed before the port was opened") + } + close(served) + if err := check(context.Background()); err != nil { + t.Errorf("servingCheck after the port opened = %v, want nil", err) + } +} diff --git a/cmd/atelet/metrics.go b/cmd/atelet/metrics.go index d7d65c2359..ed31bad2f5 100644 --- a/cmd/atelet/metrics.go +++ b/cmd/atelet/metrics.go @@ -24,7 +24,6 @@ import ( "go.ate.dev/substrate/internal/ateattr" "go.ate.dev/substrate/internal/proto/ateletpb" - "go.ate.dev/substrate/internal/resources" ) const ( @@ -154,9 +153,8 @@ func assetsAfterCollateral(prepFailedPhase string, assets time.Duration) time.Du } // restoreSnapshotKind classifies which snapshot a restore reads: local when it -// is served from the node-local copy, and otherwise golden or latest, told -// apart by the identity the manifest records for the actor that wrote the -// snapshot. An empty result means the manifest has not been read yet, so the +// is served from the node-local copy, and otherwise latest. An empty result +// means the manifest has not been read yet, so the // kind is not knowable. func restoreSnapshotKind(directLocal bool, rec *sandboxAssetsRecord) string { if directLocal { @@ -165,23 +163,14 @@ func restoreSnapshotKind(directLocal bool, rec *sandboxAssetsRecord) string { if rec == nil { return "" } - // Manifests written before the identity fields existed carry no atespace and - // fall through to latest, which is the common case for them anyway. - if rec.Atespace == resources.GoldenActorAtespace { - return ateattr.SnapshotKindGolden - } return ateattr.SnapshotKindLatest } // checkpointSnapshotKind classifies which snapshot a checkpoint writes: a pause -// writes the node-local one, a suspend the actor's durable latest, and a commit -// by an actor in the golden atespace the template's golden image. +// writes the node-local one, and a suspend writes a durable latest snapshot. func checkpointSnapshotKind(req *ateletpb.CheckpointRequest) string { if req.GetUploadConfig().GetMode() == ateletpb.UploadMode_UPLOAD_MODE_DEFERRED { return ateattr.SnapshotKindLocal } - if req.GetAtespace() == resources.GoldenActorAtespace { - return ateattr.SnapshotKindGolden - } return ateattr.SnapshotKindLatest } diff --git a/cmd/atelet/metrics_test.go b/cmd/atelet/metrics_test.go index 51508cd6ee..4145e6e91e 100644 --- a/cmd/atelet/metrics_test.go +++ b/cmd/atelet/metrics_test.go @@ -28,7 +28,6 @@ import ( "go.ate.dev/substrate/internal/ateattr" "go.ate.dev/substrate/internal/proto/ateletpb" - "go.ate.dev/substrate/internal/resources" ) const ( @@ -153,19 +152,83 @@ func TestCheckpointDurationShape(t *testing.T) { kind: ateattr.SnapshotKindLocal, fidelity: ateattr.SnapshotFidelityMemory, sandboxClass: "microvm", - }, phase{ateattr.SnapshotPhasePersist, time.Second}) + }, + phase{ateattr.SnapshotPhasePersist, time.Second}, + phase{ateattr.SnapshotPhaseCleanup, 20 * time.Millisecond}, + ) m := collectHistogram(t, reader, checkpointDurationMetric) if m.Unit != "s" { t.Errorf("unit = %q, want %q", m.Unit, "s") } - set := phaseValues(t, m)[ateattr.SnapshotPhasePersist] - if v := attrString(t, set, ateattr.SnapshotKindKey); v != ateattr.SnapshotKindLocal { - t.Errorf("snapshot kind = %q, want %q", v, ateattr.SnapshotKindLocal) + byPhase := phaseValues(t, m) + for _, wantPhase := range []string{ateattr.SnapshotPhasePersist, ateattr.SnapshotPhaseCleanup} { + set, ok := byPhase[wantPhase] + if !ok { + t.Fatalf("missing phase %q", wantPhase) + } + if v := attrString(t, set, ateattr.SnapshotKindKey); v != ateattr.SnapshotKindLocal { + t.Errorf("%s snapshot kind = %q, want %q", wantPhase, v, ateattr.SnapshotKindLocal) + } + if v := attrString(t, set, ateattr.SandboxClassKey); v != "microvm" { + t.Errorf("%s sandbox class = %q, want microvm", wantPhase, v) + } + } +} + +// TestCheckpointRecordsCleanup drives a Checkpoint: the node cleanup it does +// once the snapshot commits lands in the cleanup phase, a failed upload returns +// before it, and the phases fit in the total. +func TestCheckpointRecordsCleanup(t *testing.T) { + for _, tc := range []struct { + name string + putErr error + wantCleanup bool + }{ + {name: "succeeded", wantCleanup: true}, + {name: "upload failed", putErr: errors.New("object store unavailable")}, + } { + t.Run(tc.name, func(t *testing.T) { + a := runPluginActor(t) + var reader *sdkmetric.ManualReader + a.s.instruments, reader = newTestInstruments(t) + a.store.putErr = tc.putErr + _, err := a.checkpoint(t, a.snapshot("suspend-snap-1", 1), ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE) + if (err != nil) != (tc.putErr != nil) { + t.Fatalf("Checkpoint = %v, want an error only from the upload", err) + } + + sums := phaseSums(t, collectHistogram(t, reader, checkpointDurationMetric)) + if timed := sums[ateattr.SnapshotPhaseCleanup] > 0; timed != tc.wantCleanup { + t.Errorf("cleanup = %vs, want it timed: %v", sums[ateattr.SnapshotPhaseCleanup], tc.wantCleanup) + } + var parts float64 + for name, v := range sums { + if name != ateattr.SnapshotPhaseTotal { + parts += v + } + } + if total := sums[ateattr.SnapshotPhaseTotal]; parts > total { + t.Errorf("phases sum to %vs, more than the %vs total", parts, total) + } + }) + } +} + +// phaseSums maps each recorded phase to the sum of its observations. +func phaseSums(t *testing.T, m metricdata.Metrics) map[string]float64 { + t.Helper() + hist, ok := m.Data.(metricdata.Histogram[float64]) + if !ok { + t.Fatalf("%s is %T, want a float64 histogram", m.Name, m.Data) } - if v := attrString(t, set, ateattr.SandboxClassKey); v != "microvm" { - t.Errorf("sandbox class = %q, want microvm", v) + sums := make(map[string]float64, len(hist.DataPoints)) + for _, dp := range hist.DataPoints { + if v, ok := dp.Attributes.Value(ateattr.SnapshotPhaseKey); ok { + sums[v.AsString()] += dp.Sum + } } + return sums } // TestRecordPhasesSkipsZeroPhases pins the absence rule: a phase that never @@ -293,15 +356,15 @@ func TestRestoreSnapshotKind(t *testing.T) { want: ateattr.SnapshotKindLocal, }, { - name: "a golden actor's snapshot served locally is still local", + name: "a snapshot from another atespace served locally is still local", directLocal: true, - rec: &sandboxAssetsRecord{Atespace: resources.GoldenActorAtespace}, + rec: &sandboxAssetsRecord{Atespace: "team-b"}, want: ateattr.SnapshotKindLocal, }, { - name: "external snapshot written by a golden actor", - rec: &sandboxAssetsRecord{Atespace: resources.GoldenActorAtespace}, - want: ateattr.SnapshotKindGolden, + name: "external snapshot written in another atespace", + rec: &sandboxAssetsRecord{Atespace: "team-b"}, + want: ateattr.SnapshotKindLatest, }, { name: "external snapshot written by a tenant actor", @@ -347,13 +410,13 @@ func TestCheckpointSnapshotKind(t *testing.T) { want: ateattr.SnapshotKindLatest, }, { - name: "a golden actor's commit writes the template's golden", - req: &ateletpb.CheckpointRequest{UploadConfig: immediate, Atespace: resources.GoldenActorAtespace}, - want: ateattr.SnapshotKindGolden, + name: "a durable checkpoint in another atespace is latest", + req: &ateletpb.CheckpointRequest{UploadConfig: immediate, Atespace: "team-b"}, + want: ateattr.SnapshotKindLatest, }, { - name: "a local checkpoint in the golden atespace is still local", - req: &ateletpb.CheckpointRequest{UploadConfig: deferred, Atespace: resources.GoldenActorAtespace}, + name: "a local checkpoint in another atespace is still local", + req: &ateletpb.CheckpointRequest{UploadConfig: deferred, Atespace: "team-b"}, want: ateattr.SnapshotKindLocal, }, } diff --git a/cmd/atelet/runtime_prewarm.go b/cmd/atelet/runtime_prewarm.go new file mode 100644 index 0000000000..8d7bd5b9fd --- /dev/null +++ b/cmd/atelet/runtime_prewarm.go @@ -0,0 +1,235 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "sync" + "time" + + "k8s.io/client-go/util/workqueue" + + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/internal/imagecache" +) + +// defaultPrewarmAttemptTimeout bounds one prewarm attempt. The queue is +// drained by one worker, so an attempt that never returned would block +// every other runtime's prewarm. The asset downloads are bounded by the +// asset cache's own fetch timeout; this covers the pause image pull beside +// them. +const defaultPrewarmAttemptTimeout = 10 * time.Minute + +// prewarmMaxRetries bounds how often one runtime's failed prewarm is retried +// before it is dropped until a worker advertises it again (or first use, +// which stays the correctness path). The backoff below caps at 5 minutes, +// so this covers transient bucket or registry outages of several minutes +// without hammering a permanently broken runtime forever. +const prewarmMaxRetries = 8 + +// sandboxAssetFetcher is the one slice of Atelet the prewarmer needs. +// Prewarming through the same method as the Run/Restore path keeps the two +// fetches from ever diverging on cache layout or validation. +type sandboxAssetFetcher interface { + ensureSandboxAssets(ctx context.Context, rec *sandboxAssetsRecord, dstDir string) (map[string]string, error) +} + +// runtimePrewarmer fetches the default runtime of each worker on this node +// into the asset and image caches before any actor asks for it, so the +// fetch inside the first Run/Restore on the node is a cache hit instead of +// a download+extract on the critical path. The worker sandbox syncer hands +// it the default runtime hash of every worker it plans; the index holds +// the record to fetch. +// +// Prewarming is purely a latency optimization: every failure is logged and +// left to the on-demand fetch in ensureSandboxAssets and the pull inside +// prepareOCIBundles, which remain the correctness path. Only default +// runtimes are prewarmed; the first restore of another version a worker +// advertises pays its download. +type runtimePrewarmer struct { + index *runtimeIndex + assets sandboxAssetFetcher + images *imagecache.Store + // queue carries runtime hashes. It dedupes while an item waits, and it + // rate-limits retries after failures. A single worker drains it, which + // also serializes downloads so concurrent prewarms never compete for + // node bandwidth. + queue workqueue.TypedRateLimitingInterface[string] + // prewarmed holds the hashes whose prewarm succeeded since atelet + // started. The queue forgets an item once it is done, and every pod of + // a pool, every re-plan and every retry of a sync hands the same default + // over again; enqueue skips what is here, so a runtime is prewarmed + // once rather than re-staged and logged once per pod. Content-addressed + // hashes make that safe: the bytes a hash names never change. + mu sync.Mutex + prewarmed map[string]bool + // done is closed when the worker goroutine exits; wait blocks on it. + done chan struct{} + + // attemptTimeout is defaultPrewarmAttemptTimeout. Shortened in tests. + attemptTimeout time.Duration + // onFinished, if set, observes the result of each prewarm attempt, by + // runtime hash, once the worker's call has returned. Tests set it to + // wait for that call before tearing down what it writes into. + onFinished func(hash string, err error) +} + +func newRuntimePrewarmer(index *runtimeIndex, assets sandboxAssetFetcher, images *imagecache.Store) *runtimePrewarmer { + return &runtimePrewarmer{ + index: index, + assets: assets, + images: images, + attemptTimeout: defaultPrewarmAttemptTimeout, + // Downloads fail on the scale of network timeouts, not API conflicts, + // so back off in seconds and cap in minutes rather than the + // millisecond-based controller default. + queue: workqueue.NewTypedRateLimitingQueue( + workqueue.NewTypedItemExponentialFailureRateLimiter[string](time.Second, 5*time.Minute)), + prewarmed: map[string]bool{}, + done: make(chan struct{}), + } +} + +// enqueue asks for the runtime advertised under hash to be prewarmed, +// unless it already was. The syncer calls it as it plans a worker, after +// registering the hash in the index. +func (p *runtimePrewarmer) enqueue(hash string) { + p.mu.Lock() + already := p.prewarmed[hash] + p.mu.Unlock() + if already { + return + } + p.queue.Add(hash) +} + +// markPrewarmed records that hash's prewarm succeeded, so enqueue skips it +// from now on. +func (p *runtimePrewarmer) markPrewarmed(hash string) { + p.mu.Lock() + defer p.mu.Unlock() + p.prewarmed[hash] = true +} + +// run drains the queue until ctx ends. +func (p *runtimePrewarmer) run(ctx context.Context) { + defer close(p.done) + go func() { + <-ctx.Done() + p.queue.ShutDown() + }() + for { + hash, shutdown := p.queue.Get() + if shutdown { + return + } + p.process(ctx, hash) + } +} + +// wait blocks until the worker has exited, which happens after the context +// given to run is canceled and the in-flight item finishes. Tests wait here +// before tearing down what the worker writes into. +func (p *runtimePrewarmer) wait() { + <-p.done +} + +// process prewarms the runtime under hash, requeueing with backoff on +// failure. +func (p *runtimePrewarmer) process(ctx context.Context, hash string) { + defer p.queue.Done(hash) + rt, ok := p.index.lookup(hash) + if !ok { + // The syncer registers before it enqueues, and the index never + // forgets, so this is a programming error rather than a race. + slog.ErrorContext(ctx, "Sandbox runtime to prewarm is not in the index", slog.String("runtime_hash", hash)) + p.queue.Forget(hash) + return + } + err := p.prewarm(ctx, hash, rt) + if p.onFinished != nil { + p.onFinished(hash, err) + } + if err == nil { + p.markPrewarmed(hash) + p.queue.Forget(hash) + return + } + if ctx.Err() != nil { + // Shutting down, not a prewarm failure; drop without retry noise. + return + } + if retries := p.queue.NumRequeues(hash); retries < prewarmMaxRetries { + slog.WarnContext(ctx, "Sandbox runtime prewarm failed; will retry", + slog.String("runtime_hash", hash), slog.String("config", rt.config), slog.String("version", rt.version), + slog.Int("retries", retries), slog.Any("err", err)) + p.queue.AddRateLimited(hash) + return + } + // Best-effort: give up until a worker advertises it again or first use. + slog.WarnContext(ctx, "Sandbox runtime prewarm failed; giving up", + slog.String("runtime_hash", hash), slog.String("config", rt.config), slog.String("version", rt.version), + slog.Int("retries", prewarmMaxRetries), slog.Any("err", err)) + p.queue.Forget(hash) +} + +// prewarm fetches the runtime's assets into the asset cache and its pause +// image into the image cache. Racing an on-demand ensureSandboxAssets for +// the same assets is safe: the asset cache runs one download per asset and +// serves every caller from it, and the image cache collapses concurrent +// pulls of one digest. +// +// Prewarm has no actor to link the assets into, so it stages them into a +// scratch directory it removes again; the cache keeps the entries. +func (p *runtimePrewarmer) prewarm(ctx context.Context, hash string, rt *indexedRuntime) error { + ctx, cancel := context.WithTimeout(ctx, p.attemptTimeout) + defer cancel() + t := time.Now() + + var assetErr, imageErr error + var wg sync.WaitGroup + wg.Go(func() { + scratch, err := os.MkdirTemp(ateletpath.SandboxAssetScratchDir, "prewarm-") + if err != nil { + assetErr = fmt.Errorf("while creating prewarm scratch dir: %w", err) + return + } + defer func() { _ = os.RemoveAll(scratch) }() + _, assetErr = p.assets.ensureSandboxAssets(ctx, rt.rec, scratch) + }) + if rt.rec.PauseImage != "" { + wg.Go(func() { + if _, err := p.images.EnsureImage(ctx, rt.rec.PauseImage); err != nil { + imageErr = fmt.Errorf("while prewarming pause image %q: %w", rt.rec.PauseImage, err) + } + }) + } + wg.Wait() + if err := errors.Join(assetErr, imageErr); err != nil { + return err + } + slog.InfoContext(ctx, "Sandbox runtime prewarmed", + slog.String("runtime_hash", hash), + slog.String("config", rt.config), + slog.String("version", rt.version), + slog.Int("assets", len(rt.rec.Assets)), + slog.String("pause_image", rt.rec.PauseImage), + slog.Duration("duration", time.Since(t))) + return nil +} diff --git a/cmd/atelet/runtime_prewarm_test.go b/cmd/atelet/runtime_prewarm_test.go new file mode 100644 index 0000000000..509791eb23 --- /dev/null +++ b/cmd/atelet/runtime_prewarm_test.go @@ -0,0 +1,400 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "os" + "runtime" + "testing" + "time" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sruntime "k8s.io/apimachinery/pkg/runtime" + kubeinformers "k8s.io/client-go/informers" + kubefake "k8s.io/client-go/kubernetes/fake" + + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/internal/imagecache" + "go.ate.dev/substrate/pkg/api/v1alpha1" + atefake "go.ate.dev/substrate/pkg/client/clientset/versioned/fake" + "go.ate.dev/substrate/pkg/client/informers/externalversions" +) + +// pushPauseImage pushes a tiny image to ref and returns its manifest digest, +// so tests can later assert a cache hit by digest with the registry gone. +func pushPauseImage(t *testing.T, ref string) v1.Hash { + t.Helper() + img, err := mutate.AppendLayers(empty.Image, singleFileLayer(t, "pause", "pause bytes")) + if err != nil { + t.Fatalf("mutate.AppendLayers: %v", err) + } + digest, err := img.Digest() + if err != nil { + t.Fatalf("img.Digest: %v", err) + } + tag, err := name.ParseReference(ref, name.Insecure) + if err != nil { + t.Fatalf("name.ParseReference(%q): %v", ref, err) + } + if err := remote.Write(tag, img); err != nil { + t.Fatalf("remote.Write(%q): %v", ref, err) + } + return digest +} + +func gvisorConfig(name, url, sha string) *v1alpha1.SandboxConfig { + return &v1alpha1.SandboxConfig{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: v1alpha1.SandboxConfigSpec{ + SandboxClass: v1alpha1.SandboxClassGvisor, + DefaultVersion: "v1", + Versions: []v1alpha1.SandboxVersionConfig{{ + Name: "v1", + PauseImage: "registry.k8s.io/pause@sha256:abc", + Assets: map[string]map[string]v1alpha1.AssetFile{ + runtime.GOARCH: { + gvisorAssetName: {URL: url, SHA256: sha}, + }, + }, + }}, + }, + } +} + +// hangingObjectStorage blocks GetObject until the caller's context ends, +// simulating a bucket read that stalls without failing. +type hangingObjectStorage struct{} + +func (hangingObjectStorage) GetObject(ctx context.Context, _, _ string) (io.ReadCloser, error) { + <-ctx.Done() + return nil, ctx.Err() +} + +func (hangingObjectStorage) PutObject(_ context.Context, _, _ string, _ io.Reader) error { return nil } + +// pauseImageInTestRegistry pushes a pause image to a test registry and +// returns its digest-pinned reference, the only form a runtime hash accepts. +func pauseImageInTestRegistry(t *testing.T) (ref string, host string) { + t.Helper() + host = imageVolumeTestRegistry(t) + digest := pushPauseImage(t, host+"/pause:3.10") + return host + "/pause@" + digest.String(), host +} + +// indexedTestRuntime registers a runtime with one gvisor release asset, the +// tarball content, and the given pause image as a worker's default, and +// returns its hash. +func indexedTestRuntime(t *testing.T, index *runtimeIndex, content []byte, pauseImage string) string { + t.Helper() + rec := &sandboxAssetsRecord{ + SandboxClass: "gvisor", + PauseImage: pauseImage, + Assets: map[string]assetEntry{gvisorAssetName: {URL: "gs://bucket/gvisor.tar", SHA256: fmt.Sprintf("%x", sha256.Sum256(content))}}, + } + hash, err := versionBundleHash(rec) + if err != nil { + t.Fatal(err) + } + index.register("pod-a", advertisedClass{class: v1alpha1.SandboxClassGvisor, config: "gvisor-release", hashed: []hashedVersion{{ + versionPlan: versionPlan{name: "v1", isDefault: true, rec: rec}, hash: hash, + }}}) + return hash +} + +func newTestImageStore(t *testing.T) *imagecache.Store { + t.Helper() + store, err := imagecache.New(t.TempDir()) + if err != nil { + t.Fatalf("imagecache.New: %v", err) + } + return store +} + +// TestRuntimePrewarmProcess covers the worker's failure handling: a failed +// prewarm is requeued with backoff until prewarmMaxRetries and then dropped, +// and a hash that is not in the index is dropped at once. +func TestRuntimePrewarmProcess(t *testing.T) { + useTempAssetScratch(t) + ctx := context.Background() + index := newRuntimeIndex() + hash := indexedTestRuntime(t, index, gvisorReleaseTarball(t), "") + herder := &Atelet{sandboxAssetCache: newTestAssetCache(t), anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}} + p := newRuntimePrewarmer(index, herder, newTestImageStore(t)) + + for i := 1; i <= prewarmMaxRetries; i++ { + p.process(ctx, hash) + if got := p.queue.NumRequeues(hash); got != i { + t.Fatalf("NumRequeues after %d failed prewarms = %d, want %d (requeued with backoff)", i, got, i) + } + } + p.process(ctx, hash) + if got := p.queue.NumRequeues(hash); got != 0 { + t.Errorf("NumRequeues after giving up = %d, want 0 (forgotten)", got) + } + + p.process(ctx, "sha256:not-in-the-index") + if got := p.queue.NumRequeues("sha256:not-in-the-index"); got != 0 { + t.Errorf("NumRequeues for an unknown hash = %d, want 0 (forgotten)", got) + } +} + +// TestRuntimePrewarmOnce pins that a runtime is prewarmed once: a hash +// whose prewarm succeeded is skipped by every later enqueue, while one whose +// prewarm failed is queued again. +func TestRuntimePrewarmOnce(t *testing.T) { + useTempAssetScratch(t) + ctx := context.Background() + index := newRuntimeIndex() + release := gvisorReleaseTarball(t) + good := indexedTestRuntime(t, index, release, "") + // The store serves the same bytes for every asset, so this one's digest + // never matches and its prewarm fails. + bad := indexedTestRuntime(t, index, gvisorReleaseTarballWith(t, "another runsc"), "") + herder := &Atelet{sandboxAssetCache: newTestAssetCache(t), anonGCSClient: fakeObjectStorage{data: release}} + p := newRuntimePrewarmer(index, herder, newTestImageStore(t)) + + drain := func() { + for p.queue.Len() > 0 { + hash, _ := p.queue.Get() + p.process(ctx, hash) + } + } + p.enqueue(good) + p.enqueue(good) // a second pod of the pool: collapsed while it waits + if got := p.queue.Len(); got != 1 { + t.Fatalf("queue holds %d items after two enqueues of one hash, want 1", got) + } + drain() + p.enqueue(good) // a later pod, re-plan or retry: already prewarmed + if got := p.queue.Len(); got != 0 { + t.Errorf("queue holds %d items after enqueueing a prewarmed hash, want 0", got) + } + + p.enqueue(bad) + drain() + p.queue.Forget(bad) // drop the retry backoff for the test + p.enqueue(bad) + if got := p.queue.Len(); got != 1 { + t.Errorf("queue holds %d items after enqueueing a hash whose prewarm failed, want 1 (not skipped)", got) + } +} + +// TestRuntimePrewarmFillsBothCaches covers a successful prewarm: the asset +// lands in the asset cache with nothing left in the scratch dir, and the +// pause image lands in the image cache. +func TestRuntimePrewarmFillsBothCaches(t *testing.T) { + useTempAssetScratch(t) + ctx := context.Background() + pauseRef, _ := pauseImageInTestRegistry(t) + content := gvisorReleaseTarball(t) + index := newRuntimeIndex() + hash := indexedTestRuntime(t, index, content, pauseRef) + rt, _ := index.lookup(hash) + + root := t.TempDir() + images := newTestImageStore(t) + p := newRuntimePrewarmer(index, &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{data: content}}, images) + if err := p.prewarm(ctx, hash, rt); err != nil { + t.Fatalf("prewarm: %v", err) + } + if n := cachedAssets(t, root); n != 1 { + t.Errorf("asset cache holds %d assets after prewarm, want 1", n) + } + if left, err := os.ReadDir(ateletpath.SandboxAssetScratchDir); err != nil || len(left) != 0 { + t.Errorf("prewarm left %d scratch dirs behind (%v)", len(left), err) + } + if _, err := images.EnsureImage(ctx, pauseRef); err != nil { + t.Errorf("pause image not prewarmed into the image cache: %v", err) + } +} + +// TestRuntimePrewarmAssetFailureStillPullsImage pins that the two halves +// are independent: a failing asset fetch does not stop the pause image pull. +func TestRuntimePrewarmAssetFailureStillPullsImage(t *testing.T) { + useTempAssetScratch(t) + ctx := context.Background() + pauseRef, _ := pauseImageInTestRegistry(t) + index := newRuntimeIndex() + hash := indexedTestRuntime(t, index, gvisorReleaseTarball(t), pauseRef) + rt, _ := index.lookup(hash) + + images := newTestImageStore(t) + p := newRuntimePrewarmer(index, &Atelet{sandboxAssetCache: newTestAssetCache(t), anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}}, images) + if err := p.prewarm(ctx, hash, rt); err == nil { + t.Error("prewarm returned nil despite the asset fetch failing") + } + if _, err := images.EnsureImage(ctx, pauseRef); err != nil { + t.Errorf("pause image not pulled when the asset fetch failed: %v", err) + } +} + +// TestRuntimePrewarmAttemptTimeout verifies a single attempt is bounded by +// the prewarmer's attempt timeout: the queue has one worker, so an attempt +// that never returned would block every other runtime's prewarm. +func TestRuntimePrewarmAttemptTimeout(t *testing.T) { + useTempAssetScratch(t) + index := newRuntimeIndex() + hash := indexedTestRuntime(t, index, gvisorReleaseTarball(t), "") + rt, _ := index.lookup(hash) + p := newRuntimePrewarmer(index, &Atelet{sandboxAssetCache: newTestAssetCache(t), anonGCSClient: hangingObjectStorage{}}, newTestImageStore(t)) + p.attemptTimeout = 50 * time.Millisecond + + done := make(chan error, 1) + go func() { done <- p.prewarm(context.Background(), hash, rt) }() + select { + case err := <-done: + if err == nil { + t.Error("prewarm returned nil despite the download hanging") + } + case <-time.After(10 * time.Second): + t.Fatal("prewarm never returned; a hung download would block the worker forever") + } +} + +// TestSyncPodPrewarmsDefaultRuntime pins the hand-off: a worker's default +// runtime, and only that, is handed to the prewarmer, and before the build +// fetches anything for its probes, so the probes are the prewarm's head +// start. A build that then fails has still handed it over. +func TestSyncPodPrewarmsDefaultRuntime(t *testing.T) { + pod := workerPod("pod-a", "pool") + s := newTestSyncer(t, pod, resolvedPool("pool", "gvisor-release"), multiVersionConfig()) + var prewarmed []string + s.prewarm = func(hash string) { prewarmed = append(prewarmed, hash) } + s.assets = &fakeAssetFetcher{ + fn: func(ctx context.Context, rec *sandboxAssetsRecord) (map[string]string, error) { + if len(prewarmed) == 0 { + t.Error("the build fetched for a probe before the default runtime was handed to the prewarmer") + } + return newTestAssetFetcher().ensureSandboxAssets(ctx, rec, "") + }, + } + if err := s.syncPod(context.Background(), pod); err != nil { + t.Fatalf("syncPod: %v", err) + } + wr, ok := s.index.worker("pod-a") + if !ok { + t.Fatal("pod-a not indexed") + } + cr, _ := wr.class(v1alpha1.SandboxClassGvisor) + if len(prewarmed) != 1 || prewarmed[0] != cr.defaultHash { + t.Errorf("prewarmed %v, want the default runtime %s once", prewarmed, cr.defaultHash) + } + if len(cr.hashes) < 2 { + t.Fatalf("config advertises %d runtimes, want several so the test proves only the default is prewarmed", len(cr.hashes)) + } + + // A build that fails on the default's probe writes nothing, but the + // prewarm was queued before it started. + s = newTestSyncer(t, pod, resolvedPool("pool", "gvisor-release"), multiVersionConfig()) + prewarmed = nil + s.prewarm = func(hash string) { prewarmed = append(prewarmed, hash) } + s.runCPUFeatures = func(context.Context, string) ([]string, error) { return nil, errors.New("runsc failed") } + if err := s.syncPod(context.Background(), pod); err == nil { + t.Fatal("syncPod with the default's probe failing = nil, want an error") + } + if len(prewarmed) != 1 || prewarmed[0] != cr.defaultHash { + t.Errorf("prewarmed %v after a failed build, want the default runtime %s once", prewarmed, cr.defaultHash) + } + if _, ok := s.index.worker("pod-a"); ok { + t.Error("a failed build registered the worker") + } +} + +// TestDefaultRuntimePrewarmThroughSyncer runs the whole path: a worker pod, +// its pool and SandboxConfig in fake clientsets flow through the informers +// into the syncer, which indexes the pod's runtimes, writes its file, and +// hands its default runtime to the prewarmer, which lands the asset in the +// cache without any Run/Restore request. +func TestDefaultRuntimePrewarmThroughSyncer(t *testing.T) { + useTempAssetScratch(t) + // The worker's own prewarm call returns only when the asset has landed + // and the pause-image pull has finished. Waiting for the call keeps the + // pull's last writes from racing the TempDir cleanup. + prewarmed := make(chan error, 1) + onFinished := func(_ string, err error) { + // Non-blocking: a failed attempt is retried, and the worker must not + // block on a result nobody reads. + select { + case prewarmed <- err: + default: + } + } + + pauseRef, _ := pauseImageInTestRegistry(t) + content := gvisorReleaseTarball(t) + cfg := gvisorConfig("gvisor-release", "gs://bucket/gvisor.tar", fmt.Sprintf("%x", sha256.Sum256(content))) + cfg.Spec.Versions[0].PauseImage = pauseRef + + kube := kubefake.NewClientset([]k8sruntime.Object{workerPod("pod-a", "pool")}...) + ate := atefake.NewSimpleClientset(resolvedPool("pool", "gvisor-release"), cfg) + podFactory := kubeinformers.NewSharedInformerFactory(kube, 0) + ateFactory := externalversions.NewSharedInformerFactory(ate, 0) + ctx, cancel := context.WithCancel(context.Background()) + + root := t.TempDir() + index := newRuntimeIndex() + herder := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{data: content}} + p := newRuntimePrewarmer(index, herder, newTestImageStore(t)) + p.onFinished = onFinished + go p.run(ctx) + // Join the worker before the TempDir cleanups remove what it writes into. + t.Cleanup(func() { + cancel() + p.wait() + podFactory.Shutdown() + ateFactory.Shutdown() + }) + + s, err := startWorkerSandboxSyncer(ctx, + podFactory.Core().V1().Pods(), + ateFactory.Api().V1alpha1().WorkerPools(), + ateFactory.Api().V1alpha1().SandboxConfigs(), + newScriptAssetFetcher(t), // the syncer's runsc probe; prewarm goes through herder + t.TempDir(), index, p.enqueue) + if err != nil { + t.Fatalf("startWorkerSandboxSyncer: %v", err) + } + ateFactory.Start(ctx.Done()) + podFactory.Start(ctx.Done()) + + select { + case err := <-prewarmed: + if err != nil { + t.Fatalf("prewarm: %v", err) + } + case <-time.After(30 * time.Second): + t.Fatal("prewarm never finished") + } + if n := cachedAssets(t, root); n != 1 { + t.Fatalf("asset cache holds %d assets after prewarm, want 1", n) + } + // The prewarm is queued before the build probes, so it can finish + // before the file is written; the syncer catches up on its own. + waitFor(t, "the worker's runtimes file to be written", func() bool { return exists(s.runtimesFile("pod-a")) }) + waitFor(t, "the prewarm's and the probe's scratch dirs to be removed", func() bool { + left, err := os.ReadDir(ateletpath.SandboxAssetScratchDir) + return err == nil && len(left) == 0 + }) +} diff --git a/cmd/atelet/sandbox_assets.go b/cmd/atelet/sandbox_assets.go index b454cbb2c0..16b70b5f72 100644 --- a/cmd/atelet/sandbox_assets.go +++ b/cmd/atelet/sandbox_assets.go @@ -60,16 +60,11 @@ const sandboxManifestName = "manifest.json" // ponytail: 8GiB ceiling, make it a flag if a rootfs ever needs more. var maxAssetBytes int64 = 8 << 30 -const ( - // gvisorAssetName is the gVisor release tarball asset (gvisor.tar.zstd). The - // tarball carries `runsc` together with the `gvisor-bin/` helper binaries, - // so it is extracted into a content-addressed directory rather than as a - // single file. - gvisorAssetName = "gvisor" - - // runscAssetName is the legacy single-binary gVisor asset. - runscAssetName = "runsc" -) +// gvisorAssetName is the gVisor release tarball asset (gvisor.tar.zstd). The +// tarball carries `runsc` together with the `gvisor-bin/` helper binaries, +// so it is extracted into a content-addressed directory rather than as a +// single file. +const gvisorAssetName = "gvisor" // assetEntry is one content-addressed sandbox asset (url + sha256). type assetEntry struct { @@ -86,8 +81,7 @@ func (e assetEntry) displayURL() string { // sandboxAssetsRecord is the sandbox runtime an actor is running, projected onto // the local node's architecture: the sandbox class and pause image plus the // asset set keyed by asset name (gVisor uses a single "gvisor" release-tarball -// asset; records written before the tarball release mechanism use a bare -// "runsc" asset). +// asset). // It is both the per-actor on-node record (written at Run/Restore, read at // Checkpoint) and the snapshot manifest (written at Checkpoint, read at // Restore). @@ -159,8 +153,8 @@ func checkPauseImage(sandboxClass, pauseImage string) error { // ensureSandboxAssets stages every asset in the record under dstDir, named // by asset name, and returns a map of asset name to the local path the worker // runs it from. gVisor has a single "gvisor" release-tarball asset (staged -// as a directory, the path is its runsc) or a bare "runsc" binary; the -// micro-VM runtime has several (kata-shim, cloud-hypervisor, ...). +// as a directory, the path is its runsc); the micro-VM runtime has several +// (kata-shim, cloud-hypervisor, ...). // // Each asset is served from the node's content-addressed asset cache as a // hard link, so the download runs once per node, concurrent callers for one @@ -203,19 +197,10 @@ func plainAssetName(name string) bool { return name != "" && name != "." && name == filepath.Base(name) && filepath.IsLocal(name) } -// runscAssetNames are the assets that carry the gVisor `runsc` binary, most -// preferred first. -var runscAssetNames = []string{gvisorAssetName, runscAssetName} - // runscPathFor returns the local path of the gVisor `runsc` binary from a // fetched asset-path map, or "" if the runtime has none (e.g. micro-VM). func runscPathFor(paths map[string]string) string { - for _, name := range runscAssetNames { - if p := paths[name]; p != "" { - return p - } - } - return "" + return paths[gvisorAssetName] } // assetKey is the asset's cache key: its content hash, which the key diff --git a/cmd/atelet/sandbox_assets_test.go b/cmd/atelet/sandbox_assets_test.go index f2fc9a64ae..f7b7d3a23a 100644 --- a/cmd/atelet/sandbox_assets_test.go +++ b/cmd/atelet/sandbox_assets_test.go @@ -311,6 +311,13 @@ func stageTestAsset(t *testing.T, s *Atelet, root string, entry assetEntry) (str // gvisorReleaseTarball builds an uncompressed gVisor release tarball: runsc // next to a gvisor-bin/ helper, as the real release ships them. func gvisorReleaseTarball(t *testing.T) []byte { + t.Helper() + return gvisorReleaseTarballWith(t, "runsc bytes") +} + +// gvisorReleaseTarballWith is gvisorReleaseTarball with the given runsc +// contents, for releases that must differ in digest. +func gvisorReleaseTarballWith(t *testing.T, runsc string) []byte { t.Helper() var buf bytes.Buffer tw := tar.NewWriter(&buf) @@ -318,7 +325,7 @@ func gvisorReleaseTarball(t *testing.T) []byte { name, body string mode int64 }{ - {"runsc", "runsc bytes", 0o755}, + {"runsc", runsc, 0o755}, {"gvisor-bin/containerd-shim", "shim bytes", 0o755}, } { if err := tw.WriteHeader(&tar.Header{Name: f.name, Mode: f.mode, Size: int64(len(f.body))}); err != nil { @@ -334,6 +341,33 @@ func gvisorReleaseTarball(t *testing.T) []byte { return buf.Bytes() } +// gvisorTestRelease is a gVisor release tarball with the asset entry that +// names it: the one asset a gvisor sandbox record carries. +type gvisorTestRelease struct { + tarball []byte + entry assetEntry +} + +func newGVisorTestRelease(t *testing.T) gvisorTestRelease { + t.Helper() + tarball := gvisorReleaseTarball(t) + return gvisorTestRelease{ + tarball: tarball, + entry: assetEntry{URL: "gs://test-bucket/gvisor.tar", SHA256: fmt.Sprintf("%x", sha256.Sum256(tarball))}, + } +} + +// proto is the asset as a Run or Restore request carries it. +func (r gvisorTestRelease) proto() *ateletpb.AssetFile { + return &ateletpb.AssetFile{Url: r.entry.URL, Sha256: r.entry.SHA256} +} + +// stagedRunsc is where an actor's staged copy of the release's runsc sits +// under its sandbox assets dir. +func stagedRunsc(actorUID string) string { + return filepath.Join(ateletpath.SandboxAssetsDir(actorUID), gvisorAssetName, "runsc") +} + // TestStageGVisorRelease stages the release tree through the asset cache: // one download and extraction serves every actor, each gets its own // directory of links to the cached files, and the tarball itself is not diff --git a/cmd/atelet/sandbox_prewarm.go b/cmd/atelet/sandbox_prewarm.go deleted file mode 100644 index f1ee58068b..0000000000 --- a/cmd/atelet/sandbox_prewarm.go +++ /dev/null @@ -1,352 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "errors" - "fmt" - "log/slog" - "math/rand/v2" - "os" - "runtime" - "sync" - "time" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/client-go/tools/cache" - "k8s.io/client-go/util/workqueue" - - "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" - "go.ate.dev/substrate/internal/imagecache" - "go.ate.dev/substrate/pkg/api/v1alpha1" - listersv1alpha1 "go.ate.dev/substrate/pkg/client/listers/api/v1alpha1" -) - -// prewarmMaxJitter spreads the fleet's asset downloads after a SandboxConfig -// change. Every atelet observes a create/update within about a second, and -// without jitter they would all open the same bucket objects at once. The -// jitter is applied as the enqueue delay, so duplicate events arriving inside -// the window collapse into one queue entry instead of stacking waits in the -// worker. A var so tests can zero it. -var prewarmMaxJitter = 30 * time.Second - -// prewarmTimeout bounds a single prewarm attempt. The queue is drained by one -// worker, so without a deadline a download that hangs without failing (a -// registry that accepts the connection and stalls, a wedged bucket read) -// would block every other config forever — prewarm runs on the daemon's -// never-cancelled context. Generous enough for multi-hundred-MiB micro-VM -// guest images on a busy node; a timed-out attempt requeues with backoff like -// any other failure. A var so tests can shorten it. -var prewarmTimeout = 5 * time.Minute - -// prewarmMaxRetries bounds how often one config's failed prewarm is retried -// before it is dropped until the next config event (or first use, which stays -// the correctness path). The backoff below caps at 5 minutes, so this covers -// transient bucket or registry outages of several minutes without hammering a -// permanently broken config forever. -const prewarmMaxRetries = 8 - -// prewarmFinished observes the result of each prewarm attempt, by config -// name, once the worker's call has returned. A no-op in production; a var so -// a test can wait for the worker's own prewarm call to return before it tears -// down what that call writes into. -var prewarmFinished = func(string, error) {} - -// sandboxAssetFetcher is the one slice of Atelet the prewarmer needs. -// Prewarming through the same method as the Run/Restore path keeps the two -// fetches from ever diverging on cache layout or validation. -type sandboxAssetFetcher interface { - ensureSandboxAssets(ctx context.Context, rec *sandboxAssetsRecord, dstDir string) (map[string]string, error) -} - -// sandboxPrewarmer downloads SandboxConfig assets into the node's asset -// cache before any actor asks for them, so the fetch inside the first -// Run/Restore on the node is a cache hit instead of a download+extract on -// the critical path. -type sandboxPrewarmer struct { - assets sandboxAssetFetcher - images *imagecache.Store - // lister resolves a queued config name to its latest revision at - // processing time, so coalesced events never prewarm a stale spec. - lister listersv1alpha1.SandboxConfigLister - // queue decouples informer event handlers (which must not block) from the - // downloads, dedupes by config name so relists cannot stack duplicate - // work, and rate-limits retries after failures. A single worker drains - // it, which also serializes downloads so concurrent prewarms never - // compete for node bandwidth. - queue workqueue.TypedRateLimitingInterface[string] - // microvmCapable gates micro-VM configs: their guest images run to - // hundreds of MiB, and a node without /dev/kvm can never run that class - // (workers request the ate.dev/kvm extended resource, so they only - // schedule where the device exists). See microvmNodeCapable. - microvmCapable bool - // done is closed when the worker goroutine exits; wait blocks on it. - done chan struct{} -} - -func newSandboxPrewarmer(assets sandboxAssetFetcher, images *imagecache.Store, lister listersv1alpha1.SandboxConfigLister, microvmCapable bool) *sandboxPrewarmer { - return &sandboxPrewarmer{ - assets: assets, - images: images, - lister: lister, - // Downloads fail on the scale of network timeouts, not API conflicts, - // so back off in seconds and cap in minutes rather than the - // millisecond-based controller default. - queue: workqueue.NewTypedRateLimitingQueue( - workqueue.NewTypedItemExponentialFailureRateLimiter[string](time.Second, 5*time.Minute)), - microvmCapable: microvmCapable, - done: make(chan struct{}), - } -} - -// startSandboxAssetPrewarm registers an event handler on the SandboxConfig -// informer and starts a background worker that pre-downloads each config's -// sandbox assets for this node's architecture, and pulls its pause image into -// the image cache. Prewarming is purely a latency optimization: every failure -// is logged and left to the on-demand fetch in ensureSandboxAssets and the -// pull inside prepareOCIBundles, which remain the correctness path. -// -// TODO: the asset cache is never pruned, and prewarming every config -// revision makes stale releases accumulate faster. Add a GC that removes -// assets referenced by no current SandboxConfig and no on-node actor record. -func startSandboxAssetPrewarm(ctx context.Context, informer cache.SharedIndexInformer, assets sandboxAssetFetcher, images *imagecache.Store, microvmCapable bool) (*sandboxPrewarmer, error) { - p := newSandboxPrewarmer(assets, images, listersv1alpha1.NewSandboxConfigLister(informer.GetIndexer()), microvmCapable) - // Atelet startup never waits for this informer to sync: prewarm is - // best-effort, so a failing list/watch (e.g. Forbidden while an RBAC - // rollout lags the binary) must degrade prewarm, not hang the node. The - // handler makes that degradation visible in the log; the reflector keeps - // retrying and prewarm recovers with it. Setting it fails only on an - // already-started informer, where the default reflector logging applies. - if err := informer.SetWatchErrorHandler(func(_ *cache.Reflector, err error) { - slog.WarnContext(ctx, "SandboxConfig list/watch failed; sandbox asset prewarm degraded until it recovers", slog.Any("err", err)) - }); err != nil { - slog.InfoContext(ctx, "Could not set sandbox config watch error handler", slog.Any("err", err)) - } - // The initial List replays every existing SandboxConfig into the handler - // as an Add: a freshly booted node prewarms the current configs, not only - // future changes. - if _, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj any) { p.enqueue(ctx, obj) }, - UpdateFunc: func(_, obj any) { p.enqueue(ctx, obj) }, - }); err != nil { - return nil, fmt.Errorf("while registering sandbox config prewarm handler: %w", err) - } - go p.run(ctx) - slog.InfoContext(ctx, "Sandbox asset prewarm started", slog.Bool("microvmCapable", microvmCapable)) - return p, nil -} - -// wait blocks until the prewarm worker has exited, which happens after the -// context given to startSandboxAssetPrewarm is canceled and the in-flight -// item finishes. Tests that patch package globals the worker reads must wait -// here before restoring them. -func (p *sandboxPrewarmer) wait() { - <-p.done -} - -// skipConfig reports whether this node has nothing to prewarm for cfg, -// logging why. Both enqueue and process apply it: sandboxClass is mutable, -// so the class observed at enqueue time can be stale by the time the worker -// resolves the name after jitter or backoff, and the gate must hold for the -// revision actually prewarmed. -func (p *sandboxPrewarmer) skipConfig(ctx context.Context, cfg *v1alpha1.SandboxConfig) bool { - switch cfg.Spec.SandboxClass { - case v1alpha1.SandboxClassGvisor: - // Every node runs gVisor workers; always prewarm. - return false - case v1alpha1.SandboxClassMicroVM: - if !p.microvmCapable { - slog.DebugContext(ctx, "Skipping sandbox asset prewarm: node has no /dev/kvm, cannot run micro-VM workers", - slog.String("config", cfg.Name)) - return true - } - return false - default: - // An unknown class has no backend in this atelet (likely version skew - // with a newer control plane); nothing to prewarm. - slog.InfoContext(ctx, "Skipping sandbox asset prewarm: unknown sandbox class", - slog.String("config", cfg.Name), - slog.String("sandboxClass", string(cfg.Spec.SandboxClass))) - return true - } -} - -func (p *sandboxPrewarmer) enqueue(ctx context.Context, obj any) { - cfg, ok := obj.(*v1alpha1.SandboxConfig) - if !ok { - return - } - if p.skipConfig(ctx, cfg) { - return - } - if prewarmMaxJitter > 0 { - p.queue.AddAfter(cfg.Name, rand.N(prewarmMaxJitter)) - return - } - p.queue.Add(cfg.Name) -} - -func (p *sandboxPrewarmer) run(ctx context.Context) { - defer close(p.done) - go func() { - <-ctx.Done() - p.queue.ShutDown() - }() - for { - name, shutdown := p.queue.Get() - if shutdown { - return - } - p.process(ctx, name) - } -} - -// process prewarms the named config's current revision, requeueing with -// backoff on failure. The queue holds names, not objects, so an event that -// arrives while its config is being processed is simply requeued by the -// workqueue and prewarms the newer revision afterwards. -func (p *sandboxPrewarmer) process(ctx context.Context, name string) { - defer p.queue.Done(name) - cfg, err := p.lister.Get(name) - if apierrors.IsNotFound(err) { - // Deleted since it was enqueued; nothing to prewarm anymore. - p.queue.Forget(name) - return - } - if err == nil && p.skipConfig(ctx, cfg) { - // The class gate re-applies to the revision resolved now, which may - // differ from the one that passed enqueue's filter. - p.queue.Forget(name) - return - } - if err == nil { - err = p.prewarm(ctx, cfg) - prewarmFinished(name, err) - } - if err == nil { - p.queue.Forget(name) - return - } - if ctx.Err() != nil { - // Shutting down, not a prewarm failure; drop without retry noise. - return - } - if retries := p.queue.NumRequeues(name); retries < prewarmMaxRetries { - slog.WarnContext(ctx, "Sandbox asset prewarm failed; will retry", - slog.String("config", name), slog.Int("retries", retries), slog.Any("err", err)) - p.queue.AddRateLimited(name) - return - } - // Best-effort: give up until the next config event or first use. - slog.WarnContext(ctx, "Sandbox asset prewarm failed; giving up", - slog.String("config", name), slog.Int("retries", prewarmMaxRetries), slog.Any("err", err)) - p.queue.Forget(name) -} - -// prewarm fetches every asset of one SandboxConfig into the asset cache -// and its pause image into the image cache. Racing an on-demand -// ensureSandboxAssets for the same assets is safe: the asset cache runs one -// download per asset and serves every caller from it, and the image cache -// collapses concurrent pulls of one digest. -// -// Prewarm has no actor to link the assets into, so it stages them into a -// scratch directory it removes again; the cache keeps the entries. -func (p *sandboxPrewarmer) prewarm(ctx context.Context, cfg *v1alpha1.SandboxConfig) error { - ctx, cancel := context.WithTimeout(ctx, prewarmTimeout) - defer cancel() - t := time.Now() - - // TODO: prewarm every Enabled version, not just the default. - ver, ok := cfg.Spec.DefaultVersionConfig() - if !ok { - // The CRD rejects this shape, so this should not be reachable. - slog.WarnContext(ctx, "Skipping sandbox asset prewarm: config has no enabled default version", - slog.String("config", cfg.Name), slog.String("defaultVersion", cfg.Spec.DefaultVersion)) - return nil - } - - var imageErr error - var wg sync.WaitGroup - // schedule prewarm pause image if provided - if ver.PauseImage != "" { - wg.Go(func() { - if _, err := p.images.EnsureImage(ctx, ver.PauseImage); err != nil { - imageErr = fmt.Errorf("while prewarming pause image %q: %w", ver.PauseImage, err) - } - }) - } - // schedule prewarm sandbox assets if provided - rec, assetErr := recordFromSandboxConfig(cfg, ver) - switch { - case errors.Is(assetErr, errNoAssetsForArch): - // Permanent until the config changes, and a change re-enqueues: - // retrying would burn every attempt on a non-failure and redo the - // pause-image pull with it. Nothing to do for this node. - slog.DebugContext(ctx, "No sandbox assets to prewarm for this node's architecture", - slog.String("config", cfg.Name), slog.String("arch", runtime.GOARCH)) - assetErr = nil - case assetErr == nil: - wg.Go(func() { - scratch, err := os.MkdirTemp(ateletpath.SandboxAssetScratchDir, "prewarm-") - if err != nil { - assetErr = fmt.Errorf("while creating prewarm scratch dir: %w", err) - return - } - defer func() { _ = os.RemoveAll(scratch) }() - _, assetErr = p.assets.ensureSandboxAssets(ctx, rec, scratch) - }) - } - wg.Wait() - if err := errors.Join(assetErr, imageErr); err != nil { - return err - } - assets := 0 - if rec != nil { - assets = len(rec.Assets) - } - slog.InfoContext(ctx, "Sandbox assets prewarmed", - slog.String("config", cfg.Name), - slog.String("version", ver.Name), - slog.Int("assets", assets), - slog.String("pauseImage", ver.PauseImage), - slog.Duration("duration", time.Since(t))) - return nil -} - -// errNoAssetsForArch reports that a SandboxConfig lists no assets for this -// node's architecture. Unlike a failed download it is permanent until the -// config changes, so prewarm treats it as nothing-to-do rather than retrying. -var errNoAssetsForArch = errors.New("no sandbox assets for this architecture") - -// recordFromSandboxConfig projects the per-architecture assets of one version -// of a SandboxConfig onto the local node's architecture, mirroring -// recordFromRequest. -func recordFromSandboxConfig(cfg *v1alpha1.SandboxConfig, ver *v1alpha1.SandboxVersionConfig) (*sandboxAssetsRecord, error) { - arch := runtime.GOARCH - files := ver.Assets[arch] - if len(files) == 0 { - return nil, fmt.Errorf("sandbox config %q, version %q, architecture %q: %w", cfg.Name, ver.Name, arch, errNoAssetsForArch) - } - rec := &sandboxAssetsRecord{ - SandboxClass: string(cfg.Spec.SandboxClass), - PauseImage: ver.PauseImage, - Assets: make(map[string]assetEntry, len(files)), - } - for name, f := range files { - rec.Assets[name] = assetEntry{URL: f.URL, SHA256: f.SHA256} - } - return rec, nil -} diff --git a/cmd/atelet/sandbox_prewarm_test.go b/cmd/atelet/sandbox_prewarm_test.go deleted file mode 100644 index 820e7be022..0000000000 --- a/cmd/atelet/sandbox_prewarm_test.go +++ /dev/null @@ -1,460 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "crypto/sha256" - "errors" - "fmt" - "io" - "log" - "net/http/httptest" - "net/url" - "os" - "runtime" - "testing" - "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/cache" - "k8s.io/utils/ptr" - - "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/registry" - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/empty" - "github.com/google/go-containerregistry/pkg/v1/mutate" - "github.com/google/go-containerregistry/pkg/v1/remote" - "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" - "go.ate.dev/substrate/internal/imagecache" - "go.ate.dev/substrate/pkg/api/v1alpha1" - "go.ate.dev/substrate/pkg/client/clientset/versioned/fake" - "go.ate.dev/substrate/pkg/client/informers/externalversions" - listersv1alpha1 "go.ate.dev/substrate/pkg/client/listers/api/v1alpha1" -) - -// pushPauseImage pushes a tiny image to ref and returns its manifest digest, -// so tests can later assert a cache hit by digest with the registry gone. -func pushPauseImage(t *testing.T, ref string) v1.Hash { - t.Helper() - img, err := mutate.AppendLayers(empty.Image, singleFileLayer(t, "pause", "pause bytes")) - if err != nil { - t.Fatalf("mutate.AppendLayers: %v", err) - } - digest, err := img.Digest() - if err != nil { - t.Fatalf("img.Digest: %v", err) - } - tag, err := name.ParseReference(ref, name.Insecure) - if err != nil { - t.Fatalf("name.ParseReference(%q): %v", ref, err) - } - if err := remote.Write(tag, img); err != nil { - t.Fatalf("remote.Write(%q): %v", ref, err) - } - return digest -} - -func gvisorConfig(name, url, sha string) *v1alpha1.SandboxConfig { - return &v1alpha1.SandboxConfig{ - ObjectMeta: metav1.ObjectMeta{Name: name}, - Spec: v1alpha1.SandboxConfigSpec{ - SandboxClass: v1alpha1.SandboxClassGvisor, - DefaultVersion: "v1", - Versions: []v1alpha1.SandboxVersionConfig{{ - Name: "v1", - PauseImage: "registry.k8s.io/pause@sha256:abc", - Assets: map[string]map[string]v1alpha1.AssetFile{ - runtime.GOARCH: { - runscAssetName: {URL: url, SHA256: sha}, - }, - }, - }}, - }, - } -} - -func TestRecordFromSandboxConfig(t *testing.T) { - sha := fmt.Sprintf("%x", sha256.Sum256([]byte("runsc"))) - cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", sha) - - rec, err := recordFromSandboxConfig(cfg, &cfg.Spec.Versions[0]) - if err != nil { - t.Fatalf("recordFromSandboxConfig: %v", err) - } - if rec.SandboxClass != string(v1alpha1.SandboxClassGvisor) { - t.Errorf("SandboxClass = %q, want %q", rec.SandboxClass, v1alpha1.SandboxClassGvisor) - } - if rec.PauseImage != cfg.Spec.Versions[0].PauseImage { - t.Errorf("PauseImage = %q, want %q", rec.PauseImage, cfg.Spec.Versions[0].PauseImage) - } - want := assetEntry{URL: "gs://bucket/runsc", SHA256: sha} - if got := rec.Assets[runscAssetName]; got != want { - t.Errorf("Assets[%q] = %+v, want %+v", runscAssetName, got, want) - } - - // A config with no assets for this node's architecture cannot be - // projected, and the error carries the sentinel that keeps prewarm from - // retrying a condition only a config change can clear. - cfg.Spec.Versions[0].Assets = map[string]map[string]v1alpha1.AssetFile{ - "other-arch": {runscAssetName: {URL: "gs://bucket/runsc", SHA256: sha}}, - } - if _, err := recordFromSandboxConfig(cfg, &cfg.Spec.Versions[0]); !errors.Is(err, errNoAssetsForArch) { - t.Errorf("recordFromSandboxConfig with no assets for the local architecture = %v, want errNoAssetsForArch", err) - } -} - -func TestPrewarmEnqueueFilters(t *testing.T) { - origJitter := prewarmMaxJitter - prewarmMaxJitter = 0 // enqueue synchronously so Len is observable - t.Cleanup(func() { prewarmMaxJitter = origJitter }) - - ctx := context.Background() - microvm := &v1alpha1.SandboxConfig{ - ObjectMeta: metav1.ObjectMeta{Name: "microvm-default"}, - Spec: v1alpha1.SandboxConfigSpec{SandboxClass: v1alpha1.SandboxClassMicroVM}, - } - gvisor := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256([]byte("runsc")))) - - t.Run("node without KVM", func(t *testing.T) { - p := newSandboxPrewarmer(nil, nil, nil, false) - - p.enqueue(ctx, "not a sandbox config") - p.enqueue(ctx, microvm) - p.enqueue(ctx, &v1alpha1.SandboxConfig{ - ObjectMeta: metav1.ObjectMeta{Name: "future-class"}, - Spec: v1alpha1.SandboxConfigSpec{SandboxClass: "future-class"}, - }) - if p.queue.Len() != 0 { - t.Fatalf("queue holds %d configs after filtered enqueues, want 0", p.queue.Len()) - } - - p.enqueue(ctx, gvisor) - if p.queue.Len() != 1 { - t.Fatalf("queue holds %d configs after gvisor enqueue, want 1", p.queue.Len()) - } - // Duplicate events (e.g. a relist) must coalesce, not stack work. - p.enqueue(ctx, gvisor) - if p.queue.Len() != 1 { - t.Errorf("queue holds %d configs after duplicate enqueue, want 1", p.queue.Len()) - } - }) - - t.Run("node with KVM", func(t *testing.T) { - p := newSandboxPrewarmer(nil, nil, nil, true) - p.enqueue(ctx, microvm) - p.enqueue(ctx, gvisor) - if p.queue.Len() != 2 { - t.Errorf("queue holds %d configs, want both microvm and gvisor queued", p.queue.Len()) - } - }) -} - -// TestPrewarmProcessRetries covers the worker's failure handling: a failed -// prewarm is requeued with backoff, and a config deleted between enqueue and -// processing is forgotten without retries. -func TestPrewarmProcessRetries(t *testing.T) { - useTempAssetScratch(t) - - ctx := context.Background() - cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256([]byte("runsc")))) - // No pause image, so a failing prewarm exercises only the asset path. - cfg.Spec.Versions[0].PauseImage = "" - - indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) - if err := indexer.Add(cfg); err != nil { - t.Fatalf("indexer.Add: %v", err) - } - herder := &Atelet{sandboxAssetCache: newTestAssetCache(t), anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}} - p := newSandboxPrewarmer(herder, nil, listersv1alpha1.NewSandboxConfigLister(indexer), false) - - p.queue.Add(cfg.Name) - name, _ := p.queue.Get() - p.process(ctx, name) - if got := p.queue.NumRequeues(cfg.Name); got != 1 { - t.Errorf("NumRequeues after failed prewarm = %d, want 1 (requeued with backoff)", got) - } - - p.process(ctx, "deleted-config") - if got := p.queue.NumRequeues("deleted-config"); got != 0 { - t.Errorf("NumRequeues for a deleted config = %d, want 0 (forgotten)", got) - } -} - -// TestPrewarmProcessReappliesClassGate covers the gap between enqueue and -// processing: sandboxClass is mutable, so a config that passed the enqueue -// filter as gVisor may be micro-VM by the time the worker resolves it, and a -// node without KVM must skip it rather than download guest assets. -func TestPrewarmProcessReappliesClassGate(t *testing.T) { - useTempAssetScratch(t) - - ctx := context.Background() - // Enqueued while gVisor, edited to micro-VM before the worker ran. - cfg := gvisorConfig("mutating-config", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256([]byte("runsc")))) - cfg.Spec.SandboxClass = v1alpha1.SandboxClassMicroVM - - indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) - if err := indexer.Add(cfg); err != nil { - t.Fatalf("indexer.Add: %v", err) - } - // Any fetch would fail and requeue, so NumRequeues distinguishes - // "skipped" from "attempted and failed". - herder := &Atelet{sandboxAssetCache: newTestAssetCache(t), anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}} - p := newSandboxPrewarmer(herder, nil, listersv1alpha1.NewSandboxConfigLister(indexer), false) - - p.queue.Add(cfg.Name) - name, _ := p.queue.Get() - p.process(ctx, name) - if got := p.queue.NumRequeues(cfg.Name); got != 0 { - t.Errorf("NumRequeues after a class change to micro-VM on a non-KVM node = %d, want 0 (skipped, not retried)", got) - } -} - -// TestMicrovmNodeCapable covers the detectable negative cases; the positive -// case needs a /dev/kvm character device, which a test cannot mknod. -func TestMicrovmNodeCapable(t *testing.T) { - devRoot := t.TempDir() - if microvmNodeCapable(devRoot) { - t.Error("microvmNodeCapable = true for a dev root without kvm") - } - // A plain file named kvm is not a character device and must not count. - if err := os.WriteFile(devRoot+"/kvm", []byte("not a device"), 0o600); err != nil { - t.Fatal(err) - } - if microvmNodeCapable(devRoot) { - t.Error("microvmNodeCapable = true for a regular file named kvm") - } -} - -// TestPrewarmPauseImage covers the pause-image half of prewarm: the image -// lands in the image cache, and an asset fetch failure does not stop the -// pull (the two live in different backends). -func TestPrewarmPauseImage(t *testing.T) { - useTempAssetScratch(t) - - srv := httptest.NewServer(registry.New(registry.Logger(log.New(io.Discard, "", 0)))) - defer srv.Close() - u, err := url.Parse(srv.URL) - if err != nil { - t.Fatalf("parsing registry URL: %v", err) - } - pauseRef := u.Host + "/pause:3.10" - pauseDigest := pushPauseImage(t, pauseRef) - - newStore := func() *imagecache.Store { - s, err := imagecache.New(t.TempDir()) - if err != nil { - t.Fatalf("imagecache.New: %v", err) - } - return s - } - okStore, failStore, archStore := newStore(), newStore(), newStore() - - ctx := context.Background() - content := []byte("runsc binary bytes") - cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256(content))) - cfg.Spec.Versions[0].PauseImage = pauseRef - - p := &sandboxPrewarmer{ - assets: &Atelet{sandboxAssetCache: newTestAssetCache(t), anonGCSClient: fakeObjectStorage{data: content}}, - images: okStore, - } - if err := p.prewarm(ctx, cfg); err != nil { - t.Fatalf("prewarm: %v", err) - } - - // A different asset hash misses the asset cache, so the failing object - // storage is actually consulted — and must not keep the pause image from - // being pulled. - failCfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256([]byte("other runsc")))) - failCfg.Spec.Versions[0].PauseImage = pauseRef - p = &sandboxPrewarmer{ - assets: &Atelet{sandboxAssetCache: newTestAssetCache(t), anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}}, - images: failStore, - } - if err := p.prewarm(ctx, failCfg); err == nil { - t.Error("prewarm returned nil despite the asset fetch failing") - } - - // A config with no assets for this node's architecture still gets its - // pause image pulled, and prewarm succeeds: the missing assets are - // permanent until the config changes, not a retryable failure. - archCfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256(content))) - archCfg.Spec.Versions[0].Assets = map[string]map[string]v1alpha1.AssetFile{ - "other-arch": {runscAssetName: archCfg.Spec.Versions[0].Assets[runtime.GOARCH][runscAssetName]}, - } - archCfg.Spec.Versions[0].PauseImage = pauseRef - p = &sandboxPrewarmer{images: archStore} - if err := p.prewarm(ctx, archCfg); err != nil { - t.Errorf("prewarm with no assets for the local architecture: %v", err) - } - - // With the registry gone, only a cache hit can satisfy a digest ref. - srv.Close() - digestRef := u.Host + "/pause@" + pauseDigest.String() - for name, store := range map[string]*imagecache.Store{"ok": okStore, "asset-failure": failStore, "no-local-arch": archStore} { - if _, err := store.EnsureImage(ctx, digestRef); err != nil { - t.Errorf("pause image not prewarmed into the %s store: %v", name, err) - } - } -} - -// hangingObjectStorage blocks GetObject until the caller's context ends, -// simulating a bucket read that stalls without failing. -type hangingObjectStorage struct{} - -func (hangingObjectStorage) GetObject(ctx context.Context, _, _ string) (io.ReadCloser, error) { - <-ctx.Done() - return nil, ctx.Err() -} - -func (hangingObjectStorage) PutObject(_ context.Context, _, _ string, _ io.Reader) error { return nil } - -// TestPrewarmTimeout verifies a single prewarm attempt is bounded by -// prewarmTimeout: the queue has one worker, so an attempt that never returned -// would block every other config's prewarm. -func TestPrewarmTimeout(t *testing.T) { - useTempAssetScratch(t) - origTimeout := prewarmTimeout - prewarmTimeout = 50 * time.Millisecond - t.Cleanup(func() { prewarmTimeout = origTimeout }) - - cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256([]byte("hung runsc")))) - cfg.Spec.Versions[0].PauseImage = "" - p := &sandboxPrewarmer{assets: &Atelet{sandboxAssetCache: newTestAssetCache(t), anonGCSClient: hangingObjectStorage{}}} - - done := make(chan error, 1) - go func() { done <- p.prewarm(context.Background(), cfg) }() - select { - case err := <-done: - if err == nil { - t.Error("prewarm returned nil despite the download hanging") - } - case <-time.After(10 * time.Second): - t.Fatal("prewarm never returned; a hung download would block the worker forever") - } -} - -// TestPrewarmDefaultVersionOnly verifies prewarm fetches only the version -// defaultVersion names, and that a config without its default version, or -// whose default version is Disabled, is nothing to do rather than a retryable -// failure. -func TestPrewarmDefaultVersionOnly(t *testing.T) { - useTempAssetScratch(t) - - ctx := context.Background() - content := []byte("runsc binary bytes") - cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256(content))) - cfg.Spec.Versions[0].PauseImage = "" - // A non-default version whose asset the fake bucket cannot serve: fetching - // it would fail the hash check. - otherSHA := fmt.Sprintf("%x", sha256.Sum256([]byte("other runsc"))) - cfg.Spec.Versions = append([]v1alpha1.SandboxVersionConfig{{ - Name: "v0", - Assets: map[string]map[string]v1alpha1.AssetFile{runtime.GOARCH: {runscAssetName: {URL: "gs://bucket/other-runsc", SHA256: otherSHA}}}, - }}, cfg.Spec.Versions...) - - root := t.TempDir() - p := &sandboxPrewarmer{assets: &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{data: content}}} - if err := p.prewarm(ctx, cfg); err != nil { - t.Fatalf("prewarm: %v", err) - } - if n := cachedAssets(t, root); n != 1 { - t.Errorf("cache holds %d assets after prewarm, want 1 (the default version's only)", n) - } - - // Any fetch would fail, so a nil error means nothing was attempted. - cfg.Spec.DefaultVersion = "missing" - p = &sandboxPrewarmer{assets: &Atelet{sandboxAssetCache: newTestAssetCache(t), anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}}} - if err := p.prewarm(ctx, cfg); err != nil { - t.Errorf("prewarm of a config without its default version: %v", err) - } - - cfg.Spec.DefaultVersion = "v0" - cfg.Spec.Versions[0].State = ptr.To(v1alpha1.SandboxVersionStateDisabled) - if err := p.prewarm(ctx, cfg); err != nil { - t.Errorf("prewarm of a config whose default version is Disabled: %v", err) - } -} - -// TestSandboxAssetPrewarmDownloads runs the whole path: a SandboxConfig in a -// fake clientset flows through the informer into the prewarm worker, which -// lands the asset in the asset cache without any Run/Restore request. -func TestSandboxAssetPrewarmDownloads(t *testing.T) { - useTempAssetScratch(t) - origJitter, origFinished := prewarmMaxJitter, prewarmFinished - prewarmMaxJitter = 0 - // The worker's own prewarm call returns only when the asset has landed - // and the pause-image pull has finished: it waits for the pull on a live - // context, while the pull runs detached from that context (see - // imagecache.Store.EnsureImage). Waiting for the call keeps the pull's - // last writes from racing the TempDir cleanup. - prewarmed := make(chan error, 1) - prewarmFinished = func(_ string, err error) { - // Non-blocking: a failed attempt is retried, and the worker must not - // block on a result nobody reads. - select { - case prewarmed <- err: - default: - } - } - t.Cleanup(func() { - prewarmMaxJitter, prewarmFinished = origJitter, origFinished - }) - - content := []byte("runsc binary bytes") - sha := fmt.Sprintf("%x", sha256.Sum256(content)) - cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", sha) - cfg.Spec.Versions[0].PauseImage = "" - - ctx := t.Context() - client := fake.NewSimpleClientset(cfg) - factory := externalversions.NewSharedInformerFactory(client, 0) - informer := factory.Api().V1alpha1().SandboxConfigs().Informer() - - root := t.TempDir() - herder := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{data: content}} - // Handler first, informer start second, mirroring main: atelet startup - // must never wait on this informer's sync, and the initial List replays - // the pre-existing config into the handler as an Add. - p, err := startSandboxAssetPrewarm(ctx, informer, herder, nil, false) - if err != nil { - t.Fatalf("startSandboxAssetPrewarm: %v", err) - } - // The worker reads the patched globals above. t.Context() is canceled - // before cleanups run, and cleanups run last-registered-first, so this - // joins the worker before the earlier cleanup restores the globals — - // without it the restore can race the worker's tail. - t.Cleanup(p.wait) - stopCh := make(chan struct{}) - defer close(stopCh) - factory.Start(stopCh) - - select { - case err := <-prewarmed: - if err != nil { - t.Fatalf("prewarm: %v", err) - } - case <-time.After(30 * time.Second): - t.Fatal("prewarm never finished") - } - if n := cachedAssets(t, root); n != 1 { - t.Fatalf("cache holds %d assets after prewarm, want 1", n) - } - if left, err := os.ReadDir(ateletpath.SandboxAssetScratchDir); err != nil || len(left) != 0 { - t.Errorf("prewarm left %d scratch dirs behind (%v)", len(left), err) - } -} diff --git a/cmd/atelet/sandbox_runtime_index.go b/cmd/atelet/sandbox_runtime_index.go new file mode 100644 index 0000000000..d466a51294 --- /dev/null +++ b/cmd/atelet/sandbox_runtime_index.go @@ -0,0 +1,326 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "os" + "sync" + + "go.ate.dev/substrate/cmd/atelet/internal/atomicfile" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/hardware" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/pkg/api/v1alpha1" +) + +// runtimeIndex is this node's view of the sandbox runtimes its workers +// advertise: what the worker sandbox syncer wrote into each worker's +// versions file, kept with the asset records the hashes were computed from. +// Run and Restore resolve the runtime an actor is to use here, by the hash +// the worker registered with, so an actor runs exactly the binaries the +// control plane placed it for. +// +// Two maps answer two questions. runtimes, by hash, serves the hot path and +// never forgets an entry: a request may carry a hash a worker advertised +// earlier, config churn is rare, and an entry is a few URLs. workers, by +// worker pod UID, holds what each worker currently advertises and lives +// with the pod; it is where the default runtime to prewarm and to protect +// from eviction comes from. +// +// runtimes is also kept on disk, as one file that is the map serialized, +// rewritten durably before a new entry is added and loaded again by +// openRuntimeIndex. A worker registers a hash with the control plane only +// after atelet wrote its file, which happens after the index was written, +// so every hash a worker registered before an atelet restart resolves as +// soon as atelet serves again, with no wait on the syncer. The file stays +// small: a node sees a handful of configs, each with a few versions, in +// its lifetime. workers is rebuilt by the syncer. +// +// TODO: release runtimes that no pod has advertised for a long time. +type runtimeIndex struct { + mu sync.RWMutex + runtimes map[string]*indexedRuntime + workers map[string]*workerRuntime + // path persists runtimes; empty keeps the index in memory only. + path string +} + +// runtimeIndexFile is the on-disk form of runtimes. +type runtimeIndexFile struct { + Runtimes map[string]indexedRuntimeFile `json:"runtimes"` +} + +// indexedRuntimeFile is the on-disk form of one entry of runtimes, keyed by +// its hash in runtimeIndexFile. +type indexedRuntimeFile struct { + Config string `json:"config"` + Version string `json:"version"` + Record *sandboxAssetsRecord `json:"record"` +} + +// indexedRuntime is one advertised runtime: the record to stage, and where +// it was last computed from, for logs and errors. +type indexedRuntime struct { + rec *sandboxAssetsRecord + config string + version string +} + +// workerRuntime is what one worker's versions file advertises, by sandbox +// class: the SandboxConfig the worker's pool runs for the class and that +// config's runtimes. A pool runs one class today, so a worker advertises +// one; the shape admits the several a pool will be allowed to run. +type workerRuntime struct { + classes map[v1alpha1.SandboxClass]*classRuntime +} + +// classRuntime is what a worker advertises for one sandbox class. +type classRuntime struct { + config string + defaultHash string + // hashes holds every advertised runtime, the default first, each once. + hashes []string +} + +// class returns what the worker advertises for c. +func (w *workerRuntime) class(c v1alpha1.SandboxClass) (*classRuntime, bool) { + cr, ok := w.classes[c] + return cr, ok +} + +// advertisedClass is one sandbox class a worker advertises: the config its +// pool runs for the class, with the config's versions hashed. +type advertisedClass struct { + class v1alpha1.SandboxClass + config string + hashed []hashedVersion +} + +// newRuntimeIndex returns an index kept in memory only. +func newRuntimeIndex() *runtimeIndex { + return &runtimeIndex{ + runtimes: map[string]*indexedRuntime{}, + workers: map[string]*workerRuntime{}, + } +} + +// openRuntimeIndex returns the index persisted at path, loaded if the file +// exists. A file that cannot be decoded starts the index empty with a +// warning, and an entry whose record does not hash to its key is dropped +// with one: the syncer writes the current runtimes again either way. +func openRuntimeIndex(ctx context.Context, path string) (*runtimeIndex, error) { + x := newRuntimeIndex() + x.path = path + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + slog.InfoContext(ctx, "Sandbox runtime index starts empty", slog.String("path", path)) + return x, nil + } + if err != nil { + return nil, fmt.Errorf("while reading the sandbox runtime index: %w", err) + } + var f runtimeIndexFile + if err := json.Unmarshal(data, &f); err != nil { + slog.WarnContext(ctx, "Sandbox runtime index is unreadable; starting empty", slog.String("path", path), slog.Any("err", err)) + return x, nil + } + for hash, e := range f.Runtimes { + if err := checkIndexedRuntime(hash, e); err != nil { + slog.WarnContext(ctx, "Dropping a sandbox runtime index entry", slog.String("runtime_hash", hash), slog.Any("err", err)) + continue + } + x.runtimes[hash] = &indexedRuntime{rec: e.Record, config: e.Config, version: e.Version} + } + slog.InfoContext(ctx, "Sandbox runtime index loaded", slog.String("path", path), slog.Int("runtimes", len(x.runtimes))) + return x, nil +} + +// checkIndexedRuntime reports whether e belongs under hash: its record is +// present and hashes to it. +func checkIndexedRuntime(hash string, e indexedRuntimeFile) error { + if e.Record == nil { + return errors.New("no record") + } + got, err := versionBundleHash(e.Record) + if err != nil { + return fmt.Errorf("while hashing the record: %w", err) + } + if got != hash { + return fmt.Errorf("record hashes to %s", got) + } + return nil +} + +// writeIndex persists runtimes plus extra under extraHash, durably: once it +// returns, a crash cannot lose the file. Called with mu held. +func (x *runtimeIndex) writeIndex(extraHash string, extra *indexedRuntime) error { + f := runtimeIndexFile{Runtimes: make(map[string]indexedRuntimeFile, len(x.runtimes)+1)} + for hash, rt := range x.runtimes { + f.Runtimes[hash] = indexedRuntimeFile{Config: rt.config, Version: rt.version, Record: rt.rec} + } + f.Runtimes[extraHash] = indexedRuntimeFile{Config: extra.config, Version: extra.version, Record: extra.rec} + data, err := json.Marshal(f) + if err != nil { + return fmt.Errorf("while encoding the sandbox runtime index: %w", err) + } + if err := atomicfile.WriteSync(x.path, data, 0o600); err != nil { + return fmt.Errorf("while writing the sandbox runtime index: %w", err) + } + return nil +} + +// register records what the worker pod advertises, replacing what it +// advertised before, and makes each runtime resolvable by its hash. The +// syncer calls it once the pod's file is written, so a worker's entry says +// what the file the worker registers from says; the hashes themselves are +// registered earlier through registerRuntimes. It fails only if a runtime +// not yet indexed cannot be persisted. +func (x *runtimeIndex) register(podUID string, advertised ...advertisedClass) error { + x.mu.Lock() + defer x.mu.Unlock() + wr := &workerRuntime{classes: make(map[v1alpha1.SandboxClass]*classRuntime, len(advertised))} + for _, a := range advertised { + cr := &classRuntime{config: a.config} + seen := make(map[string]bool, len(a.hashed)) + for _, v := range a.hashed { + if err := x.addRuntime(a.config, v); err != nil { + return err + } + if v.isDefault { + cr.defaultHash = v.hash + } + if !seen[v.hash] { + seen[v.hash] = true + cr.hashes = append(cr.hashes, v.hash) + } + } + wr.classes[a.class] = cr + } + x.workers[podUID] = wr + return nil +} + +// registerRuntimes makes each hashed version of config resolvable without +// changing what any worker advertises. The syncer calls it the moment a +// SandboxConfig or WorkerPool changes, ahead of the jittered sync that +// rewrites the files, so the index knows a config's runtimes as soon as the +// listers do, and a request for the new default that arrives within that +// delay resolves (see enqueuePool). It fails only if a runtime not yet +// indexed cannot be persisted; the ones before it are indexed. +func (x *runtimeIndex) registerRuntimes(config string, hashed []hashedVersion) error { + x.mu.Lock() + defer x.mu.Unlock() + for _, v := range hashed { + if err := x.addRuntime(config, v); err != nil { + return err + } + } + return nil +} + +// addRuntime records v under its hash, unless the hash is already known, +// persisting the index first when it has a path. The hash covers every +// asset's name and digest and the pause image, so two versions with one +// hash differ at most in where identical bytes are fetched from; keeping +// the first keeps the version the file was built from. Called with mu held. +func (x *runtimeIndex) addRuntime(config string, v hashedVersion) error { + if _, ok := x.runtimes[v.hash]; ok { + return nil + } + rt := &indexedRuntime{rec: v.rec, config: config, version: v.name} + if x.path != "" { + if err := x.writeIndex(v.hash, rt); err != nil { + return err + } + } + x.runtimes[v.hash] = rt + return nil +} + +// forget drops what the worker pod advertised. The runtimes themselves stay +// resolvable. +func (x *runtimeIndex) forget(podUID string) { + x.mu.Lock() + defer x.mu.Unlock() + delete(x.workers, podUID) +} + +// lookup returns the runtime advertised under hash. +func (x *runtimeIndex) lookup(hash string) (*indexedRuntime, bool) { + x.mu.RLock() + defer x.mu.RUnlock() + rt, ok := x.runtimes[hash] + return rt, ok +} + +// worker returns what the worker pod currently advertises. +func (x *runtimeIndex) worker(podUID string) (*workerRuntime, bool) { + x.mu.RLock() + defer x.mu.RUnlock() + wr, ok := x.workers[podUID] + return wr, ok +} + +// advertisedRuntime returns the sandbox assets to stage for an actor placed +// on runtime rt: the record of the runtime a worker on this node advertises +// under rt's bundle hash. ate-api-server fills rt from the worker +// assignment, which is a runtime the worker registered from the file this +// node wrote, so the hash resolves as long as the request reached the node +// the actor was placed on. The request's sandbox_assets are not consulted. +// +// A miss is not served from the request: it means the request reached a +// node whose workers never advertised the runtime, which must surface. A +// runtime registered before an atelet restart is no miss: the index is +// loaded from disk before atelet serves. +func (s *Atelet) advertisedRuntime(ctx context.Context, rt *ateletpb.SandboxRuntime) (*sandboxAssetsRecord, error) { + hash, err := bundleHashOf(rt) + if err != nil { + return nil, apierror.InvalidArgument("sandbox_runtime: %v", err) + } + indexed, ok := s.runtimes.lookup(hash) + if !ok { + slog.WarnContext(ctx, "Sandbox runtime is not advertised by any worker on this node", + slog.String("bundle_hash", hash), + slog.String("sandbox_class", rt.GetSandboxClass())) + return nil, apierror.FailedPrecondition("sandbox runtime %s (class %s) is not advertised by any worker on this node", hash, rt.GetSandboxClass()) + } + if class := rt.GetSandboxClass(); class != indexed.rec.SandboxClass { + return nil, apierror.FailedPrecondition("sandbox runtime %s is advertised for class %s, not %s", hash, indexed.rec.SandboxClass, class) + } + return indexed.rec, nil +} + +// bundleHashOf returns the sandbox_bundle_hash attribute of rt, which names +// the SandboxConfig version a worker runs an actor under. +func bundleHashOf(rt *ateletpb.SandboxRuntime) (string, error) { + if rt == nil { + return "", errors.New("missing") + } + for _, a := range rt.GetVersion().GetAttributes() { + if a.GetKey() != hardware.AttrSandboxBundleHash { + continue + } + if a.GetValue() == "" { + return "", fmt.Errorf("empty %s attribute", hardware.AttrSandboxBundleHash) + } + return a.GetValue(), nil + } + return "", fmt.Errorf("no %s attribute", hardware.AttrSandboxBundleHash) +} diff --git a/cmd/atelet/sandbox_runtime_index_test.go b/cmd/atelet/sandbox_runtime_index_test.go new file mode 100644 index 0000000000..535939924c --- /dev/null +++ b/cmd/atelet/sandbox_runtime_index_test.go @@ -0,0 +1,307 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "slices" + "testing" + + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/hardware" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/pkg/api/v1alpha1" + "google.golang.org/grpc/codes" +) + +// hashedTestVersion is a version of config whose runtime is identified by +// its one asset's digest. +func hashedTestVersion(t *testing.T, name string, isDefault bool, sha string) hashedVersion { + t.Helper() + rec := &sandboxAssetsRecord{ + SandboxClass: "gvisor", + PauseImage: testPauseImage, + Assets: map[string]assetEntry{gvisorAssetName: {URL: "gs://bucket/" + name, SHA256: sha}}, + } + h, err := versionBundleHash(rec) + if err != nil { + t.Fatal(err) + } + return hashedVersion{versionPlan: versionPlan{name: name, isDefault: isDefault, rec: rec}, hash: h} +} + +func TestRuntimeIndex(t *testing.T) { + const ( + shaA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + shaB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + ) + x := newRuntimeIndex() + v1, v2 := hashedTestVersion(t, "v1", true, shaA), hashedTestVersion(t, "v2", false, shaB) + + gvisor := v1alpha1.SandboxClassGvisor + x.register("pod-1", advertisedClass{class: gvisor, config: "config-x", hashed: []hashedVersion{v1, v2}}) + for _, v := range []hashedVersion{v1, v2} { + rt, ok := x.lookup(v.hash) + if !ok || rt.rec != v.rec || rt.config != "config-x" || rt.version != v.name { + t.Errorf("lookup(%s) = %+v, %v; want %s's record from config-x", v.name, rt, ok, v.name) + } + } + wr, ok := x.worker("pod-1") + if !ok { + t.Fatal("worker(pod-1) not indexed") + } + cr, ok := wr.class(gvisor) + if !ok || cr.config != "config-x" || cr.defaultHash != v1.hash || !slices.Equal(cr.hashes, []string{v1.hash, v2.hash}) { + t.Errorf("worker(pod-1).class(gvisor) = %+v, %v; want config-x with %s default and both hashes", cr, ok, v1.hash) + } + if _, ok := wr.class(v1alpha1.SandboxClassMicroVM); ok { + t.Error("worker(pod-1) advertises a class its pool does not run") + } + + // Another config with the same binaries under another name shares the + // runtime; the entry keeps the record it was first registered with, + // which is the one the first worker's file was built from. + same := hashedTestVersion(t, "release-1", true, shaA) + x.register("pod-2", advertisedClass{class: gvisor, config: "config-y", hashed: []hashedVersion{same}}) + if same.hash != v1.hash { + t.Fatal("identical binaries hashed differently") + } + if rt, _ := x.lookup(v1.hash); rt.config != "config-x" || rt.version != "v1" { + t.Errorf("shared runtime provenance = %s/%s, want the first registration's config-x/v1", rt.config, rt.version) + } + if wr, _ := x.worker("pod-2"); wr.classes[gvisor].defaultHash != same.hash { + t.Error("pod-2 does not advertise the shared runtime as its default") + } + + // A pod's re-plan replaces what it advertises; the old runtime stays + // resolvable, since a request may still carry its hash. + x.register("pod-1", advertisedClass{class: gvisor, config: "config-x", hashed: []hashedVersion{hashedTestVersion(t, "v2", true, shaB)}}) + if wr, _ := x.worker("pod-1"); wr.classes[gvisor].defaultHash != v2.hash || len(wr.classes[gvisor].hashes) != 1 { + t.Errorf("worker(pod-1) after re-plan = %+v, want v2 as the only runtime", wr.classes[gvisor]) + } + if _, ok := x.lookup(v1.hash); !ok { + t.Error("runtime v1 forgotten after pod-1 stopped advertising it") + } + + // A gone pod drops out; its runtimes do not. + x.forget("pod-2") + if _, ok := x.worker("pod-2"); ok { + t.Error("pod-2 still indexed after forget") + } + if _, ok := x.lookup(same.hash); !ok { + t.Error("runtime forgotten with its pod") + } + if _, ok := x.lookup("sha256:" + shaB[:60] + "0000"); ok { + t.Error("lookup of an unknown hash succeeded") + } +} + +// TestRuntimeIndexRegisterRuntimes pins the early registration the syncer +// does from an event handler: the hashes resolve, and no worker's entry +// appears or changes until its file is written. +func TestRuntimeIndexRegisterRuntimes(t *testing.T) { + const ( + shaA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + shaB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + shaC = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + ) + x := newRuntimeIndex() + gvisor := v1alpha1.SandboxClassGvisor + old := hashedTestVersion(t, "v1", true, shaA) + x.register("pod-1", advertisedClass{class: gvisor, config: "config-x", hashed: []hashedVersion{old}}) + + next := hashedTestVersion(t, "v2", true, shaB) + x.registerRuntimes("config-x", []hashedVersion{next}) + rt, ok := x.lookup(next.hash) + if !ok || rt.version != "v2" || rt.config != "config-x" { + t.Fatalf("lookup(%s) = %+v, %v; want v2 of config-x", next.hash, rt, ok) + } + wr, ok := x.worker("pod-1") + if !ok { + t.Fatal("worker pod-1 gone after registerRuntimes") + } + if cr, _ := wr.class(gvisor); cr.defaultHash != old.hash { + t.Errorf("pod-1 default = %s after registerRuntimes, want the written file's %s", cr.defaultHash, old.hash) + } + x.registerRuntimes("config-y", []hashedVersion{hashedTestVersion(t, "v1", true, shaC)}) + if _, ok := x.worker("pod-2"); ok { + t.Error("registerRuntimes created a worker entry") + } +} + +func TestResolveRuntime(t *testing.T) { + const sha = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + s := &Atelet{runtimes: newRuntimeIndex()} + advertised := hashedTestVersion(t, "v1", true, sha) + runtimeWith := func(class string, attrs ...*ateletpb.AttributeEntry) *ateletpb.SandboxRuntime { + return &ateletpb.SandboxRuntime{SandboxClass: class, Version: &ateletpb.VersionedSandboxCompat{SchemaVersion: hardware.SchemaVersionV1, Attributes: attrs}} + } + placed := runtimeWith("gvisor", + &ateletpb.AttributeEntry{Key: hardware.AttrArchitecture, Value: "amd64"}, + &ateletpb.AttributeEntry{Key: hardware.AttrSandboxBundleHash, Value: advertised.hash}) + + if _, err := s.advertisedRuntime(t.Context(), placed); apierror.Code(err) != codes.FailedPrecondition { + t.Fatalf("advertisedRuntime before any worker advertises it = %v, want %s", err, codes.FailedPrecondition) + } + s.runtimes.register("pod-1", advertisedClass{class: v1alpha1.SandboxClassGvisor, config: "config-x", hashed: []hashedVersion{advertised}}) + rec, err := s.advertisedRuntime(t.Context(), placed) + if err != nil { + t.Fatalf("advertisedRuntime: %v", err) + } + if rec != advertised.rec { + t.Errorf("advertisedRuntime returned %+v, want the advertised record", rec) + } + + for name, rt := range map[string]*ateletpb.SandboxRuntime{ + "no runtime": nil, + "no attributes": runtimeWith("gvisor"), + "no bundle hash": runtimeWith("gvisor", + &ateletpb.AttributeEntry{Key: hardware.AttrArchitecture, Value: "amd64"}), + "empty bundle hash": runtimeWith("gvisor", + &ateletpb.AttributeEntry{Key: hardware.AttrSandboxBundleHash, Value: ""}), + } { + if _, err := s.advertisedRuntime(t.Context(), rt); apierror.Code(err) != codes.InvalidArgument { + t.Errorf("advertisedRuntime with %s = %v, want %s", name, err, codes.InvalidArgument) + } + } + otherClass := runtimeWith("microvm", &ateletpb.AttributeEntry{Key: hardware.AttrSandboxBundleHash, Value: advertised.hash}) + if _, err := s.advertisedRuntime(t.Context(), otherClass); apierror.Code(err) != codes.FailedPrecondition { + t.Errorf("advertisedRuntime with the hash under another class = %v, want %s", err, codes.FailedPrecondition) + } +} + +// TestRuntimeIndexPersists pins what survives an atelet restart: every +// runtime registered, with its provenance, through a file rewritten before +// the entry exists in memory; an entry whose record does not hash to its +// key is dropped on load without taking the rest with it, and a file that +// cannot be decoded starts the index empty rather than failing startup. +func TestRuntimeIndexPersists(t *testing.T) { + const ( + shaA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + shaB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + ) + ctx := context.Background() + path := filepath.Join(t.TempDir(), "sandbox-runtime-index.json") + x, err := openRuntimeIndex(ctx, path) + if err != nil { + t.Fatalf("openRuntimeIndex with no file: %v", err) + } + v1, v2 := hashedTestVersion(t, "v1", true, shaA), hashedTestVersion(t, "v2", false, shaB) + if err := x.registerRuntimes("config-x", []hashedVersion{v1}); err != nil { + t.Fatal(err) + } + if err := x.register("pod-1", advertisedClass{class: v1alpha1.SandboxClassGvisor, config: "config-x", hashed: []hashedVersion{v1, v2}}); err != nil { + t.Fatal(err) + } + + // Restart: a new index over the same file resolves both, with what they + // were registered as, and knows no worker until the syncer runs. + reopened, err := openRuntimeIndex(ctx, path) + if err != nil { + t.Fatalf("openRuntimeIndex over the written file: %v", err) + } + for _, v := range []hashedVersion{v1, v2} { + rt, ok := reopened.lookup(v.hash) + if !ok { + t.Fatalf("runtime %s not loaded", v.name) + } + if rt.config != "config-x" || rt.version != v.name || rt.rec.Assets[gvisorAssetName].SHA256 != v.rec.Assets[gvisorAssetName].SHA256 { + t.Errorf("runtime %s loaded as %s/%s with digest %s; want config-x/%s with the registered digest", v.name, rt.config, rt.version, rt.rec.Assets[gvisorAssetName].SHA256, v.name) + } + } + if _, ok := reopened.worker("pod-1"); ok { + t.Error("a worker entry survived the restart; only the syncer knows what a worker advertises now") + } + + // An entry filed under the wrong hash is dropped; the other survives. + // Here v1's record is moved under a hash it does not match. + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var f runtimeIndexFile + if err := json.Unmarshal(data, &f); err != nil { + t.Fatal(err) + } + const wrongHash = "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + f.Runtimes[wrongHash] = f.Runtimes[v1.hash] + delete(f.Runtimes, v1.hash) + mismatched, err := json.Marshal(f) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, mismatched, 0o600); err != nil { + t.Fatal(err) + } + damaged, err := openRuntimeIndex(ctx, path) + if err != nil { + t.Fatalf("openRuntimeIndex over a file with a bad entry: %v", err) + } + if _, ok := damaged.lookup(v2.hash); !ok { + t.Error("the intact entry was not loaded") + } + if _, ok := damaged.lookup(wrongHash); ok { + t.Error("an entry whose record does not hash to its key was loaded") + } + if _, ok := damaged.lookup(v1.hash); ok { + t.Error("v1 loaded though the file no longer holds it under its hash") + } + // Registering v1 again rewrites the file with the good entries only. + if err := damaged.registerRuntimes("config-x", []hashedVersion{v1}); err != nil { + t.Fatal(err) + } + rewritten, err := openRuntimeIndex(ctx, path) + if err != nil { + t.Fatal(err) + } + for _, v := range []hashedVersion{v1, v2} { + if _, ok := rewritten.lookup(v.hash); !ok { + t.Errorf("runtime %s missing after the rewrite", v.name) + } + } + if len(rewritten.runtimes) != 2 { + t.Errorf("rewritten index holds %d runtimes, want 2 (the bad entry gone)", len(rewritten.runtimes)) + } + + // A file that does not decode starts the index empty, not atelet failed. + if err := os.WriteFile(path, []byte("{"), 0o600); err != nil { + t.Fatal(err) + } + empty, err := openRuntimeIndex(ctx, path) + if err != nil { + t.Fatalf("openRuntimeIndex over a corrupt file: %v", err) + } + if len(empty.runtimes) != 0 { + t.Errorf("corrupt file loaded %d runtimes, want 0", len(empty.runtimes)) + } +} + +// TestRuntimeIndexPersistFailureIsReported pins that an index that cannot +// write its file reports it and does not index the runtime, so the sync +// that asked retries rather than advertising a hash a restart would lose. +func TestRuntimeIndexPersistFailureIsReported(t *testing.T) { + x := newRuntimeIndex() + x.path = filepath.Join(t.TempDir(), "missing", "sandbox-runtime-index.json") // dir never created + v := hashedTestVersion(t, "v1", true, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + if err := x.registerRuntimes("config-x", []hashedVersion{v}); err == nil { + t.Fatal("registerRuntimes with an unwritable path = nil, want an error") + } + if _, ok := x.lookup(v.hash); ok { + t.Error("a runtime whose file could not be written was indexed") + } +} diff --git a/cmd/atelet/snapshot_transfer_test.go b/cmd/atelet/snapshot_transfer_test.go index 73423ae490..d92692998d 100644 --- a/cmd/atelet/snapshot_transfer_test.go +++ b/cmd/atelet/snapshot_transfer_test.go @@ -58,8 +58,9 @@ func newPluginHerder(t *testing.T, store objectstorage.ObjectStorage) *Atelet { return &Atelet{ objectStorePlugin: objectstoreplugintest.NodeClient(plugin), snapshotScratchDir: t.TempDir(), - snapshots: snapshottracker.New(t.Context(), clock.RealClock{}), + snapshots: snapshottracker.New(t.Context(), nil, clock.RealClock{}), sandboxAssetCache: newTestAssetCache(t), + runtimes: newRuntimeIndex(), } } @@ -110,7 +111,7 @@ func TestUploadSnapshotManifestLast(t *testing.T) { } } - if err := s.uploadSnapshot(context.Background(), uri, dir, &rec, "ate-demo", "counter"); err != nil { + if err := s.uploadSnapshot(context.Background(), uri, dir, &rec); err != nil { t.Fatalf("uploadSnapshot: %v", err) } if len(store.puts) != 4 { @@ -194,7 +195,7 @@ func TestObjectStorePluginErrorCodeThroughAtelet(t *testing.T) { {"paused upload", true, func(ctx context.Context, s *Atelet) error { dir := t.TempDir() writeLocalSnapshot(t, dir, rec, map[string]string{"a": "a"}) - _, err := s.uploadLocalCheckpointDir(ctx, validPromoteSnapshotRequest(), dir, uri) + _, _, err := s.uploadLocalCheckpointDir(ctx, validPromoteSnapshotRequest(), dir, uri) return err }}, {"checkpoint upload", false, func(ctx context.Context, s *Atelet) error { @@ -260,7 +261,7 @@ func TestObjectStorePluginErrorKeepsCredentialsOut(t *testing.T) { {"paused upload", "", func(ctx context.Context, s *Atelet) error { dir := t.TempDir() writeLocalSnapshot(t, dir, rec, map[string]string{"a": "a"}) - _, err := s.uploadLocalCheckpointDir(ctx, validPromoteSnapshotRequest(), dir, uri) + _, _, err := s.uploadLocalCheckpointDir(ctx, validPromoteSnapshotRequest(), dir, uri) return err }}, } @@ -412,7 +413,7 @@ func TestCheckpointUploadWaitsOutPluginOutage(t *testing.T) { } } - if _, err := s.uploadLocalCheckpointDir(ctx, validPromoteSnapshotRequest(), pausedDir, uri); apierror.Code(err) != codes.Unavailable { + if _, _, err := s.uploadLocalCheckpointDir(ctx, validPromoteSnapshotRequest(), pausedDir, uri); apierror.Code(err) != codes.Unavailable { t.Fatalf("paused upload during the outage = %v, want %s", err, codes.Unavailable) } diff --git a/cmd/atelet/snapshotreport.go b/cmd/atelet/snapshotreport.go new file mode 100644 index 0000000000..2ae96e6b2f --- /dev/null +++ b/cmd/atelet/snapshotreport.go @@ -0,0 +1,95 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + + "go.ate.dev/substrate/cmd/atelet/internal/snapshottracker" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" +) + +// trackerReporter is the snapshottracker.Reporter that reports to the control +// plane's WorkerService, with atelet's own client certificate. +type trackerReporter struct{ workers ateapipb.WorkerServiceClient } + +var _ snapshottracker.Reporter = (*trackerReporter)(nil) + +// ReportActorSnapshotState sends state with +// WorkerService.ReportActorSnapshotState, and returns the control plane's +// error unwrapped, so the tracker can tell a refusal from a failure to reach +// it. +func (r *trackerReporter) ReportActorSnapshotState(ctx context.Context, actorUID string, actor resources.ActorRef, state *ateletpb.ActorSnapshotState) error { + _, err := r.workers.ReportActorSnapshotState(ctx, &ateapipb.ReportActorSnapshotStateRequest{ + Actor: actor.ToObjectRef(), + ActorUid: actorUID, + State: toAteAPIActorSnapshotState(state), + }) + return err +} + +// toAteAPIActorSnapshotState converts atelet's ActorSnapshotState to the +// control plane's, which carries each snapshot's fields inline. A snapshot +// atelet does not hold stays unset, since the control plane reads a state with +// neither latest_local nor uploading as a detach. +func toAteAPIActorSnapshotState(in *ateletpb.ActorSnapshotState) *ateapipb.ActorSnapshotState { + if in == nil { + return nil + } + return &ateapipb.ActorSnapshotState{ + LatestLocal: toAteAPISnapshotReport(in.GetLatestLocal()), + Uploading: toAteAPISnapshotReport(in.GetUploading()), + LatestDurable: toAteAPISnapshotReport(in.GetLatestDurable()), + UploadDueTime: in.GetUploadDueTime(), + } +} + +// toAteAPISnapshotReport converts one of atelet's SnapshotReports to the +// control plane's. A report that names no snapshot converts to nil. +func toAteAPISnapshotReport(in *ateletpb.SnapshotReport) *ateapipb.SnapshotReport { + snap := in.GetSnapshot() + if snap == nil { + return nil + } + out := &ateapipb.SnapshotReport{ + Uuid: snap.GetSnapshotUuid(), + Generation: snap.GetGeneration(), + Fidelity: toAteAPIFidelity(snap.GetFidelity()), + LocalCommitTime: in.GetLocalCommitTime(), + DurableCompleteTime: in.GetDurableCompleteTime(), + } + if obj := snap.GetObject(); obj != nil { + out.Object = &ateapipb.ObjectSnapshot{SnapshotUri: obj.GetSnapshotUri()} + } + return out +} + +// toAteAPIFidelity maps atelet's fidelity enum onto the identical public one, +// level by level. Anything else goes out as UNSPECIFIED, which the control +// plane rejects. +func toAteAPIFidelity(in ateletpb.SnapshotFidelity) ateapipb.SnapshotFidelity { + switch in { + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: + return ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS: + return ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: + return ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY + default: + return ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED + } +} diff --git a/cmd/atelet/snapshotreport_test.go b/cmd/atelet/snapshotreport_test.go new file mode 100644 index 0000000000..b74c290a92 --- /dev/null +++ b/cmd/atelet/snapshotreport_test.go @@ -0,0 +1,216 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "testing" + "time" + + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/cmd/atelet/internal/snapshottracker" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" + clocktesting "k8s.io/utils/clock/testing" +) + +// fakeReportService is a WorkerServiceClient that records the snapshot state +// reports it receives, and fails them with err. +type fakeReportService struct { + ateapipb.WorkerServiceClient + + reports chan *ateapipb.ReportActorSnapshotStateRequest + err error +} + +func newFakeReportService() *fakeReportService { + return &fakeReportService{reports: make(chan *ateapipb.ReportActorSnapshotStateRequest, 10)} +} + +func (s *fakeReportService) ReportActorSnapshotState(_ context.Context, in *ateapipb.ReportActorSnapshotStateRequest, _ ...grpc.CallOption) (*ateapipb.ReportActorSnapshotStateResponse, error) { + s.reports <- in + if s.err != nil { + return nil, s.err + } + return &ateapipb.ReportActorSnapshotStateResponse{}, nil +} + +// TestTrackerReporterSendsState reports the actor's state as the control +// plane's ActorSnapshotState: each snapshot's fields inline, and a snapshot +// atelet does not hold left unset, so a detached actor reads as a detach. +func TestTrackerReporterSendsState(t *testing.T) { + commit := timestamppb.New(time.Unix(1_700_000_000, 0)) + upload := timestamppb.New(time.Unix(1_700_000_060, 0)) + due := timestamppb.New(time.Unix(1_700_000_300, 0)) + local := func(name string, generation int32) *ateletpb.SnapshotReport { + return &ateletpb.SnapshotReport{ + Snapshot: lifecycleSnapshot(pluginActorUID, name, generation, ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY), + LocalCommitTime: commit, + } + } + durable := local("s1", 1) + durable.DurableCompleteTime = upload + reported := func(name string, generation int32) *ateapipb.SnapshotReport { + return &ateapipb.SnapshotReport{ + Uuid: name, + Generation: generation, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: "gs://test-bucket/atespaces/ate-demo/actors/" + pluginActorUID + "/snapshots/" + name}, + LocalCommitTime: commit, + } + } + reportedDurable := reported("s1", 1) + reportedDurable.DurableCompleteTime = upload + + for _, tc := range []struct { + name string + state *ateletpb.ActorSnapshotState + want *ateapipb.ActorSnapshotState + }{ + { + name: "every snapshot", + state: &ateletpb.ActorSnapshotState{ + LatestLocal: local("s3", 3), + Uploading: local("s2", 2), + LatestDurable: durable, + UploadDueTime: due, + }, + want: &ateapipb.ActorSnapshotState{ + LatestLocal: reported("s3", 3), + Uploading: reported("s2", 2), + LatestDurable: reportedDurable, + UploadDueTime: due, + }, + }, + { + name: "detached", + state: &ateletpb.ActorSnapshotState{LatestDurable: durable}, + want: &ateapipb.ActorSnapshotState{LatestDurable: reportedDurable}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + workers := newFakeReportService() + actor := resources.ActorRef{Atespace: "ate-demo", Name: "counter"} + if err := (&trackerReporter{workers: workers}).ReportActorSnapshotState(t.Context(), pluginActorUID, actor, tc.state); err != nil { + t.Fatalf("ReportActorSnapshotState: %v", err) + } + want := &ateapipb.ReportActorSnapshotStateRequest{ + Actor: &ateapipb.ObjectRef{Atespace: "ate-demo", Name: "counter"}, + ActorUid: pluginActorUID, + State: tc.want, + } + if got := <-workers.reports; !proto.Equal(got, want) { + t.Errorf("request = %v, want %v", got, want) + } + }) + } +} + +// TestTrackerReporterReturnsRefusal returns the control plane's error +// unwrapped, so the tracker can tell a refusal from a failure to reach it. +func TestTrackerReporterReturnsRefusal(t *testing.T) { + workers := newFakeReportService() + workers.err = status.Error(codes.NotFound, "actor not found") + state := &ateletpb.ActorSnapshotState{LatestDurable: &ateletpb.SnapshotReport{ + Snapshot: lifecycleSnapshot(pluginActorUID, "s1", 1, ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY), + }} + err := (&trackerReporter{workers: workers}).ReportActorSnapshotState(t.Context(), pluginActorUID, resources.ActorRef{Atespace: "ate-demo", Name: "counter"}, state) + if status.Code(err) != codes.NotFound { + t.Errorf("ReportActorSnapshotState = %v, want NotFound", err) + } +} + +// TestToAteAPIFidelity maps every atelet fidelity onto the public one of the +// same name, and an unknown one onto UNSPECIFIED. +func TestToAteAPIFidelity(t *testing.T) { + for value, name := range ateletpb.SnapshotFidelity_name { + if got := toAteAPIFidelity(ateletpb.SnapshotFidelity(value)); got.String() != name { + t.Errorf("toAteAPIFidelity(%s) = %s, want the public fidelity of the same name", name, got) + } + } + if got := toAteAPIFidelity(ateletpb.SnapshotFidelity(99)); got != ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED { + t.Errorf("toAteAPIFidelity(99) = %s, want UNSPECIFIED", got) + } +} + +// nextReport steps clk a second at a time until workers receives a report, +// and returns it. The tracker's queues arm their timers from a clock reading +// taken just before, so a step that lands in between leaves the timer late; +// further steps get past it. +func nextReport(t *testing.T, clk *clocktesting.FakeClock, workers *fakeReportService) *ateapipb.ReportActorSnapshotStateRequest { + t.Helper() + var got *ateapipb.ReportActorSnapshotStateRequest + waitFor(t, "a snapshot state report", func() bool { + select { + case got = <-workers.reports: + return true + case <-time.After(10 * time.Millisecond): + clk.Step(time.Second) + return false + } + }) + return got +} + +// TestBackgroundUploadAndDetachAreReported pauses an actor and leaves it +// paused: atelet reports the background upload of its snapshot, then its +// background detach, to the control plane, for the actor its Checkpoint named. +func TestBackgroundUploadAndDetachAreReported(t *testing.T) { + a := runPluginActor(t) + clk := clocktesting.NewFakeClock(time.Now()) + workers := newFakeReportService() + a.s.snapshots = snapshottracker.New(t.Context(), &trackerStore{s: a.s}, clk, + snapshottracker.WithUploadDelay(time.Second), + snapshottracker.WithDetachDelay(time.Hour), + snapshottracker.WithReporter(&trackerReporter{workers: workers})) + a.s.snapshots.Start() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if err := a.s.snapshots.Wait(ctx); err != nil { + t.Errorf("Wait: %v", err) + } + }) + if _, err := a.checkpoint(t, a.snapshot("pause-snap-1", 1), ateletpb.UploadMode_UPLOAD_MODE_DEFERRED); err != nil { + t.Fatalf("Checkpoint: %v", err) + } + wantActor := &ateapipb.ObjectRef{Atespace: "ate-demo", Name: "counter"} + + uploaded := nextReport(t, clk, workers) + if !proto.Equal(uploaded.GetActor(), wantActor) || uploaded.GetActorUid() != pluginActorUID { + t.Errorf("upload report for actor %v %q, want %v %q", uploaded.GetActor(), uploaded.GetActorUid(), wantActor, pluginActorUID) + } + if st := uploaded.GetState(); st.GetLatestDurable().GetUuid() != "pause-snap-1" || st.GetLatestLocal().GetUuid() != "pause-snap-1" { + t.Errorf("upload report state = %v, want pause-snap-1 durable and still local", st) + } + + clk.Step(time.Hour) + detached := nextReport(t, clk, workers) + if !proto.Equal(detached.GetActor(), wantActor) { + t.Errorf("detach report for actor %v, want %v", detached.GetActor(), wantActor) + } + if st := detached.GetState(); st.GetLatestLocal() != nil || st.GetUploading() != nil || st.GetLatestDurable().GetUuid() != "pause-snap-1" { + t.Errorf("detach report state = %v, want nothing local or uploading, and pause-snap-1 durable", st) + } + if exists(ateletpath.LocalSnapshotDir(pluginActorUID, "pause-snap-1")) { + t.Error("pause-snap-1 is still on the node after the reported detach") + } +} diff --git a/cmd/atelet/worker_sandbox_syncer.go b/cmd/atelet/worker_sandbox_syncer.go index 1307cf0d5f..df88d4f313 100644 --- a/cmd/atelet/worker_sandbox_syncer.go +++ b/cmd/atelet/worker_sandbox_syncer.go @@ -70,7 +70,9 @@ const orphanMinAge = time.Minute // applied as the enqueue delay, so duplicate events inside the window // collapse into one queue entry. A pod bound to this node is // enqueued without it, since only this atelet acts on it and its worker waits -// for the file. +// for the file. Only the file waits: the changed config's runtimes are +// registered in the index at once (see enqueuePool), so an actor the +// control plane places for the new default inside the window resolves. const syncMaxJitter = 30 * time.Second // syncMaxRetries bounds how often one pod's errUnresolved sync is retried @@ -94,6 +96,13 @@ const syncWorkers = 2 // it writes the pod's file and removes it when the pod is gone. type workerSandboxSyncer struct { resolver *workerSandboxResolver + // index is told each pod's runtimes before its file is written, and + // what the pod advertises once it is. + index *runtimeIndex + // prewarm, if set, is handed the default runtime hash of every pod + // whose file was written, so its assets are fetched before an actor + // asks for them. + prewarm func(hash string) // pods holds this node's worker pods, indexed by podUIDIndex. pods cache.Indexer assets sandboxAssetFetcher @@ -124,12 +133,14 @@ type workerSandboxSyncer struct { cpuFeatures map[string]string } -func newWorkerSandboxSyncer(resolver *workerSandboxResolver, pods cache.Indexer, assets sandboxAssetFetcher, workersDir string) *workerSandboxSyncer { +func newWorkerSandboxSyncer(resolver *workerSandboxResolver, pods cache.Indexer, assets sandboxAssetFetcher, workersDir string, index *runtimeIndex, prewarm func(hash string)) *workerSandboxSyncer { return &workerSandboxSyncer{ resolver: resolver, pods: pods, assets: assets, scratchDir: ateletpath.SandboxAssetScratchDir, + index: index, + prewarm: prewarm, // Backoff doubles from 1s for informer lag and unresolved pools, and // caps at 5m where uncapped retries for transient asset fetch or probe // failures stay until they recover. @@ -149,10 +160,14 @@ func newWorkerSandboxSyncer(resolver *workerSandboxResolver, pods cache.Indexer, // worker pods and must not have started yet. Worker directories live under // workersDir. // -// The syncer never holds up atelet startup: until the informers sync, or -// while a list/watch fails (e.g. Forbidden while an RBAC rollout lags the -// binary), files wait and existing ones stay as they are. -func startWorkerSandboxSyncer(ctx context.Context, pods coreinformers.PodInformer, workerPools informersv1alpha1.WorkerPoolInformer, configs informersv1alpha1.SandboxConfigInformer, assets sandboxAssetFetcher, workersDir string) (*workerSandboxSyncer, error) { +// A syncer that cannot start is fatal to atelet: without it no runtime is +// ever indexed, so no actor could run. A syncer that cannot sync does not +// hold up serving: every runtime a worker registered before a restart was +// loaded from disk with the index, so while the informers have not synced, +// or a list/watch fails (e.g. Forbidden while an RBAC rollout lags the +// binary), existing files and index entries stay as they are and only +// changes wait. +func startWorkerSandboxSyncer(ctx context.Context, pods coreinformers.PodInformer, workerPools informersv1alpha1.WorkerPoolInformer, configs informersv1alpha1.SandboxConfigInformer, assets sandboxAssetFetcher, workersDir string, index *runtimeIndex, prewarm func(hash string)) (*workerSandboxSyncer, error) { if err := pods.Informer().AddIndexers(cache.Indexers{podUIDIndex: func(obj any) ([]string, error) { pod, ok := obj.(*corev1.Pod) if !ok { @@ -166,7 +181,7 @@ func startWorkerSandboxSyncer(ctx context.Context, pods coreinformers.PodInforme workerPools: workerPools.Lister(), configs: configs.Lister(), } - s := newWorkerSandboxSyncer(resolver, pods.Informer().GetIndexer(), assets, workersDir) + s := newWorkerSandboxSyncer(resolver, pods.Informer().GetIndexer(), assets, workersDir, index, prewarm) if err := watch(ctx, "worker pod", pods.Informer(), s.podHandler()); err != nil { return nil, err } @@ -263,11 +278,20 @@ func (s *workerSandboxSyncer) sandboxConfigHandler() cache.ResourceEventHandler // enqueuePool enqueues every local worker pod of pool, each after a random // delay below maxDelay when it is not zero. +// +// A delayed pool has its runtimes registered first, so the index knows a +// config's runtimes as soon as the listers do rather than once the files +// are rewritten. A request for a runtime the index does not know fails the +// actor, and the registration costs no IO. func (s *workerSandboxSyncer) enqueuePool(pool *v1alpha1.WorkerPool, maxDelay time.Duration) { pods, err := corelisters.NewPodLister(s.pods).Pods(pool.Namespace).List(labels.SelectorFromSet(labels.Set{workerPoolLabel: pool.Name})) - if err != nil { + if err != nil || len(pods) == 0 { return } + if maxDelay > 0 { + // Once per pool: every pod of it plans the same config. + s.registerAhead(pool) + } for _, pod := range pods { if maxDelay > 0 { s.queue.AddAfter(string(pod.UID), rand.N(maxDelay)) @@ -277,6 +301,26 @@ func (s *workerSandboxSyncer) enqueuePool(pool *v1alpha1.WorkerPool, maxDelay ti } } +// registerAhead makes the runtimes of pool's current plan resolvable before +// its pods' syncs run. Planning and hashing read the listers only, so this +// is cheap enough for an event handler. Only the hashes are registered; what +// a pod advertises changes when its sync writes the file. A pool that does +// not plan is left to the syncs, which report and retry it. +func (s *workerSandboxSyncer) registerAhead(pool *v1alpha1.WorkerPool) { + plan, err := s.resolver.planPool(pool) + if err != nil { + return + } + hashed, err := hashVersions(context.Background(), plan) + if err != nil { + return + } + if err := s.index.registerRuntimes(plan.config, hashed); err != nil { + // The syncs write it again, and report it if it keeps failing. + slog.Warn("Could not index a changed pool's runtimes ahead of its sync", slog.String("pool", pool.Name), slog.Any("err", err)) + } +} + // enqueueExistingDirs enqueues the UID of every worker directory on disk, so // one left by a pod that went away while atelet was down is removed. A // directory younger than minAge is enqueued once it reaches that age. @@ -399,6 +443,7 @@ func (s *workerSandboxSyncer) finish(ctx context.Context, uid string, err error) } func (s *workerSandboxSyncer) removeDir(ctx context.Context, uid string) error { + s.index.forget(uid) if err := os.RemoveAll(s.podDir(uid)); err != nil { return fmt.Errorf("while removing worker directory: %w", err) } diff --git a/cmd/atelet/worker_sandbox_syncer_test.go b/cmd/atelet/worker_sandbox_syncer_test.go index fc0ebfcd01..84aa2bbbda 100644 --- a/cmd/atelet/worker_sandbox_syncer_test.go +++ b/cmd/atelet/worker_sandbox_syncer_test.go @@ -23,6 +23,7 @@ import ( "slices" "strings" "testing" + "testing/synctest" "time" "github.com/google/go-cmp/cmp" @@ -167,7 +168,9 @@ func newTestSyncer(t *testing.T, objs ...any) *workerSandboxSyncer { s := newWorkerSandboxSyncer(&workerSandboxResolver{ workerPools: listersv1alpha1.NewWorkerPoolLister(pools), configs: listersv1alpha1.NewSandboxConfigLister(configs), - }, pods, newTestAssetFetcher(), t.TempDir()) + }, pods, newTestAssetFetcher(), t.TempDir(), newRuntimeIndex(), nil) + // The queue starts a goroutine; stop it with the test. + t.Cleanup(s.queue.ShutDown) s.scratchDir = t.TempDir() // the runsc probe stages the binary there s.probeCPUIdentity = func() (string, string, error) { return testCPUVendor, testCPUModel, nil } s.runCPUFeatures = func(context.Context, string) ([]string, error) { return testCPUFeatures(), nil } @@ -348,20 +351,70 @@ func TestWorkerSandboxSyncerJitter(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := newTestSyncer(t, append(pods, pool)...) - s.maxJitter = 50 * time.Millisecond - tt.event(s) - if diff := cmp.Diff(tt.want, drain(s)); diff != "" { - t.Errorf("UIDs enqueued right away mismatch (-want +got):\n%s", diff) - } - time.Sleep(2 * s.maxJitter) - if diff := cmp.Diff(tt.wantLater, drain(s)); diff != "" { - t.Errorf("UIDs enqueued after the jitter mismatch (-want +got):\n%s", diff) - } + // On the bubble's clock the sleep fires every delay at once. + synctest.Test(t, func(t *testing.T) { + s := newTestSyncer(t, append(pods, pool)...) + s.maxJitter = syncMaxJitter + tt.event(s) + synctest.Wait() + if diff := cmp.Diff(tt.want, drain(s)); diff != "" { + t.Errorf("UIDs enqueued right away mismatch (-want +got):\n%s", diff) + } + time.Sleep(s.maxJitter) + synctest.Wait() + if diff := cmp.Diff(tt.wantLater, drain(s)); diff != "" { + t.Errorf("UIDs enqueued after the jitter mismatch (-want +got):\n%s", diff) + } + }) }) } } +// TestWorkerSandboxSyncerRegistersAheadOfJitter pins what closes the window +// between a SandboxConfig change and the jittered sync: the config's +// runtimes resolve as soon as the event is handled, while the file, and +// with it what the worker advertises, waits for the sync. +func TestWorkerSandboxSyncerRegistersAheadOfJitter(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + // Two pods of one pool: the pool is planned once, both pods are delayed. + pod := workerPod("pod-a", "pool") + s := newTestSyncer(t, pod, workerPod("pod-b", "pool"), resolvedPool("pool", "gvisor-release"), multiVersionConfig()) + s.maxJitter = syncMaxJitter + + cfg := multiVersionConfig() + cfg.Generation = 2 + s.sandboxConfigHandler().OnUpdate(multiVersionConfig(), cfg) + synctest.Wait() + + plan, err := s.resolver.plan(pod) + if err != nil { + t.Fatal(err) + } + hashed, err := hashVersions(context.Background(), plan) + if err != nil { + t.Fatal(err) + } + for _, v := range hashed { + if _, ok := s.index.lookup(v.hash); !ok { + t.Errorf("runtime %s (%s) does not resolve before the jittered sync", v.name, v.hash) + } + } + for _, uid := range []string{"pod-a", "pod-b"} { + if _, ok := s.index.worker(uid); ok { + t.Errorf("%s advertises runtimes before its file is written", uid) + } + } + if got := drain(s); len(got) != 0 { + t.Errorf("enqueued %v right away, want the jittered delay", got) + } + time.Sleep(s.maxJitter) + synctest.Wait() + if diff := cmp.Diff([]string{"pod-a", "pod-b"}, drain(s)); diff != "" { + t.Errorf("UIDs enqueued after the jitter mismatch (-want +got):\n%s", diff) + } + }) +} + func TestWorkerSandboxSyncerProcess(t *testing.T) { ctx := context.Background() deleting := workerPod("pod-a", "pool") @@ -384,45 +437,59 @@ func TestWorkerSandboxSyncerProcess(t *testing.T) { // wantUntouched is whether the existing file must not be rewritten. wantUntouched bool wantRetries int + // indexed registers pod-a in the index before processing, as an + // earlier sync would have; wantIndexed is whether it is there after. + indexed, wantIndexed bool }{{ - name: "writes the file", - objs: []any{workerPod("pod-a", "pool"), resolvedPool("pool", "gvisor-release"), multiVersionConfig()}, - want: wantVersions(), - wantDir: true, + name: "writes the file", + wantIndexed: true, + objs: []any{workerPod("pod-a", "pool"), resolvedPool("pool", "gvisor-release"), multiVersionConfig()}, + want: wantVersions(), + wantDir: true, }, { - name: "replaces a stale file", - objs: []any{workerPod("pod-a", "pool"), resolvedPool("pool", "gvisor-release"), multiVersionConfig()}, - existing: stale, - want: wantVersions(), - wantDir: true, + name: "replaces a stale file", + wantIndexed: true, + objs: []any{workerPod("pod-a", "pool"), resolvedPool("pool", "gvisor-release"), multiVersionConfig()}, + existing: stale, + want: wantVersions(), + wantDir: true, }, { name: "leaves an equal file alone", + wantIndexed: true, objs: []any{workerPod("pod-a", "pool"), resolvedPool("pool", "gvisor-release"), multiVersionConfig()}, existing: wantVersions(), want: wantVersions(), wantDir: true, wantUntouched: true, }, { - // The Delete event removes the directory. - name: "pod being deleted", - objs: []any{deleting, resolvedPool("pool", "gvisor-release"), multiVersionConfig()}, - existing: stale, - want: stale, - wantDir: true, + // The Delete event removes the directory, and the index entry with it. + name: "pod being deleted", + objs: []any{deleting, resolvedPool("pool", "gvisor-release"), multiVersionConfig()}, + existing: stale, + indexed: true, + want: stale, + wantDir: true, + wantIndexed: true, }, { name: "pod gone", objs: []any{resolvedPool("pool", "gvisor-release"), multiVersionConfig()}, existing: stale, + indexed: true, }, { - name: "pod gone, no directory", - objs: []any{resolvedPool("pool", "gvisor-release"), multiVersionConfig()}, + name: "pod gone, no directory", + objs: []any{resolvedPool("pool", "gvisor-release"), multiVersionConfig()}, + indexed: true, }, { + // A sync that fails before the write leaves the index entry as it + // left the file. name: "unresolved pool keeps the existing file", objs: []any{workerPod("pod-a", "pool"), multiVersionConfig()}, existing: stale, + indexed: true, want: stale, wantDir: true, wantRetries: 1, + wantIndexed: true, }, { name: "unresolved pool retries up to syncMaxRetries", objs: []any{workerPod("pod-a", "pool"), multiVersionConfig()}, @@ -452,8 +519,9 @@ func TestWorkerSandboxSyncerProcess(t *testing.T) { }, { // A transient failure on a restorable version writes the default // runtime right away and requeues without the syncMaxRetries cap. - name: "incomplete restorable version writes partial file and requeues past syncMaxRetries", - objs: []any{workerPod("pod-a", "pool"), resolvedPool("pool", "gvisor-release"), multiVersionConfig()}, + name: "incomplete restorable version writes partial file and requeues past syncMaxRetries", + wantIndexed: true, + objs: []any{workerPod("pod-a", "pool"), resolvedPool("pool", "gvisor-release"), multiVersionConfig()}, assets: &fakeAssetFetcher{ fn: func(ctx context.Context, rec *sandboxAssetsRecord) (map[string]string, error) { if rec.Assets[gvisorAssetName].SHA256 != "sha-v2" { @@ -476,6 +544,9 @@ func TestWorkerSandboxSyncerProcess(t *testing.T) { if tt.assets != nil { s.assets = tt.assets } + if tt.indexed { + s.index.register("pod-a") + } limiter := workqueue.NewTypedItemExponentialFailureRateLimiter[string](time.Second, 5*time.Minute) for range tt.priorRetries { limiter.When("pod-a") @@ -523,6 +594,9 @@ func TestWorkerSandboxSyncerProcess(t *testing.T) { if got := s.queue.NumRequeues("pod-a"); got != tt.wantRetries { t.Errorf("NumRequeues = %d, want %d", got, tt.wantRetries) } + if _, ok := s.index.worker("pod-a"); ok != tt.wantIndexed { + t.Errorf("pod-a in the index = %v, want %v", ok, tt.wantIndexed) + } requeued := make(chan struct{}) go func() { defer close(requeued) @@ -574,38 +648,45 @@ func TestWorkerSandboxSyncerExistingDirs(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := newTestSyncer(t) - // Short enough to observe a delayed enqueue. - s.minAge = 50 * time.Millisecond - if tt.entries == nil { - s.workersDir = filepath.Join(s.workersDir, "missing") - } - for name, kind := range tt.entries { - path := filepath.Join(s.workersDir, name) - var err error - switch kind { - case file: - err = os.WriteFile(path, nil, 0o600) - case youngDir: - err = os.Mkdir(path, 0o700) - case oldDir: - if err = os.Mkdir(path, 0o700); err == nil { - old := time.Now().Add(-2 * orphanMinAge) - err = os.Chtimes(path, old, old) + // On the bubble's clock the sleep fires the delayed enqueue at + // once. Directory ages are stamped from that clock too. + synctest.Test(t, func(t *testing.T) { + s := newTestSyncer(t) + if tt.entries == nil { + s.workersDir = filepath.Join(s.workersDir, "missing") + } + now := time.Now() + for name, kind := range tt.entries { + path := filepath.Join(s.workersDir, name) + var err error + switch kind { + case file: + err = os.WriteFile(path, nil, 0o600) + case youngDir: + if err = os.Mkdir(path, 0o700); err == nil { + err = os.Chtimes(path, now, now) + } + case oldDir: + if err = os.Mkdir(path, 0o700); err == nil { + old := now.Add(-2 * s.minAge) + err = os.Chtimes(path, old, old) + } + } + if err != nil { + t.Fatal(err) } } - if err != nil { - t.Fatal(err) + s.enqueueExistingDirs(context.Background()) + synctest.Wait() + if diff := cmp.Diff(tt.want, drain(s)); diff != "" { + t.Errorf("enqueued UIDs mismatch (-want +got):\n%s", diff) } - } - s.enqueueExistingDirs(context.Background()) - if diff := cmp.Diff(tt.want, drain(s)); diff != "" { - t.Errorf("enqueued UIDs mismatch (-want +got):\n%s", diff) - } - time.Sleep(2 * s.minAge) - if diff := cmp.Diff(tt.wantLater, drain(s)); diff != "" { - t.Errorf("UIDs enqueued after minAge mismatch (-want +got):\n%s", diff) - } + time.Sleep(s.minAge) + synctest.Wait() + if diff := cmp.Diff(tt.wantLater, drain(s)); diff != "" { + t.Errorf("UIDs enqueued after minAge mismatch (-want +got):\n%s", diff) + } + }) }) } } @@ -771,7 +852,7 @@ func TestStartWorkerSandboxSyncer(t *testing.T) { ateFactory.Api().V1alpha1().WorkerPools(), ateFactory.Api().V1alpha1().SandboxConfigs(), newScriptAssetFetcher(t), - dir) + dir, newRuntimeIndex(), nil) if err != nil { t.Fatalf("startWorkerSandboxSyncer() error = %v", err) } @@ -795,3 +876,38 @@ func TestStartWorkerSandboxSyncer(t *testing.T) { }) } } + +// TestWorkerSandboxSyncerRunIndexesExistingPods pins what run does with the +// pods already on the node: each is processed once the caches synced, so a +// pod that resolves is indexed, and one whose pool does not exist is not +// and is retried in the background without holding anything else. +func TestWorkerSandboxSyncerRunIndexesExistingPods(t *testing.T) { + s := newTestSyncer(t, workerPod("pod-live", "pool"), workerPod("pod-orphan", "missing-pool"), resolvedPool("pool", "gvisor-release"), multiVersionConfig()) + // The initial list replays each pod as an Add before the caches sync. + s.queue.Add("pod-live") + s.queue.Add("pod-orphan") + ctx, cancel := context.WithCancel(context.Background()) + stopped := make(chan struct{}) + go func() { + defer close(stopped) + s.run(ctx, func() bool { return true }) + }() + t.Cleanup(func() { + cancel() + <-stopped + }) + + waitFor(t, "pod-live to be indexed", func() bool { _, ok := s.index.worker("pod-live"); return ok }) + wr, _ := s.index.worker("pod-live") + cr, ok := wr.class(v1alpha1.SandboxClassGvisor) + if !ok || cr.config != "gvisor-release" || cr.defaultHash == "" { + t.Errorf("index.worker(pod-live).class(gvisor) = %+v, %v; want gvisor-release with a default runtime", cr, ok) + } + if _, ok := s.index.lookup(cr.defaultHash); !ok { + t.Error("pod-live's default runtime is not resolvable") + } + waitFor(t, "pod-orphan to be retried", func() bool { return s.queue.NumRequeues("pod-orphan") > 0 }) + if _, ok := s.index.worker("pod-orphan"); ok { + t.Error("a pod whose pool does not exist was indexed") + } +} diff --git a/cmd/atelet/worker_sandbox_versions.go b/cmd/atelet/worker_sandbox_versions.go index fefdffb667..7181306bf8 100644 --- a/cmd/atelet/worker_sandbox_versions.go +++ b/cmd/atelet/worker_sandbox_versions.go @@ -19,7 +19,7 @@ package main // - resolve: find the SandboxConfig of the pod's WorkerPool // - plan: pick the config's versions this node can run // - build: hash each version's assets and pause image digests into its -// sandbox_runtime_hash, and probe what a snapshot taken on the runtime +// sandbox_bundle_hash, and probe what a snapshot taken on the runtime // must find again: the host CPU identity for microVM, the runsc // binary's CPU features for gVisor // - write: replace the file in the worker directory if its content changed @@ -117,6 +117,15 @@ func poolConfigName(pool *v1alpha1.WorkerPool) string { // a pod in no pool, every failure can succeed once an informer catches up or // the atecontroller resolves the pool. func (r *workerSandboxResolver) resolve(pod *corev1.Pod) (*v1alpha1.SandboxConfig, error) { + pool, err := r.pool(pod) + if err != nil { + return nil, err + } + return r.config(pool) +} + +// pool returns the WorkerPool a worker pod on this node belongs to. +func (r *workerSandboxResolver) pool(pod *corev1.Pod) (*v1alpha1.WorkerPool, error) { poolName := pod.Labels[workerPoolLabel] if poolName == "" { return nil, fmt.Errorf("%w: worker pod %s/%s has no %s label", errPermanent, pod.Namespace, pod.Name, workerPoolLabel) @@ -125,9 +134,14 @@ func (r *workerSandboxResolver) resolve(pod *corev1.Pod) (*v1alpha1.SandboxConfi if err != nil { return nil, fmt.Errorf("%w: while getting WorkerPool %s/%s: %w", errUnresolved, pod.Namespace, poolName, err) } + return pool, nil +} + +// config returns the SandboxConfig the workers of pool run. +func (r *workerSandboxResolver) config(pool *v1alpha1.WorkerPool) (*v1alpha1.SandboxConfig, error) { configName := poolConfigName(pool) if configName == "" { - return nil, fmt.Errorf("%w: WorkerPool %s/%s has not resolved a SandboxConfig for sandbox class %q", errUnresolved, pod.Namespace, poolName, pool.Spec.DefaultSandboxClass()) + return nil, fmt.Errorf("%w: WorkerPool %s/%s has not resolved a SandboxConfig for sandbox class %q", errUnresolved, pool.Namespace, pool.Name, pool.Spec.DefaultSandboxClass()) } cfg, err := r.configs.Get(configName) if err != nil { @@ -138,7 +152,17 @@ func (r *workerSandboxResolver) resolve(pod *corev1.Pod) (*v1alpha1.SandboxConfi // plan resolves pod's SandboxConfig and the versions of it this node can run. func (r *workerSandboxResolver) plan(pod *corev1.Pod) (*sandboxPlan, error) { - cfg, err := r.resolve(pod) + pool, err := r.pool(pod) + if err != nil { + return nil, err + } + return r.planPool(pool) +} + +// planPool is plan for a pool already in hand: the config its workers run +// and the versions of it this node can run. +func (r *workerSandboxResolver) planPool(pool *v1alpha1.WorkerPool) (*sandboxPlan, error) { + cfg, err := r.config(pool) if err != nil { return nil, err } @@ -195,14 +219,41 @@ func recordForArch(cfg *v1alpha1.SandboxConfig, ver *v1alpha1.SandboxVersionConf return rec, true } -// buildFile returns the file content naming the runtimes of the versions in -// plan. A version whose runtime hash equals an earlier one's is left out, so -// the file lists each runtime once. The default version's failure fails the -// sync, which leaves the existing file in place until a retry succeeds. A -// restorable version whose attributes cannot be built is left out; when the -// failure is transient the file is returned with an error wrapping -// errIncomplete, so the caller writes the runtimes that resolved right away -// and retries for the rest. +// hashedVersion is one version of a plan with its bundle hash. +type hashedVersion struct { + versionPlan + hash string +} + +// hashVersions hashes each version of plan. The hashes come from the +// SandboxConfig spec alone, so it does no IO. A version that cannot be +// hashed is left out, unless it is the default, without which no worker of +// the pool could start actors on this node. +func hashVersions(ctx context.Context, plan *sandboxPlan) ([]hashedVersion, error) { + hashed := make([]hashedVersion, 0, len(plan.versions)) + for _, v := range plan.versions { + h, err := versionBundleHash(v.rec) + if err != nil { + err = fmt.Errorf("SandboxConfig %q, version %q: %w", plan.config, v.name, err) + if v.isDefault { + return nil, fmt.Errorf("%w: %w", errPermanent, err) + } + slog.WarnContext(ctx, "Leaving a sandbox version out of the worker sandbox runtimes file", slog.Any("err", err)) + continue + } + hashed = append(hashed, hashedVersion{versionPlan: v, hash: h}) + } + return hashed, nil +} + +// buildFile returns the file content naming the runtimes of the hashed +// versions, the default first. A version whose bundle hash equals an +// earlier one's is left out, so the file lists each runtime once. The +// default version's failure fails the sync, which leaves the existing file +// in place until a retry succeeds. A restorable version whose attributes +// cannot be built is left out; when the failure is transient the file is +// returned with an error wrapping errIncomplete, so the caller writes the +// runtimes that resolved right away and retries for the rest. // // TODO: write the file as soon as the default version's attributes are known // and add restorable runtimes on a later pass. On a fresh node the first @@ -211,22 +262,18 @@ func recordForArch(cfg *v1alpha1.SandboxConfig, ver *v1alpha1.SandboxVersionConf // // Listing a runtime does not promise its assets are on the node: probing // fetches a gVisor version's runsc the first time this atelet sees it, but -// not its pause image, and nothing of a microVM version. The prewarmer -// fetches each config's default version, and Run/Restore fetch any other -// version on demand. -func (s *workerSandboxSyncer) buildFile(ctx context.Context, plan *sandboxPlan) (*sandboxruntimes.File, error) { +// not its pause image, and nothing of a microVM version. The syncer has +// the default runtime prewarmed once the file is written, and Run/Restore +// fetch any other version on demand. +func (s *workerSandboxSyncer) buildFile(ctx context.Context, plan *sandboxPlan, hashed []hashedVersion) (*sandboxruntimes.File, error) { out := &sandboxruntimes.File{} - seen := make(map[string]bool, len(plan.versions)) + seen := make(map[string]bool, len(hashed)) var incomplete error - for _, v := range plan.versions { - h, err := versionRuntimeHash(v.rec) - if err == nil && seen[h] { + for _, v := range hashed { + if seen[v.hash] { continue } - var attrs sandboxruntimes.Attributes - if err == nil { - attrs, err = s.versionAttrs(ctx, plan.sandboxClass, v.rec, h) - } + attrs, err := s.versionAttrs(ctx, plan.sandboxClass, v.rec, v.hash) if err != nil { err = fmt.Errorf("SandboxConfig %q, version %q: %w", plan.config, v.name, err) if v.isDefault { @@ -242,7 +289,7 @@ func (s *workerSandboxSyncer) buildFile(ctx context.Context, plan *sandboxPlan) } continue } - seen[h] = true + seen[v.hash] = true if v.isDefault { out.DefaultRuntime = attrs } else { @@ -256,13 +303,13 @@ func (s *workerSandboxSyncer) buildFile(ctx context.Context, plan *sandboxPlan) } // versionAttrs returns the attributes of one version's runtime, whose -// sandbox_runtime_hash is hash: architecture and the hash for every class, +// sandbox_bundle_hash is hash: architecture and the hash for every class, // the host CPU identity for a microVM runtime, and the runsc binary's // cpu_features for a gVisor one. func (s *workerSandboxSyncer) versionAttrs(ctx context.Context, class v1alpha1.SandboxClass, rec *sandboxAssetsRecord, hash string) (sandboxruntimes.Attributes, error) { attrs := sandboxruntimes.Attributes{ - hardware.AttrArchitecture: runtime.GOARCH, - hardware.AttrSandboxRuntimeHash: hash, + hardware.AttrArchitecture: runtime.GOARCH, + hardware.AttrSandboxBundleHash: hash, } switch class { case v1alpha1.SandboxClassMicroVM: @@ -306,20 +353,18 @@ func (s *workerSandboxSyncer) hostCPUIdentity() (vendor, model string, err error // The first time this atelet sees the binary's digest it fetches the binary // into the local asset cache and runs ` cpu-features`. func (s *workerSandboxSyncer) gvisorCPUFeatures(ctx context.Context, rec *sandboxAssetsRecord) (string, error) { - asset, ok := runscAssetOf(rec) - if !ok { - return "", fmt.Errorf("%w: gvisor version has no %q or %q asset", errPermanent, gvisorAssetName, runscAssetName) + digest := rec.Assets[gvisorAssetName].SHA256 + if digest == "" { + return "", fmt.Errorf("%w: gvisor version has no %q asset", errPermanent, gvisorAssetName) } - digest := rec.Assets[asset].SHA256 s.mu.Lock() mask, ok := s.cpuFeatures[digest] s.mu.Unlock() if ok { return mask, nil } - // A download that hangs would hold a sync worker; prewarmTimeout bounds - // the fetch for the same reason it bounds prewarming. - ctx, cancel := context.WithTimeout(ctx, prewarmTimeout) + // Backstop for the sync worker this runs on; see probeFetchTimeout. + ctx, cancel := context.WithTimeout(ctx, probeFetchTimeout) defer cancel() // The probe has no actor to stage the binary for: it links it into a // scratch dir it removes again, and the cache keeps the entry. @@ -332,7 +377,7 @@ func (s *workerSandboxSyncer) gvisorCPUFeatures(ctx context.Context, rec *sandbo if err != nil { return "", err } - names, err := s.runCPUFeatures(ctx, paths[asset]) + names, err := s.runCPUFeatures(ctx, runscPathFor(paths)) if err != nil { return "", err } @@ -346,16 +391,14 @@ func (s *workerSandboxSyncer) gvisorCPUFeatures(ctx context.Context, rec *sandbo return mask, nil } -// runscAssetOf returns the name of the asset in rec that carries runsc, the -// most preferred of runscAssetNames that is present with a digest. -func runscAssetOf(rec *sandboxAssetsRecord) (string, bool) { - for _, name := range runscAssetNames { - if a, ok := rec.Assets[name]; ok && a.SHA256 != "" { - return name, true - } - } - return "", false -} +// probeFetchTimeout bounds the fetch of a runsc binary for its probe, from +// the first cache lookup to the staged link. The asset cache bounds each +// download with its own fetch timeout, and the exec below has +// cpuFeaturesTimeout; this is the backstop for everything around them, +// such as waiting on a download another caller started, so a probe cannot +// hold one of the syncer's few workers for longer than one download may +// take. +const probeFetchTimeout = 5 * time.Minute // cpuFeaturesTimeout bounds one `runsc cpu-features` run, which takes // milliseconds. A runsc that hangs would hold a sync worker. A var so tests @@ -394,11 +437,11 @@ func runRunscCPUFeatures(ctx context.Context, runscPath string) ([]string, error return strings.Split(printed, ","), nil } -// versionRuntimeHash returns the runtime hash of one version. The CRD -// requires every pause image to be pinned by digest, which is the digest the -// image cache resolves the ref to, so the hash needs no pull. An unpinned one -// is errPermanent: only a spec change fixes it. -func versionRuntimeHash(rec *sandboxAssetsRecord) (string, error) { +// versionBundleHash returns the bundle hash of one version. The CRD requires +// every pause image to be pinned by digest, which is the digest the image +// cache resolves the ref to, so the hash needs no pull. An unpinned one is +// errPermanent: only a spec change fixes it. +func versionBundleHash(rec *sandboxAssetsRecord) (string, error) { var pauseDigest string if rec.PauseImage != "" { ref, err := name.NewDigest(rec.PauseImage) @@ -407,13 +450,14 @@ func versionRuntimeHash(rec *sandboxAssetsRecord) (string, error) { } pauseDigest = ref.DigestStr() } - return runtimeHash(rec, pauseDigest), nil + return bundleHash(rec, pauseDigest), nil } -// runtimeHash is the sandbox_runtime_hash of a version: a sha256 over one -// sorted "key=value" line per asset digest and pause image digest. Version names and URLs are left out, so the same binaries under -// another name or in another bucket are the same runtime. -func runtimeHash(rec *sandboxAssetsRecord, pauseDigest string) string { +// bundleHash is the sandbox_bundle_hash of a version: a sha256 over one +// sorted "key=value" line per asset digest and pause image digest. Version +// names and URLs are left out, so the same binaries under another name or in +// another bucket are the same runtime. +func bundleHash(rec *sandboxAssetsRecord, pauseDigest string) string { lines := make([]string, 0, len(rec.Assets)+1) for name, a := range rec.Assets { lines = append(lines, "asset/"+name+"="+a.SHA256) @@ -428,8 +472,14 @@ func runtimeHash(rec *sandboxAssetsRecord, pauseDigest string) string { // syncPod writes pod's sandbox runtimes file: // - plan: resolve the pool's config and its versions for this node -// - build: hash and probe each version's runtime -// - write: replace the file if its content changed +// - hash: hash each version's runtime, and make each resolvable by its +// hash before the file can let the worker register it +// - prewarm: have the default runtime's assets fetched, off this +// goroutine, so the probe below is its head start and the file never +// waits on a download +// - build: probe what a snapshot taken on each runtime must find again +// - write: replace the file if its content changed, then record what +// the worker now advertises in the index // // A file missing restorable runtimes for a transient reason is written all // the same, and the errIncomplete from build is returned so finish retries. @@ -438,16 +488,60 @@ func (s *workerSandboxSyncer) syncPod(ctx context.Context, pod *corev1.Pod) erro if err != nil { return err } - want, buildErr := s.buildFile(ctx, plan) + hashed, err := hashVersions(ctx, plan) + if err != nil { + return err + } + // Resolvable, and on disk, before the file can let the worker register + // any of them. + if err := s.index.registerRuntimes(plan.config, hashed); err != nil { + return err + } + // Queued now rather than after the write: the build fetches and probes + // every restorable version serially, longest on a fresh node, and the + // file it ends with is what lets the worker take its first actor. The + // prewarm gets that whole duration to finish first. + if s.prewarm != nil { + for _, v := range hashed { + if v.isDefault { + s.prewarm(v.hash) + } + } + } + want, buildErr := s.buildFile(ctx, plan, hashed) if buildErr != nil && !errors.Is(buildErr, errIncomplete) { return buildErr } if err := s.write(ctx, string(pod.UID), plan.config, want); err != nil { return err } + // What the worker advertises is what its file now says: a sync that + // failed before the write left the previous entry, and one that wrote + // a partial file names only the runtimes in it. One class per pool + // today; a pool that runs several will yield one plan per class here. + if err := s.index.register(string(pod.UID), advertisedClass{class: plan.sandboxClass, config: plan.config, hashed: writtenVersions(hashed, want)}); err != nil { + return err + } return buildErr } +// writtenVersions returns the versions of hashed whose runtime file names, +// in hashed's order: every version whose attributes were built. +func writtenVersions(hashed []hashedVersion, file *sandboxruntimes.File) []hashedVersion { + written := make(map[string]bool, 1+len(file.RestorableRuntimes)) + written[file.DefaultRuntime[hardware.AttrSandboxBundleHash]] = true + for _, attrs := range file.RestorableRuntimes { + written[attrs[hardware.AttrSandboxBundleHash]] = true + } + out := make([]hashedVersion, 0, len(hashed)) + for _, v := range hashed { + if written[v.hash] { + out = append(out, v) + } + } + return out +} + func (s *workerSandboxSyncer) write(ctx context.Context, uid, config string, want *sandboxruntimes.File) error { if err := os.MkdirAll(s.podDir(uid), 0o700); err != nil { return fmt.Errorf("while creating worker directory: %w", err) @@ -460,7 +554,7 @@ func (s *workerSandboxSyncer) write(ctx context.Context, uid, config string, wan slog.InfoContext(ctx, "Wrote worker sandbox runtimes", slog.String("pod_uid", uid), slog.String("sandbox_config", config), - slog.String("default_runtime_hash", want.DefaultRuntime[hardware.AttrSandboxRuntimeHash]), + slog.String("default_bundle_hash", want.DefaultRuntime[hardware.AttrSandboxBundleHash]), slog.Int("restorable_runtimes", len(want.RestorableRuntimes))) } return nil diff --git a/cmd/atelet/worker_sandbox_versions_test.go b/cmd/atelet/worker_sandbox_versions_test.go index c2870ebc01..fe7c00ea79 100644 --- a/cmd/atelet/worker_sandbox_versions_test.go +++ b/cmd/atelet/worker_sandbox_versions_test.go @@ -22,6 +22,7 @@ import ( "os" "path/filepath" "runtime" + "slices" "strings" "testing" "time" @@ -74,22 +75,22 @@ func multiVersionConfig() *v1alpha1.SandboxConfig { } func TestRecordForArch(t *testing.T) { - sha := fmt.Sprintf("%x", sha256.Sum256([]byte("runsc"))) + sha := fmt.Sprintf("%x", sha256.Sum256([]byte("gvisor release"))) tests := []struct { name string assets map[string]map[string]v1alpha1.AssetFile want *sandboxAssetsRecord }{{ name: "assets for this architecture", - assets: map[string]map[string]v1alpha1.AssetFile{runtime.GOARCH: {runscAssetName: {URL: "gs://bucket/runsc", SHA256: sha}}}, + assets: map[string]map[string]v1alpha1.AssetFile{runtime.GOARCH: {gvisorAssetName: {URL: "gs://bucket/gvisor", SHA256: sha}}}, want: &sandboxAssetsRecord{ SandboxClass: string(v1alpha1.SandboxClassGvisor), PauseImage: "registry.k8s.io/pause@sha256:abc", - Assets: map[string]assetEntry{runscAssetName: {URL: "gs://bucket/runsc", SHA256: sha}}, + Assets: map[string]assetEntry{gvisorAssetName: {URL: "gs://bucket/gvisor", SHA256: sha}}, }, }, { name: "assets for another architecture only", - assets: map[string]map[string]v1alpha1.AssetFile{"other-arch": {runscAssetName: {URL: "gs://bucket/runsc", SHA256: sha}}}, + assets: map[string]map[string]v1alpha1.AssetFile{"other-arch": {gvisorAssetName: {URL: "gs://bucket/gvisor", SHA256: sha}}}, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -106,7 +107,7 @@ func TestRecordForArch(t *testing.T) { } } -func TestRuntimeHash(t *testing.T) { +func TestBundleHash(t *testing.T) { gvisor := map[string]assetEntry{gvisorAssetName: {URL: "gs://bucket/gvisor", SHA256: "d547d81401461fd1c679c5c4fa0a6c2b8ef7dc3c22ce23c9e25dcc4c69cfd06f"}} pause := "sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" @@ -134,9 +135,9 @@ func TestRuntimeHash(t *testing.T) { want: "sha256:f86e6f2bd8e25ac31a22e51e6e57e0eb6741eb0b0df77082989cc1e40f906fe1", }, { name: "the asset under another name", - rec: &sandboxAssetsRecord{SandboxClass: "gvisor", Assets: map[string]assetEntry{runscAssetName: gvisor[gvisorAssetName]}}, + rec: &sandboxAssetsRecord{SandboxClass: "gvisor", Assets: map[string]assetEntry{"renamed": gvisor[gvisorAssetName]}}, pauseDigest: pause, - want: "sha256:91e241c70c94d67b16127f1ba9c239e76ed1e91cb99fd9587b071cb8ccf7d3d8", + want: "sha256:04bf18376ca0c7227ae9fa65675127b1045d7de33f05ea5d2375eb11a1a8e62e", }, { name: "several assets", rec: &sandboxAssetsRecord{SandboxClass: "microvm", Assets: map[string]assetEntry{ @@ -147,8 +148,8 @@ func TestRuntimeHash(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := runtimeHash(tt.rec, tt.pauseDigest); got != tt.want { - t.Errorf("runtimeHash() = %q, want %q", got, tt.want) + if got := bundleHash(tt.rec, tt.pauseDigest); got != tt.want { + t.Errorf("bundleHash() = %q, want %q", got, tt.want) } }) } @@ -191,7 +192,7 @@ func TestPoolConfigName(t *testing.T) { } func TestWorkerSandboxResolverResolve(t *testing.T) { - cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", "abc") + cfg := gvisorConfig("gvisor-default", "gs://bucket/gvisor", "abc") tests := []struct { name string @@ -280,7 +281,7 @@ func TestWorkerSandboxResolverResolve(t *testing.T) { } } -func TestVersionRuntimeHash(t *testing.T) { +func TestVersionBundleHash(t *testing.T) { gvisor := map[string]assetEntry{gvisorAssetName: {URL: "gs://bucket/gvisor", SHA256: "d547d81401461fd1c679c5c4fa0a6c2b8ef7dc3c22ce23c9e25dcc4c69cfd06f"}} pause := "sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" @@ -292,14 +293,14 @@ func TestVersionRuntimeHash(t *testing.T) { }{{ name: "pause image pinned by digest", pauseImage: "registry.k8s.io/pause:3.10.2@" + pause, - want: runtimeHash(&sandboxAssetsRecord{Assets: gvisor}, pause), + want: bundleHash(&sandboxAssetsRecord{Assets: gvisor}, pause), }, { name: "pause image pinned by digest only", pauseImage: "gcr.io/gke-release/pause@" + pause, - want: runtimeHash(&sandboxAssetsRecord{Assets: gvisor}, pause), + want: bundleHash(&sandboxAssetsRecord{Assets: gvisor}, pause), }, { name: "no pause image", - want: runtimeHash(&sandboxAssetsRecord{Assets: gvisor}, ""), + want: bundleHash(&sandboxAssetsRecord{Assets: gvisor}, ""), }, { // The CRD rejects this shape. name: "pause image by tag", @@ -308,15 +309,15 @@ func TestVersionRuntimeHash(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := versionRuntimeHash(&sandboxAssetsRecord{SandboxClass: "gvisor", PauseImage: tt.pauseImage, Assets: gvisor}) + got, err := versionBundleHash(&sandboxAssetsRecord{SandboxClass: "gvisor", PauseImage: tt.pauseImage, Assets: gvisor}) if (err != nil) != tt.wantErr { - t.Fatalf("versionRuntimeHash() error = %v, want error %v", err, tt.wantErr) + t.Fatalf("versionBundleHash() error = %v, want error %v", err, tt.wantErr) } if err != nil && !errors.Is(err, errPermanent) { - t.Errorf("versionRuntimeHash() error = %v, want errPermanent", err) + t.Errorf("versionBundleHash() error = %v, want errPermanent", err) } if got != tt.want { - t.Errorf("versionRuntimeHash() = %q, want %q", got, tt.want) + t.Errorf("versionBundleHash() = %q, want %q", got, tt.want) } }) } @@ -331,9 +332,9 @@ func wantRuntime(name string) sandboxruntimes.Attributes { func wantRuntimeWithFeatures(name, cpuFeatures string) sandboxruntimes.Attributes { rec := &sandboxAssetsRecord{Assets: map[string]assetEntry{gvisorAssetName: {SHA256: "sha-" + name}}} return sandboxruntimes.Attributes{ - hardware.AttrArchitecture: runtime.GOARCH, - hardware.AttrCPUFeatures: cpuFeatures, - hardware.AttrSandboxRuntimeHash: runtimeHash(rec, pauseDigest(name)), + hardware.AttrArchitecture: runtime.GOARCH, + hardware.AttrCPUFeatures: cpuFeatures, + hardware.AttrSandboxBundleHash: bundleHash(rec, pauseDigest(name)), } } @@ -351,7 +352,11 @@ func planAndBuild(s *workerSandboxSyncer, pod *corev1.Pod) (*sandboxruntimes.Fil if err != nil { return nil, err } - return s.buildFile(context.Background(), plan) + hashed, err := hashVersions(context.Background(), plan) + if err != nil { + return nil, err + } + return s.buildFile(context.Background(), plan, hashed) } func TestPlanAndBuild(t *testing.T) { @@ -381,8 +386,8 @@ func TestPlanAndBuild(t *testing.T) { unsupportedClass := multiVersionConfig() unsupportedClass.Spec.SandboxClass = "unknown" - noRunscAsset := multiVersionConfig() - noRunscAsset.Spec.Versions[3].Assets = map[string]map[string]v1alpha1.AssetFile{ + noGVisorAsset := multiVersionConfig() + noGVisorAsset.Spec.Versions[3].Assets = map[string]map[string]v1alpha1.AssetFile{ runtime.GOARCH: {"other": {URL: "gs://bucket/other", SHA256: "sha-other"}}, } @@ -428,8 +433,8 @@ func TestPlanAndBuild(t *testing.T) { wantErr: true, wantPermanent: true, }, { - name: "gvisor version has no gvisor or runsc asset", - cfg: noRunscAsset, + name: "gvisor version has no gvisor asset", + cfg: noGVisorAsset, wantErr: true, wantPermanent: true, }, { @@ -482,10 +487,10 @@ func TestPlanAndBuildMicroVM(t *testing.T) { wantMicroVM := func(sha string) sandboxruntimes.Attributes { rec := &sandboxAssetsRecord{Assets: map[string]assetEntry{"kernel": {SHA256: sha}}} return sandboxruntimes.Attributes{ - hardware.AttrArchitecture: runtime.GOARCH, - hardware.AttrCPUVendor: testCPUVendor, - hardware.AttrCPUModel: testCPUModel, - hardware.AttrSandboxRuntimeHash: runtimeHash(rec, ""), + hardware.AttrArchitecture: runtime.GOARCH, + hardware.AttrCPUVendor: testCPUVendor, + hardware.AttrCPUModel: testCPUModel, + hardware.AttrSandboxBundleHash: bundleHash(rec, ""), } } @@ -536,7 +541,7 @@ func TestGVisorCPUFeaturesMemoizationAndErrors(t *testing.T) { feats := testCPUFeatures() t.Run("a version with an earlier version's binaries in another bucket is neither fetched nor probed", func(t *testing.T) { - // runtimeHash leaves URLs out, so v3 is v1's runtime. The syncer must + // bundleHash leaves URLs out, so v3 is v1's runtime. The syncer must // see that from the spec alone, before touching v3's bucket. cfg := multiVersionConfig() v3 := *cfg.Spec.Versions[1].DeepCopy() @@ -565,7 +570,7 @@ func TestGVisorCPUFeaturesMemoizationAndErrors(t *testing.T) { t.Run("memoizes runsc cpu-features by asset digest across versions and syncs", func(t *testing.T) { cfg := multiVersionConfig() // Add a third version "v3" that shares v1's gvisor asset digest ("sha-v1") - // but has its own pause image digest, so it has a distinct runtime hash + // but has its own pause image digest, so it has a distinct bundle hash // while reusing the same runsc binary. v3 := *cfg.Spec.Versions[1].DeepCopy() v3.Name = "v3" @@ -597,9 +602,9 @@ func TestGVisorCPUFeaturesMemoizationAndErrors(t *testing.T) { RestorableRuntimes: []sandboxruntimes.Attributes{ wantRuntimeWithFeatures("v1", "1"), { - hardware.AttrArchitecture: runtime.GOARCH, - hardware.AttrCPUFeatures: "1", - hardware.AttrSandboxRuntimeHash: runtimeHash(v3Rec, pauseDigest("v3")), + hardware.AttrArchitecture: runtime.GOARCH, + hardware.AttrCPUFeatures: "1", + hardware.AttrSandboxBundleHash: bundleHash(v3Rec, pauseDigest("v3")), }, }, } @@ -742,3 +747,69 @@ func TestRunRunscCPUFeatures(t *testing.T) { t.Errorf("runRunscCPUFeatures() on a hanging binary = %v, want context.DeadlineExceeded", err) } } + +// TestSyncPodAdvertisesWhatItWrote pins when a worker's index entry changes: +// with its file. A sync that fails before the write leaves the entry of the +// file still on disk, and one that writes a partial file names only the +// runtimes in it, while every hashed runtime resolves either way. +func TestSyncPodAdvertisesWhatItWrote(t *testing.T) { + ctx := context.Background() + pod := workerPod("pod-a", "pool") + hashOf := func(name string) string { return wantRuntime(name)[hardware.AttrSandboxBundleHash] } + advertised := func(t *testing.T, s *workerSandboxSyncer) *classRuntime { + t.Helper() + wr, ok := s.index.worker("pod-a") + if !ok { + t.Fatal("pod-a has no index entry") + } + cr, ok := wr.class(v1alpha1.SandboxClassGvisor) + if !ok { + t.Fatal("pod-a advertises nothing for gvisor") + } + return cr + } + + t.Run("a failed sync keeps the entry of the file on disk", func(t *testing.T) { + s := newTestSyncer(t, resolvedPool("pool", "gvisor-release"), multiVersionConfig()) + if err := s.syncPod(ctx, pod); err != nil { + t.Fatalf("syncPod: %v", err) + } + before := advertised(t, s) + if before.defaultHash != hashOf("v2") || !slices.Equal(before.hashes, []string{hashOf("v2"), hashOf("v1")}) { + t.Fatalf("after a full sync pod-a advertises default %s, hashes %v; want v2 and [v2 v1]", before.defaultHash, before.hashes) + } + + // The default's probe now fails permanently: nothing is written. + s.mu.Lock() + s.cpuFeatures = map[string]string{} + s.mu.Unlock() + s.runCPUFeatures = func(context.Context, string) ([]string, error) { return []string{"not_a_known_feature"}, nil } + if err := s.syncPod(ctx, pod); !errors.Is(err, errPermanent) { + t.Fatalf("syncPod with the default's probe failing = %v, want errPermanent", err) + } + if after := advertised(t, s); after.defaultHash != before.defaultHash || !slices.Equal(after.hashes, before.hashes) { + t.Errorf("a sync that wrote nothing changed pod-a's entry: default %s, hashes %v", after.defaultHash, after.hashes) + } + }) + + t.Run("a partial file advertises only its runtimes", func(t *testing.T) { + s := newTestSyncer(t, resolvedPool("pool", "gvisor-release"), multiVersionConfig()) + s.assets = &fakeAssetFetcher{ + fn: func(_ context.Context, rec *sandboxAssetsRecord) (map[string]string, error) { + if sha := rec.Assets[gvisorAssetName].SHA256; sha != "sha-v2" { + return nil, fmt.Errorf("temporary download failure for %s", sha) + } + return map[string]string{gvisorAssetName: "sha-v2"}, nil + }, + } + if err := s.syncPod(ctx, pod); !errors.Is(err, errIncomplete) { + t.Fatalf("syncPod with a restorable version failing = %v, want errIncomplete", err) + } + if cr := advertised(t, s); cr.defaultHash != hashOf("v2") || !slices.Equal(cr.hashes, []string{hashOf("v2")}) { + t.Errorf("pod-a advertises default %s, hashes %v; want only the written v2", cr.defaultHash, cr.hashes) + } + if _, ok := s.index.lookup(hashOf("v1")); !ok { + t.Error("the runtime left out of the partial file does not resolve; it must, before the retry writes it") + } + }) +} diff --git a/cmd/atelet/workersupport_test.go b/cmd/atelet/workersupport_test.go index 6b9c4d5301..da90205146 100644 --- a/cmd/atelet/workersupport_test.go +++ b/cmd/atelet/workersupport_test.go @@ -121,7 +121,7 @@ func TestRegisterWorker(t *testing.T) { SchemaVersion: "v1", Attributes: []*ateletpb.AttributeEntry{ {Key: "architecture", Value: "amd64"}, - {Key: "gvisor_asset_hash", Value: hash}, + {Key: "sandbox_bundle_hash", Value: hash}, }, }, } @@ -133,7 +133,7 @@ func TestRegisterWorker(t *testing.T) { SchemaVersion: "v1", Attributes: []*ateapipb.AttributeEntry{ {Key: "architecture", Value: "amd64"}, - {Key: "gvisor_asset_hash", Value: hash}, + {Key: "sandbox_bundle_hash", Value: hash}, }, }, } diff --git a/cmd/atenet/internal/router/cmd.go b/cmd/atenet/internal/router/cmd.go index 5a26645f6f..bf52fe9144 100644 --- a/cmd/atenet/internal/router/cmd.go +++ b/cmd/atenet/internal/router/cmd.go @@ -122,6 +122,6 @@ func bindRouterFlags(fs *pflag.FlagSet, cfg *routerConfig) { // must propagate to the Service endpoints before the drain starts. fs.DurationVar(&cfg.DrainDelay, "drain-delay", 13*time.Second, "How long to keep serving after SIGTERM before starting the drain, covering readiness-probe detection and Service endpoint propagation") fs.DurationVar(&cfg.DrainTimeout, "drain-timeout", 0, "Deadline for the ext_proc drain on shutdown; streams still open past it (parked requests included) are forcefully cancelled. 0 (the default) derives --parked-request-budget + the drain route budget + margin so parked requests always finish normally. It does not follow --route-timeout. Explicit values must be >= --parked-request-budget") - fs.StringVar(&cfg.EnvoyAdminAddr, "envoy-admin-address", "localhost:9901", "Envoy admin interface the shutdown sequence drives to drain the sidecar (healthcheck/fail, drain_listeners, stats polling). Ignored with --atenet-dataplane=agentgateway") + fs.StringVar(&cfg.EnvoyAdminAddr, "envoy-admin-address", "localhost:9901", "Envoy admin interface the shutdown sequence drives to drain the sidecar (healthcheck/fail, drain_listeners, stats polling). A localhost host tries both 127.0.0.1 and ::1, whatever the host's resolver says. Ignored with --atenet-dataplane=agentgateway") fs.StringVar(&cfg.DrainCompleteFile, "drain-complete-file", defaultDrainCompleteFile, "Marker file created (on a pod-shared emptyDir) once the shutdown drain completes; the dataplane container's preStop hook polls for it so the proxy exits as soon as — and no sooner than — the drain is done. Removed at startup to defuse stale markers. Empty disables the handshake") } diff --git a/cmd/atenet/internal/router/dataplane.go b/cmd/atenet/internal/router/dataplane.go index 352e3eca35..793bdda115 100644 --- a/cmd/atenet/internal/router/dataplane.go +++ b/cmd/atenet/internal/router/dataplane.go @@ -40,8 +40,9 @@ type dataplaneHealthCheck struct { func (r atenetRouter) healthCheck() dataplaneHealthCheck { switch r { case atenetRouterEnvoy: - // localhost, not 127.0.0.1: the admin socket binds `::`, so the dial - // has to be able to fall through to the IPv6 loopback. + // Use `localhost` because we must try both IPv6 and IPv4. + // + // TODO: Shouldn't this work for the agentgateway case as well? return dataplaneHealthCheck{url: "http://localhost:9901/ready", expectedBody: "LIVE"} case atenetRouterAgentgateway: return dataplaneHealthCheck{url: "http://127.0.0.1:15021/healthz/ready", expectedBody: "ready"} diff --git a/cmd/atenet/internal/router/egress/credentials.go b/cmd/atenet/internal/router/egress/credentials.go index 31f292df8f..283819c7b6 100644 --- a/cmd/atenet/internal/router/egress/credentials.go +++ b/cmd/atenet/internal/router/egress/credentials.go @@ -19,6 +19,7 @@ import ( "fmt" "log/slog" "strings" + "time" corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" @@ -32,6 +33,10 @@ import ( "go.ate.dev/substrate/pkg/proto/credproviderpb" ) +// injectionTimeout caps resolving a request's credentials, under the ext_proc +// message_timeout so a slow provider is a retryable 503 instead of an Envoy 504 +const injectionTimeout = 4 * time.Second + // mapCredentialProviderError converts a FetchSecret failure into a // client-facing ext_proc denial, mirroring mapEgressIdentityError: a credential // the provider does not hold or will not release (NotFound, PermissionDenied) @@ -85,6 +90,8 @@ func (h *Handler) applyEffects(ctx context.Context, ref resources.ActorRef, dest return nil, nil } + ctx, cancel := context.WithTimeout(ctx, injectionTimeout) + defer cancel() setHeaders := make([]*corev3.HeaderValueOption, 0, len(injections)) for _, inj := range injections { if err := validateInjectHeader(inj.GetHeader()); err != nil { diff --git a/cmd/atenet/internal/router/egress/credentials_test.go b/cmd/atenet/internal/router/egress/credentials_test.go index e90d9aae93..32723686a8 100644 --- a/cmd/atenet/internal/router/egress/credentials_test.go +++ b/cmd/atenet/internal/router/egress/credentials_test.go @@ -19,6 +19,8 @@ import ( "context" "errors" "testing" + "testing/synctest" + "time" corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" @@ -254,6 +256,29 @@ func TestInjectionDenials(t *testing.T) { } } +// blockingProvider never answers, like a provider stuck dialing +type blockingProvider struct{} + +func (blockingProvider) FetchSecret(ctx context.Context, _ *credproviderpb.FetchSecretRequest, _ ...grpc.CallOption) (*credproviderpb.FetchSecretResponse, error) { + <-ctx.Done() + return nil, status.FromContextError(ctx.Err()).Err() +} + +// a hung provider must be a retryable 503 before Envoy's ext_proc +// message_timeout turns it into a 504 +func TestInjectionProviderHangs(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + h := injectionHandlerFor(credentialInjectionPolicySample("api.example.com"), blockingProvider{}, injectionProviderName) + start := time.Now() + _, err := h.HandleRequestHeaders(context.Background(), + withAuthorization(innerMetadata(extproc.EgressTLSMITMFilterChainName, "GET", "api.example.com", nil))) + wantStatus(t, err, envoy_type.StatusCode_ServiceUnavailable) + if took := time.Since(start); took >= 5*time.Second { + t.Errorf("denied after %v, want under the 5s ext_proc message_timeout", took) + } + }) +} + // actorJWTHeader replaces header with an actor JWT bound to api.example.com. func actorJWTHeader(header string) *ateapipb.CredentialHeader { return &ateapipb.CredentialHeader{ diff --git a/cmd/atenet/internal/router/egress/request.go b/cmd/atenet/internal/router/egress/request.go index e75ad8bd19..382285ccf5 100644 --- a/cmd/atenet/internal/router/egress/request.go +++ b/cmd/atenet/internal/router/egress/request.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "log/slog" + "net/http" "strings" extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" @@ -29,7 +30,10 @@ import ( "go.ate.dev/substrate/internal/resources" ) -const websocketDeniedBody = "WebSocket egress is not supported" +const ( + websocketDeniedBody = "WebSocket egress is not supported" + connectDeniedBody = "CONNECT egress is not supported" +) // handleRequest authorizes one request the gateway can read: cleartext HTTP, // or HTTPS the gateway terminated. It runs per request, because the @@ -45,6 +49,11 @@ func (h *Handler) handleRequest(ctx context.Context, md *extproc.RequestMetadata slog.WarnContext(ctx, "egress denied: WebSocket upgrades are not supported", slog.String("leg", leg), slog.String("host", md.Host)) return extproc.Result{}, extproc.NewReqError(envoy_type.StatusCode_Forbidden, websocketDeniedBody) } + // We don't support Actor making a CONNECT (inside the tunnel). + if strings.EqualFold(md.Method, http.MethodConnect) { + slog.WarnContext(ctx, "egress denied: CONNECT inside the egress tunnel is not supported", slog.String("leg", leg), slog.String("authority", md.Host)) + return extproc.Result{}, extproc.NewReqError(envoy_type.StatusCode_Forbidden, connectDeniedBody) + } ref, err := actorFromFilterState(md) if err != nil { diff --git a/cmd/atenet/internal/router/egress/request_test.go b/cmd/atenet/internal/router/egress/request_test.go index 954d8dd32f..18df8950b1 100644 --- a/cmd/atenet/internal/router/egress/request_test.go +++ b/cmd/atenet/internal/router/egress/request_test.go @@ -344,6 +344,27 @@ func TestRequestLegDeniesHTTP1WebSocketUpgrade(t *testing.T) { } } +func TestRequestLegDeniesConnect(t *testing.T) { + for name, policy := range map[string]*ateapipb.EgressPolicy{ + "allow-all policy": allowAllPolicy(), + "no policy": nil, + } { + for _, leg := range []string{extproc.EgressCleartextFilterChainName, extproc.EgressTLSMITMFilterChainName} { + for _, method := range []string{"CONNECT", "connect"} { + t.Run(name+"/"+leg+"/"+method, func(t *testing.T) { + h := policyHandler(policy) + md := innerMetadata(leg, method, "api.example.com:443", nil) + _, err := h.HandleRequestHeaders(context.Background(), md) + wantStatus(t, err, envoy_type.StatusCode_Forbidden) + if err.Error() != connectDeniedBody { + t.Errorf("denial body = %q, want %q", err, connectDeniedBody) + } + }) + } + } + } +} + func TestRequestLegAllowsOrdinaryHTTPSRequest(t *testing.T) { h := policyHandler(allowAllPolicy()) for _, tc := range []struct { diff --git a/cmd/atenet/internal/router/envoydrain.go b/cmd/atenet/internal/router/envoydrain.go index 22aa3e591b..9ce8646c33 100644 --- a/cmd/atenet/internal/router/envoydrain.go +++ b/cmd/atenet/internal/router/envoydrain.go @@ -47,7 +47,7 @@ type envoyDrainer struct { func newEnvoyDrainer(adminAddr string) *envoyDrainer { return &envoyDrainer{ adminAddr: adminAddr, - client: &http.Client{Timeout: 2 * time.Second}, + client: &http.Client{Transport: newLoopbackTransport(), Timeout: 2 * time.Second}, pollInterval: envoyDrainPollInterval, } } diff --git a/cmd/atenet/internal/router/health.go b/cmd/atenet/internal/router/health.go index 335917945b..7505ff8b96 100644 --- a/cmd/atenet/internal/router/health.go +++ b/cmd/atenet/internal/router/health.go @@ -77,7 +77,7 @@ func newRouterHealth(interval time.Duration, clientset kubernetes.Interface, api clientset: clientset, apiClient: apiClient, cfg: cfg, - dataplaneClient: &http.Client{Transport: otelhttp.NewTransport(http.DefaultTransport)}, + dataplaneClient: &http.Client{Transport: otelhttp.NewTransport(newLoopbackTransport())}, } } diff --git a/cmd/atenet/internal/router/loopback.go b/cmd/atenet/internal/router/loopback.go new file mode 100644 index 0000000000..f2b76e2caa --- /dev/null +++ b/cmd/atenet/internal/router/loopback.go @@ -0,0 +1,53 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package router + +import ( + "context" + "errors" + "net" + "net/http" +) + +// Loopback addresses to try when re-contacting Envoy. +var loopbackIPs = []string{"127.0.0.1", "::1"} + +// newLoopbackTransport returns a clone of http.DefaultTransport whose dials to +// "localhost" go through dialLoopback. +func newLoopbackTransport() *http.Transport { + t := http.DefaultTransport.(*http.Transport).Clone() + t.DialContext = dialLoopback(&net.Dialer{}) + return t +} + +// dialLoopback provides special handling to try both IPv6 and IPv4 when +// connecting to "localhost". +func dialLoopback(d *net.Dialer) func(ctx context.Context, network, addr string) (net.Conn, error) { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + host, port, err := net.SplitHostPort(addr) + if err != nil || host != "localhost" { + return d.DialContext(ctx, network, addr) + } + var errs []error + for _, ip := range loopbackIPs { + conn, err := d.DialContext(ctx, network, net.JoinHostPort(ip, port)) + if err == nil { + return conn, nil + } + errs = append(errs, err) + } + return nil, errors.Join(errs...) + } +} diff --git a/cmd/atenet/internal/router/loopback_test.go b/cmd/atenet/internal/router/loopback_test.go new file mode 100644 index 0000000000..35b276d010 --- /dev/null +++ b/cmd/atenet/internal/router/loopback_test.go @@ -0,0 +1,82 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package router + +import ( + "context" + "net" + "strconv" + "strings" + "testing" +) + +// TestDialLoopback checks that dialLoopback reaches a listener on either +// loopback through "localhost", and dials any other host as given. +func TestDialLoopback(t *testing.T) { + for _, tc := range []struct{ network, listen, dialHost string }{ + {"tcp4", "127.0.0.1:0", "localhost"}, + {"tcp6", "[::1]:0", "localhost"}, + {"tcp4", "127.0.0.1:0", "127.0.0.1"}, + {"tcp6", "[::1]:0", "::1"}, + } { + t.Run(tc.network+"/"+tc.dialHost, func(t *testing.T) { + ln, err := net.Listen(tc.network, tc.listen) + if err != nil { + t.Skipf("no %s loopback on this host: %v", tc.network, err) + } + defer ln.Close() + accepted := make(chan net.Addr, 1) + go func() { + c, err := ln.Accept() + if err != nil { + return + } + accepted <- c.LocalAddr() + c.Close() + }() + + port := strconv.Itoa(ln.Addr().(*net.TCPAddr).Port) + conn, err := dialLoopback(&net.Dialer{})(context.Background(), "tcp", net.JoinHostPort(tc.dialHost, port)) + if err != nil { + t.Fatalf("dial %s:%s: %v", tc.dialHost, port, err) + } + conn.Close() + if got := <-accepted; got.String() != ln.Addr().String() { + t.Errorf("connection accepted on %s, want %s", got, ln.Addr()) + } + }) + } +} + +// TestDialLoopbackNothingListening checks that a localhost dial with no +// listener on either loopback fails, naming both addresses it tried. +func TestDialLoopbackNothingListening(t *testing.T) { + ln, err := net.Listen("tcp4", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + port := strconv.Itoa(ln.Addr().(*net.TCPAddr).Port) + ln.Close() + + _, err = dialLoopback(&net.Dialer{})(context.Background(), "tcp", net.JoinHostPort("localhost", port)) + if err == nil { + t.Fatal("dial succeeded with nothing listening") + } + for _, ip := range loopbackIPs { + if want := net.JoinHostPort(ip, port); !strings.Contains(err.Error(), want) { + t.Errorf("dial error %q does not mention %s", err, want) + } + } +} diff --git a/cmd/ateworker-gvisor/actorpath.go b/cmd/ateworker-gvisor/actorpath.go index 32ed676d7a..09defa5b65 100644 --- a/cmd/ateworker-gvisor/actorpath.go +++ b/cmd/ateworker-gvisor/actorpath.go @@ -47,11 +47,6 @@ func pidFilePath(actorDirs *ateworkerpb.ActorDirs, containerName string) string return filepath.Join(pidFileDir(actorDirs), containerName+".pid") } -// resolvConfPath is the resolver bind source outside the actor's rootfs. -func resolvConfPath(actorDirs *ateworkerpb.ActorDirs) string { - return filepath.Join(actorDirs.GetRootDir(), "resolv.conf") -} - // resetRunscStateAndPidFileDirs empties both directories for a new activation. // runsc can leave mounts behind in its state directory (its null-netns), which // must be detached in this mount namespace before they can be removed. diff --git a/cmd/ateworker-gvisor/durable.go b/cmd/ateworker-gvisor/durable.go deleted file mode 100644 index 0fa0fe6c8e..0000000000 --- a/cmd/ateworker-gvisor/durable.go +++ /dev/null @@ -1,94 +0,0 @@ -//go:build linux - -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "errors" - "fmt" - "io/fs" - "os" - "path/filepath" - - "go.ate.dev/substrate/internal/proto/ateworkerpb" - "go.ate.dev/substrate/internal/resources" -) - -const fsCheckpointManifest = "fscheckpoint.pb" - -// durableFSCheckpointPaths returns the runsc filesystem checkpoint target -// specs (:) for all durable-dir volume mounts -// across containers. Every mount is emitted even when a volume is shared across -// multiple mounts: runsc creates one shared tmpfs under whichever mount is -// set up first and deduplicates by vfs.Filesystem when saving. -func durableFSCheckpointPaths(containers []*ateworkerpb.Container) []string { - var paths []string - for _, c := range containers { - for _, m := range c.GetDurableDirVolumeMounts() { - paths = append(paths, fmt.Sprintf("%s:%s", c.GetName(), m.GetMountPath())) - } - } - return paths -} - -// durableSnapshotFiles returns the subset of snapshotFiles that belong to the -// filesystem checkpoint in the fs/ subdirectory. -func durableSnapshotFiles(snapshotFiles []string) []string { - var out []string - for _, f := range snapshotFiles { - if filepath.Dir(f) == resources.FSCheckpointSubdir { - out = append(out, f) - } - } - return out -} - -// fsRestoreArgs returns `--fs-restore-image-path /fs` if a -// filesystem checkpoint manifest exists in /fs, or nil if no -// filesystem checkpoint is present in the snapshot. -func fsRestoreArgs(checkpointDir string) ([]string, error) { - fsDir := filepath.Join(checkpointDir, resources.FSCheckpointSubdir) - manifestPath := filepath.Join(fsDir, fsCheckpointManifest) - if _, err := os.Stat(manifestPath); err != nil { - if errors.Is(err, fs.ErrNotExist) { - return nil, nil - } - return nil, fmt.Errorf("while checking filesystem checkpoint manifest %q: %w", manifestPath, err) - } - return []string{"--fs-restore-image-path", fsDir}, nil -} - -// fsRestoreContainers returns the container names that own the master mount of -// at least one durable-dir volume (the first container to mount each volume), -// in order. Only those containers register an entry in runsc's fsRestore -// waitMap for `runsc wait -fsrestore `. -func fsRestoreContainers(containers []*ateworkerpb.Container) []string { - seen := make(map[string]bool) - var out []string - for _, c := range containers { - ownsMount := false - for _, m := range c.GetDurableDirVolumeMounts() { - if !seen[m.GetName()] { - seen[m.GetName()] = true - ownsMount = true - } - } - if ownsMount { - out = append(out, c.GetName()) - } - } - return out -} diff --git a/cmd/ateworker-gvisor/durable_test.go b/cmd/ateworker-gvisor/durable_test.go deleted file mode 100644 index 09caee38a4..0000000000 --- a/cmd/ateworker-gvisor/durable_test.go +++ /dev/null @@ -1,183 +0,0 @@ -//go:build linux - -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "os" - "path/filepath" - "slices" - "testing" - - "go.ate.dev/substrate/internal/proto/ateworkerpb" -) - -func TestDurableFSCheckpointPaths(t *testing.T) { - containers := []*ateworkerpb.Container{ - { - Name: "app", - DurableDirVolumeMounts: []*ateworkerpb.VolumeMount{ - {Name: "data", MountPath: "/var/data"}, - {Name: "cache", MountPath: "/var/cache"}, - }, - }, - { - Name: "sidecar", - DurableDirVolumeMounts: []*ateworkerpb.VolumeMount{ - {Name: "logs", MountPath: "/var/log/app"}, - }, - }, - } - - got := durableFSCheckpointPaths(containers) - want := []string{ - "app:/var/data", - "app:/var/cache", - "sidecar:/var/log/app", - } - if !slices.Equal(got, want) { - t.Errorf("durableFSCheckpointPaths() = %v, want %v", got, want) - } -} - -func TestDurableSnapshotFiles(t *testing.T) { - snapshotFiles := []string{ - "checkpoint.img", - "fs/fscheckpoint.pb", - "fs/multitar.img", - "fs/pages.img", - "fs/pages_meta.img", - "pages.img", - "pages_meta.img", - } - - got := durableSnapshotFiles(snapshotFiles) - want := []string{ - "fs/fscheckpoint.pb", - "fs/multitar.img", - "fs/pages.img", - "fs/pages_meta.img", - } - if !slices.Equal(got, want) { - t.Errorf("durableSnapshotFiles() = %v, want %v", got, want) - } -} - -func TestListSnapshotFilesWithFSSubdir(t *testing.T) { - checkpointDir := t.TempDir() - fsDir := filepath.Join(checkpointDir, "fs") - if err := os.MkdirAll(fsDir, 0o700); err != nil { - t.Fatal(err) - } - for _, rel := range []string{ - "checkpoint.img", - "pages.img", - "pages_meta.img", - "fs/fscheckpoint.pb", - "fs/multitar.img", - "fs/pages.img", - "fs/pages_meta.img", - } { - if err := os.WriteFile(filepath.Join(checkpointDir, rel), []byte("data"), 0o600); err != nil { - t.Fatal(err) - } - } - - got, err := listSnapshotFiles(checkpointDir) - if err != nil { - t.Fatalf("listSnapshotFiles() = %v", err) - } - want := []string{ - "checkpoint.img", - "fs/fscheckpoint.pb", - "fs/multitar.img", - "fs/pages.img", - "fs/pages_meta.img", - "pages.img", - "pages_meta.img", - } - if !slices.Equal(got, want) { - t.Errorf("listSnapshotFiles() = %v, want %v", got, want) - } -} - -func TestFSRestoreArgs(t *testing.T) { - t.Run("no fs checkpoint present", func(t *testing.T) { - checkpointDir := t.TempDir() - got, err := fsRestoreArgs(checkpointDir) - if err != nil { - t.Fatalf("fsRestoreArgs() = %v", err) - } - if len(got) != 0 { - t.Errorf("fsRestoreArgs() = %v, want empty", got) - } - }) - - t.Run("fs checkpoint present in fs subdir", func(t *testing.T) { - checkpointDir := t.TempDir() - fsDir := filepath.Join(checkpointDir, "fs") - if err := os.MkdirAll(fsDir, 0o700); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(fsDir, "fscheckpoint.pb"), []byte("manifest"), 0o600); err != nil { - t.Fatal(err) - } - - got, err := fsRestoreArgs(checkpointDir) - if err != nil { - t.Fatalf("fsRestoreArgs() = %v", err) - } - want := []string{ - "--fs-restore-image-path", fsDir, - } - if !slices.Equal(got, want) { - t.Errorf("fsRestoreArgs() = %v, want %v", got, want) - } - }) -} - -func TestFSRestoreContainers(t *testing.T) { - containers := []*ateworkerpb.Container{ - { - Name: "init", - }, - { - Name: "app", - DurableDirVolumeMounts: []*ateworkerpb.VolumeMount{ - {Name: "data", MountPath: "/var/data"}, - }, - }, - { - Name: "sidecar-shared-only", - DurableDirVolumeMounts: []*ateworkerpb.VolumeMount{ - {Name: "data", MountPath: "/mnt/data"}, - }, - }, - { - Name: "sidecar-with-own-vol", - DurableDirVolumeMounts: []*ateworkerpb.VolumeMount{ - {Name: "data", MountPath: "/mnt/data"}, - {Name: "logs", MountPath: "/var/log"}, - }, - }, - } - - got := fsRestoreContainers(containers) - want := []string{"app", "sidecar-with-own-vol"} - if !slices.Equal(got, want) { - t.Errorf("fsRestoreContainers() = %v, want %v", got, want) - } -} diff --git a/cmd/ateworker-gvisor/fidelity_test.go b/cmd/ateworker-gvisor/fidelity_test.go index 486a9caef5..9779e51cd2 100644 --- a/cmd/ateworker-gvisor/fidelity_test.go +++ b/cmd/ateworker-gvisor/fidelity_test.go @@ -27,9 +27,7 @@ import ( "go.ate.dev/substrate/internal/proto/ateworkerpb" ) -// TestValidateFidelity pins what this runtime serves. gVisor keeps rootfs -// changes inside its memory checkpoint, so ROOTFS is refused until it can -// capture them on their own; the error names the levels it does serve. +// TestValidateFidelity pins what this runtime serves. func TestValidateFidelity(t *testing.T) { for _, tc := range []struct { name string @@ -37,8 +35,8 @@ func TestValidateFidelity(t *testing.T) { wantErr bool }{ {"volumes", ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, false}, + {"rootfs", ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, false}, {"memory", ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, false}, - {"rootfs is not served", ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, true}, {"unspecified", ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED, true}, {"outside the enum", ateworkerpb.SnapshotFidelity(99), true}, } { @@ -52,37 +50,32 @@ func TestValidateFidelity(t *testing.T) { } }) } - err := validateFidelity(ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS) - for _, want := range []string{"SNAPSHOT_FIDELITY_VOLUMES", "SNAPSHOT_FIDELITY_MEMORY"} { + err := validateFidelity(ateworkerpb.SnapshotFidelity(99)) + for _, want := range []string{"SNAPSHOT_FIDELITY_VOLUMES", "SNAPSHOT_FIDELITY_ROOTFS", "SNAPSHOT_FIDELITY_MEMORY"} { if !strings.Contains(err.Error(), want) { - t.Errorf("rootfs rejection %q does not list %s as supported", err, want) + t.Errorf("invalid fidelity rejection %q does not list %s as supported", err, want) } } } -// TestRPCsRejectRootfsBeforeTouchingTheSandbox pins that a ROOTFS checkpoint -// or restore fails validation up front, before the actor lock or any file -// system work, so a misrouted request cannot leave the actor half-handled. -func TestRPCsRejectRootfsBeforeTouchingTheSandbox(t *testing.T) { +// TestRPCsRejectInvalidFidelityBeforeTouchingTheSandbox pins that an invalid +// fidelity checkpoint or restore fails validation up front, before the actor +// lock or any file system work, so a misrouted request cannot leave the actor +// half-handled. +func TestRPCsRejectInvalidFidelityBeforeTouchingTheSandbox(t *testing.T) { s := &AteWorkerService{} - dirs := &ateworkerpb.ActorDirs{ - RootDir: "/node/actors/actor-a", - OciBundleDir: "/node/actors/actor-a/bundle", - CheckpointDir: "/node/actors/actor-a/checkpoint-state", - RestoreDir: "/node/actors/actor-a/restore-state", - DurableDirVolumeMountsDir: "/node/actors/actor-a/durable-dirs", - SystemInfoVolumeRootsDir: "/node/actors/actor-a/system-info", - ExternalVolumeMountsDir: "/node/actors/actor-a/volumes", - SandboxAssetsDir: "/node/actors/actor-a/sandbox-assets", - } - rootfs := ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS + unspecified := ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED - _, err := s.CheckpointWorkload(context.Background(), &ateworkerpb.CheckpointWorkloadRequest{ActorUid: "actor-a", ActorDirs: dirs, Fidelity: rootfs}) + ckpt := validCheckpointRequest() + ckpt.Fidelity = unspecified + _, err := s.CheckpointWorkload(context.Background(), ckpt) if apierror.Code(err) != codes.InvalidArgument || !strings.Contains(err.Error(), "fidelity") { - t.Errorf("CheckpointWorkload(ROOTFS) = %v, want InvalidArgument naming the fidelity", err) + t.Errorf("CheckpointWorkload(UNSPECIFIED) = %v, want InvalidArgument naming the fidelity", err) } - _, err = s.RestoreWorkload(context.Background(), &ateworkerpb.RestoreWorkloadRequest{ActorUid: "actor-a", ActorDirs: dirs, Fidelity: rootfs}) + restore := validRestoreRequest() + restore.Fidelity = unspecified + _, err = s.RestoreWorkload(context.Background(), restore) if apierror.Code(err) != codes.InvalidArgument || !strings.Contains(err.Error(), "fidelity") { - t.Errorf("RestoreWorkload(ROOTFS) = %v, want InvalidArgument naming the fidelity", err) + t.Errorf("RestoreWorkload(UNSPECIFIED) = %v, want InvalidArgument naming the fidelity", err) } } diff --git a/cmd/ateworker-gvisor/fscheckpoint.go b/cmd/ateworker-gvisor/fscheckpoint.go new file mode 100644 index 0000000000..03891c583d --- /dev/null +++ b/cmd/ateworker-gvisor/fscheckpoint.go @@ -0,0 +1,134 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/resources" +) + +const fsCheckpointManifest = "fscheckpoint.pb" + +// rootfsFSCheckpointPaths returns the runsc filesystem checkpoint target specs +// (rootfs/=:/) for all workload containers' rootfs overlays, +// grouping them into the "rootfs/" subdirectory bundle. +func rootfsFSCheckpointPaths(containers []*ateworkerpb.Container) []string { + paths := make([]string, 0, len(containers)) + for _, c := range containers { + paths = append(paths, fmt.Sprintf("%s/=%s:/", resources.RootfsCheckpointSubdir, c.GetName())) + } + return paths +} + +// durableFSCheckpointPaths returns the runsc filesystem checkpoint target +// specs (fs/=:) for all durable-dir volume mounts +// across containers, grouping them into the "fs/" subdirectory bundle. Every +// mount is emitted even when a volume is shared across multiple mounts: runsc +// creates one shared tmpfs under whichever mount is set up first and +// deduplicates by vfs.Filesystem when saving. +func durableFSCheckpointPaths(containers []*ateworkerpb.Container) []string { + var paths []string + for _, c := range containers { + for _, m := range c.GetDurableDirVolumeMounts() { + paths = append(paths, fmt.Sprintf("%s/=%s:%s", resources.FSCheckpointSubdir, c.GetName(), m.GetMountPath())) + } + } + return paths +} + +// durableSnapshotFiles returns the subset of snapshotFiles that belong to the +// durable-dir filesystem checkpoint in the fs/ subdirectory. +func durableSnapshotFiles(snapshotFiles []string) []string { + var out []string + for _, f := range snapshotFiles { + if filepath.Dir(f) == resources.FSCheckpointSubdir { + out = append(out, f) + } + } + return out +} + +// fsRestoreArgs returns repeatable `--fs-restore-image-path` arguments for the +// filesystem checkpoint bundles in checkpointDir that apply at the requested +// fidelity: +// - ROOTFS and MEMORY require `/rootfs/fscheckpoint.pb`. +// - Every fidelity uses `/fs` when +// `/fs/fscheckpoint.pb` exists. +func fsRestoreArgs(checkpointDir string, fidelity ateworkerpb.SnapshotFidelity) ([]string, error) { + var subdirs []string + if fidelity == ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS || + fidelity == ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { + subdirs = append(subdirs, resources.RootfsCheckpointSubdir) + } + subdirs = append(subdirs, resources.FSCheckpointSubdir) + + var args []string + for _, sub := range subdirs { + dir := filepath.Join(checkpointDir, sub) + manifestPath := filepath.Join(dir, fsCheckpointManifest) + if _, err := os.Stat(manifestPath); err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("while checking filesystem checkpoint manifest %q: %w", manifestPath, err) + } + if sub == resources.RootfsCheckpointSubdir { + return nil, fmt.Errorf("%v snapshot has no rootfs filesystem checkpoint %q: %w", fidelity, manifestPath, err) + } + continue + } + args = append(args, "--fs-restore-image-path", dir) + } + return args, nil +} + +// fsRestoreContainers returns, in order, the containers that register an entry +// in runsc's fsRestore waitMap for `runsc wait -fsrestore `: those +// whose filesystems fsRestoreArgs restores at the requested fidelity. ROOTFS +// and MEMORY restore every container's rootfs overlay, so every container has +// an entry. VOLUMES restores only durable-dir volumes, so only a container +// that owns the master mount of at least one volume (the first container to +// mount it) has one. +func fsRestoreContainers(fidelity ateworkerpb.SnapshotFidelity, containers []*ateworkerpb.Container) []string { + if fidelity == ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS || + fidelity == ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { + out := make([]string, 0, len(containers)) + for _, c := range containers { + out = append(out, c.GetName()) + } + return out + } + seen := make(map[string]bool) + var out []string + for _, c := range containers { + ownsMount := false + for _, m := range c.GetDurableDirVolumeMounts() { + if !seen[m.GetName()] { + seen[m.GetName()] = true + ownsMount = true + } + } + if ownsMount { + out = append(out, c.GetName()) + } + } + return out +} diff --git a/cmd/ateworker-gvisor/fscheckpoint_test.go b/cmd/ateworker-gvisor/fscheckpoint_test.go new file mode 100644 index 0000000000..26c17e329d --- /dev/null +++ b/cmd/ateworker-gvisor/fscheckpoint_test.go @@ -0,0 +1,320 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "go.ate.dev/substrate/internal/proto/ateworkerpb" +) + +func TestRootfsFSCheckpointPaths(t *testing.T) { + containers := []*ateworkerpb.Container{ + {Name: "app"}, + {Name: "sidecar"}, + } + + got := rootfsFSCheckpointPaths(containers) + want := []string{ + "rootfs/=app:/", + "rootfs/=sidecar:/", + } + if !slices.Equal(got, want) { + t.Errorf("rootfsFSCheckpointPaths() = %v, want %v", got, want) + } +} + +func TestDurableFSCheckpointPaths(t *testing.T) { + containers := []*ateworkerpb.Container{ + { + Name: "app", + DurableDirVolumeMounts: []*ateworkerpb.VolumeMount{ + {Name: "data", MountPath: "/var/data"}, + {Name: "cache", MountPath: "/var/cache"}, + }, + }, + { + Name: "sidecar", + DurableDirVolumeMounts: []*ateworkerpb.VolumeMount{ + {Name: "logs", MountPath: "/var/log/app"}, + }, + }, + } + + got := durableFSCheckpointPaths(containers) + want := []string{ + "fs/=app:/var/data", + "fs/=app:/var/cache", + "fs/=sidecar:/var/log/app", + } + if !slices.Equal(got, want) { + t.Errorf("durableFSCheckpointPaths() = %v, want %v", got, want) + } +} + +func TestDurableSnapshotFiles(t *testing.T) { + snapshotFiles := []string{ + "checkpoint.img", + "fs/fscheckpoint.pb", + "fs/multitar.img", + "fs/pages.img", + "fs/pages_meta.img", + "pages.img", + "pages_meta.img", + "rootfs/fscheckpoint.pb", + "rootfs/multitar.img", + "rootfs/pages.img", + "rootfs/pages_meta.img", + } + + got := durableSnapshotFiles(snapshotFiles) + want := []string{ + "fs/fscheckpoint.pb", + "fs/multitar.img", + "fs/pages.img", + "fs/pages_meta.img", + } + if !slices.Equal(got, want) { + t.Errorf("durableSnapshotFiles() = %v, want %v", got, want) + } +} + +func TestListSnapshotFilesWithFSSubdir(t *testing.T) { + checkpointDir := t.TempDir() + for _, sub := range []string{"fs", "rootfs"} { + if err := os.MkdirAll(filepath.Join(checkpointDir, sub), 0o700); err != nil { + t.Fatal(err) + } + } + for _, rel := range []string{ + "checkpoint.img", + "pages.img", + "pages_meta.img", + "rootfs/fscheckpoint.pb", + "rootfs/multitar.img", + "rootfs/pages.img", + "rootfs/pages_meta.img", + "fs/fscheckpoint.pb", + "fs/multitar.img", + "fs/pages.img", + "fs/pages_meta.img", + } { + if err := os.WriteFile(filepath.Join(checkpointDir, rel), []byte("data"), 0o600); err != nil { + t.Fatal(err) + } + } + + got, err := listSnapshotFiles(checkpointDir) + if err != nil { + t.Fatalf("listSnapshotFiles() = %v", err) + } + want := []string{ + "checkpoint.img", + "fs/fscheckpoint.pb", + "fs/multitar.img", + "fs/pages.img", + "fs/pages_meta.img", + "pages.img", + "pages_meta.img", + "rootfs/fscheckpoint.pb", + "rootfs/multitar.img", + "rootfs/pages.img", + "rootfs/pages_meta.img", + } + if !slices.Equal(got, want) { + t.Errorf("listSnapshotFiles() = %v, want %v", got, want) + } +} + +func TestFSRestoreArgs(t *testing.T) { + t.Run("no filesystem checkpoint present", func(t *testing.T) { + checkpointDir := t.TempDir() + got, err := fsRestoreArgs(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES) + if err != nil { + t.Fatalf("fsRestoreArgs(VOLUMES) = %v", err) + } + if len(got) != 0 { + t.Errorf("fsRestoreArgs(VOLUMES) = %v, want empty", got) + } + for _, fidelity := range []ateworkerpb.SnapshotFidelity{ + ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, + ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + } { + if got, err := fsRestoreArgs(checkpointDir, fidelity); err == nil { + t.Errorf("fsRestoreArgs(%v) = (%v, nil), want error when rootfs/ is missing", fidelity, got) + } + } + }) + + t.Run("only rootfs checkpoint present", func(t *testing.T) { + checkpointDir := t.TempDir() + rootfsDir := filepath.Join(checkpointDir, "rootfs") + if err := os.MkdirAll(rootfsDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(rootfsDir, "fscheckpoint.pb"), []byte("manifest"), 0o600); err != nil { + t.Fatal(err) + } + + for _, fidelity := range []ateworkerpb.SnapshotFidelity{ + ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, + ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + } { + got, err := fsRestoreArgs(checkpointDir, fidelity) + if err != nil { + t.Fatalf("fsRestoreArgs(%v) = %v", fidelity, err) + } + want := []string{ + "--fs-restore-image-path", rootfsDir, + } + if !slices.Equal(got, want) { + t.Errorf("fsRestoreArgs(%v) = %v, want %v", fidelity, got, want) + } + } + + gotVolumes, err := fsRestoreArgs(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES) + if err != nil { + t.Fatalf("fsRestoreArgs(VOLUMES) = %v", err) + } + if len(gotVolumes) != 0 { + t.Errorf("fsRestoreArgs(VOLUMES) = %v, want empty", gotVolumes) + } + }) + + t.Run("ROOTFS and MEMORY with only fs checkpoint present fail", func(t *testing.T) { + checkpointDir := t.TempDir() + fsDir := filepath.Join(checkpointDir, "fs") + if err := os.MkdirAll(fsDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(fsDir, "fscheckpoint.pb"), []byte("manifest"), 0o600); err != nil { + t.Fatal(err) + } + + for _, fidelity := range []ateworkerpb.SnapshotFidelity{ + ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, + ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + } { + got, err := fsRestoreArgs(checkpointDir, fidelity) + if err == nil { + t.Errorf("fsRestoreArgs(%v) = (%v, nil), want error when rootfs/ is missing", fidelity, got) + } else if !strings.Contains(err.Error(), "rootfs") { + t.Errorf("fsRestoreArgs(%v) error = %q, want it to name the rootfs checkpoint", fidelity, err) + } + } + + gotVolumes, err := fsRestoreArgs(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES) + if err != nil { + t.Fatalf("fsRestoreArgs(VOLUMES) = %v", err) + } + if want := []string{"--fs-restore-image-path", fsDir}; !slices.Equal(gotVolumes, want) { + t.Errorf("fsRestoreArgs(VOLUMES) = %v, want %v", gotVolumes, want) + } + }) + + t.Run("both rootfs and fs subdir checkpoints present", func(t *testing.T) { + checkpointDir := t.TempDir() + rootfsDir := filepath.Join(checkpointDir, "rootfs") + if err := os.MkdirAll(rootfsDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(rootfsDir, "fscheckpoint.pb"), []byte("manifest"), 0o600); err != nil { + t.Fatal(err) + } + fsDir := filepath.Join(checkpointDir, "fs") + if err := os.MkdirAll(fsDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(fsDir, "fscheckpoint.pb"), []byte("manifest"), 0o600); err != nil { + t.Fatal(err) + } + + for _, fidelity := range []ateworkerpb.SnapshotFidelity{ + ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, + ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + } { + got, err := fsRestoreArgs(checkpointDir, fidelity) + if err != nil { + t.Fatalf("fsRestoreArgs(%v) = %v", fidelity, err) + } + want := []string{ + "--fs-restore-image-path", rootfsDir, + "--fs-restore-image-path", fsDir, + } + if !slices.Equal(got, want) { + t.Errorf("fsRestoreArgs(%v) = %v, want %v", fidelity, got, want) + } + } + + gotVolumes, err := fsRestoreArgs(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES) + if err != nil { + t.Fatalf("fsRestoreArgs(VOLUMES) = %v", err) + } + wantVolumes := []string{ + "--fs-restore-image-path", fsDir, + } + if !slices.Equal(gotVolumes, wantVolumes) { + t.Errorf("fsRestoreArgs(VOLUMES) = %v, want %v", gotVolumes, wantVolumes) + } + }) +} + +func TestFSRestoreContainers(t *testing.T) { + containers := []*ateworkerpb.Container{ + { + Name: "init", + }, + { + Name: "app", + DurableDirVolumeMounts: []*ateworkerpb.VolumeMount{ + {Name: "data", MountPath: "/var/data"}, + }, + }, + { + Name: "sidecar-shared-only", + DurableDirVolumeMounts: []*ateworkerpb.VolumeMount{ + {Name: "data", MountPath: "/mnt/data"}, + }, + }, + { + Name: "sidecar-with-own-vol", + DurableDirVolumeMounts: []*ateworkerpb.VolumeMount{ + {Name: "data", MountPath: "/mnt/data"}, + {Name: "logs", MountPath: "/var/log"}, + }, + }, + } + + for _, tc := range []struct { + fidelity ateworkerpb.SnapshotFidelity + want []string + }{ + // Only fs/ is restored, so only durable-dir mount owners wait. + {ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, []string{"app", "sidecar-with-own-vol"}}, + // rootfs/ is restored for every container, so every container waits. + {ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, []string{"init", "app", "sidecar-shared-only", "sidecar-with-own-vol"}}, + {ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, []string{"init", "app", "sidecar-shared-only", "sidecar-with-own-vol"}}, + } { + if got := fsRestoreContainers(tc.fidelity, containers); !slices.Equal(got, tc.want) { + t.Errorf("fsRestoreContainers(%v) = %v, want %v", tc.fidelity, got, tc.want) + } + } +} diff --git a/cmd/ateworker-gvisor/hosted.go b/cmd/ateworker-gvisor/hosted.go index 0761de4ed8..c603bd07f8 100644 --- a/cmd/ateworker-gvisor/hosted.go +++ b/cmd/ateworker-gvisor/hosted.go @@ -37,8 +37,6 @@ type hostedActor struct { // Attribution is available during boot. attribution resources.ActorAttribution network *actornet.SandboxSession - // resolvConf is the actor's resolver bind source, removed with its network. - resolvConf string // Set once the containers exist. session *workloadSession // usage is this activation's epoch, CPU baseline, and latest sample. @@ -73,9 +71,6 @@ func (s *AteWorkerService) admitActor(attribution resources.ActorAttribution) (* // hostActor sets up the actor's network, replacing any stale one. func (s *AteWorkerService) hostActor(ctx context.Context, attribution resources.ActorAttribution, actorDirs *ateworkerpb.ActorDirs) (*hostedActor, error) { uid := attribution.UID - if uid == "" { - return nil, fmt.Errorf("actor UID is required") - } hosted, stale, err := s.admitActor(attribution) if err != nil { @@ -106,17 +101,8 @@ func (s *AteWorkerService) hostActor(ctx context.Context, attribution resources. return nil, fmt.Errorf("while setting up the sandbox network: %w", err) } - // Use the actor's gateway as its DNS resolver. - resolvConf, err := actorResolvConf(actorDirs) - if err != nil { - _ = session.Close(ctx) - evict() - return nil, err - } - s.actorsMu.Lock() hosted.network = session - hosted.resolvConf = resolvConf s.actorsMu.Unlock() return hosted, nil } @@ -141,7 +127,6 @@ func (s *AteWorkerService) unhostActor(ctx context.Context, actorUID string) err s.actorsMu.Unlock() }() - removeActorResolvConf(ctx, hosted.resolvConf) if hosted.network == nil { return nil } diff --git a/cmd/ateworker-gvisor/kodata/internal_flags.json b/cmd/ateworker-gvisor/kodata/internal_flags.json index 89c3244df2..ac63d5178c 100644 --- a/cmd/ateworker-gvisor/kodata/internal_flags.json +++ b/cmd/ateworker-gvisor/kodata/internal_flags.json @@ -2,7 +2,9 @@ "globalFlags": [ "-log-format", "json", - "--alsologtostderr" + "--alsologtostderr", + "--allow-suid", + "--host-settings=ignore" ], "commands": { "create": { diff --git a/cmd/ateworker-gvisor/main.go b/cmd/ateworker-gvisor/main.go index bd57d47d18..ce61ef3269 100644 --- a/cmd/ateworker-gvisor/main.go +++ b/cmd/ateworker-gvisor/main.go @@ -45,6 +45,7 @@ import ( "go.ate.dev/substrate/internal/ateworker/pid1" "go.ate.dev/substrate/internal/ateworker/register" "go.ate.dev/substrate/internal/ateworker/tunnel" + "go.ate.dev/substrate/internal/ateworkervalidation" "go.ate.dev/substrate/internal/imagecache" "go.ate.dev/substrate/internal/nodepath" "go.ate.dev/substrate/internal/ocispec" @@ -558,50 +559,36 @@ func containerNames(containers []*ateworkerpb.Container) []string { return names } -// validateActorDirs rejects a request whose actor directories are unusable. // validateRunscPath ensures we only execute runsc from the actor's own // sandbox assets dir, where atelet staged it. The dir is derived from the // actor UID here and the request's sandbox_assets_dir must match it, so a // request cannot name a root of its own choosing. actorDirs must have passed -// validateActorDirs. +// the generated validator. func validateRunscPath(actorUID string, actorDirs *ateworkerpb.ActorDirs, p string) error { root := actorDirs.GetSandboxAssetsDir() errs := resources.ValidateSandboxAssetsDir(actorUID, root, field.NewPath("actor_dirs", "sandbox_assets_dir")) if len(errs) == 0 { errs = resources.ValidateRuntimeAssetPath(root, p, field.NewPath("runsc_path")) } - if len(errs) > 0 { - return resources.ToAPIError(errs) - } - return nil + return resources.ToAPIError(errs) } -func validateActorDirs(actorDirs *ateworkerpb.ActorDirs) error { - if errs := resources.ValidateActorDirs(actorDirs, field.NewPath("actor_dirs")); len(errs) > 0 { - return apierror.InvalidArgument("%v", errs.ToAggregate()) - } - return nil -} - -// supportedFidelities are the snapshot fidelities this runtime serves. gVisor -// keeps rootfs changes inside its memory checkpoint, so it cannot capture -// ROOTFS without MEMORY. +// supportedFidelities are the snapshot fidelities this runtime serves. var supportedFidelities = []ateworkerpb.SnapshotFidelity{ ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, + ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, } // validateFidelity rejects a checkpoint or restore request whose fidelity // this runtime cannot serve. func validateFidelity(fidelity ateworkerpb.SnapshotFidelity) error { - if errs := resources.ValidateSnapshotFidelity(fidelity, supportedFidelities, field.NewPath("fidelity")); len(errs) > 0 { - return apierror.InvalidArgument("%v", errs.ToAggregate()) - } - return nil + errs := resources.ValidateSnapshotFidelity(fidelity, supportedFidelities, field.NewPath("fidelity")) + return resources.ToAPIError(errs) } func (s *AteWorkerService) RunWorkload(ctx context.Context, req *ateworkerpb.RunWorkloadRequest) (resp *ateworkerpb.RunWorkloadResponse, retErr error) { - if err := validateActorDirs(req.GetActorDirs()); err != nil { + if err := ateworkervalidation.ValidateRunWorkloadRequest(ctx, req); err != nil { return nil, err } if err := validateRunscPath(req.GetActorUid(), req.GetActorDirs(), req.GetRunscPath()); err != nil { @@ -704,7 +691,7 @@ func (s *AteWorkerService) RunWorkload(ctx context.Context, req *ateworkerpb.Run return nil, fmt.Errorf("while starting json log pipe for %q: %w", ac.GetName(), err) } defer pw.Close() - if err := imagecache.SetupBundleRootfs(ociBundlePath(req.GetActorDirs(), ac.GetName())); err != nil { + if err := setupAppRootfs(req.GetActorDirs(), ac.GetName()); err != nil { return nil, fmt.Errorf("while composing %q rootfs: %w", ac.GetName(), err) } containersToDelete = append(containersToDelete, ac.GetName()) @@ -734,11 +721,11 @@ func (s *AteWorkerService) RunWorkload(ctx context.Context, req *ateworkerpb.Run // Allow checkpointing even if the pod is shutting down. This will allow actors // (or the harness) to suspend on shutdown. func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworkerpb.CheckpointWorkloadRequest) (_ *ateworkerpb.CheckpointWorkloadResponse, err error) { - // Taken before the lock, so prep covers the wait for it. - tStart := time.Now() - if err := validateActorDirs(req.GetActorDirs()); err != nil { + if err := ateworkervalidation.ValidateCheckpointWorkloadRequest(ctx, req); err != nil { return nil, err } + // Taken before the lock, so prep covers the wait for it. + tStart := time.Now() if err := validateFidelity(req.GetFidelity()); err != nil { return nil, err } @@ -820,6 +807,7 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke return err } + fsPaths := slices.Concat(rootfsFSCheckpointPaths(containers), durablePaths) var checkpointFn func(context.Context) error switch fidelity { case ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: @@ -828,15 +816,25 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke } checkpointFn = func(ctx context.Context) error { err := rcmd.cmdFsCheckpoint(ctx, ocispec.PauseContainer, checkpointPath, durablePaths) - timing.durableDir = lap(&tLast) + timing.fsCheckpoint = lap(&tLast) if err != nil { return fmt.Errorf("while checkpointing durable-dir volumes: %w", err) } return nil } + case ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS: + checkpointFn = func(ctx context.Context) error { + err := rcmd.cmdFsCheckpoint(ctx, ocispec.PauseContainer, checkpointPath, fsPaths) + timing.fsCheckpoint = lap(&tLast) + if err != nil { + return fmt.Errorf("while checkpointing container filesystems: %w", err) + } + return nil + } case ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: - // Checkpoint pause container (root of the sandbox), splitting durable-dir - // volumes into /fs when present. + // Checkpoint pause container (root of the sandbox), splitting container + // rootfs overlays into the rootfs bundle and durable-dir volumes into + // /fs when present. checkpointFn = func(ctx context.Context) error { sandbox, sbErr := openSandboxProcess(req.GetActorDirs()) if sbErr != nil { @@ -844,7 +842,7 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke } else { defer sandbox.Close() } - err := rcmd.cmdCheckpoint(ctx, ocispec.PauseContainer, checkpointPath, durablePaths) + err := rcmd.cmdCheckpoint(ctx, ocispec.PauseContainer, checkpointPath, fsPaths) if err != nil { timing.checkpoint = lap(&tLast) return fmt.Errorf("while checkpointing pause: %w", err) @@ -866,8 +864,8 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke } // Report exactly the files runsc wrote so atelet ships precisely this set - // (checkpoint.img plus any pages images and fs/ durable-dir files), rather - // than a hardcoded list. + // (checkpoint.img and its page files, plus the fs/ and rootfs/ bundle files), + // rather than a hardcoded list. snapshotFiles, err := listSnapshotFiles(checkpointPath) if err != nil { return nil, fmt.Errorf("while listing checkpoint files: %w", err) @@ -880,7 +878,8 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke } // listSnapshotFiles returns the (relative) names of regular files directly under -// dir and under dir/fs, which atelet ships to object storage as the snapshot. +// dir, dir/fs, and dir/rootfs, which atelet ships to object storage as the +// snapshot. func listSnapshotFiles(dir string) ([]string, error) { entries, err := os.ReadDir(dir) if err != nil { @@ -892,13 +891,15 @@ func listSnapshotFiles(dir string) ([]string, error) { files = append(files, e.Name()) } } - fsEntries, err := os.ReadDir(filepath.Join(dir, resources.FSCheckpointSubdir)) - if err != nil && !errors.Is(err, os.ErrNotExist) { - return nil, err - } - for _, e := range fsEntries { - if e.Type().IsRegular() { - files = append(files, filepath.Join(resources.FSCheckpointSubdir, e.Name())) + for _, sub := range []string{resources.FSCheckpointSubdir, resources.RootfsCheckpointSubdir} { + subEntries, err := os.ReadDir(filepath.Join(dir, sub)) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + for _, e := range subEntries { + if e.Type().IsRegular() { + files = append(files, filepath.Join(sub, e.Name())) + } } } sort.Strings(files) @@ -963,11 +964,11 @@ func isContainerAlreadyGone(ctx context.Context, rcmd containerRuntime, name str } func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb.RestoreWorkloadRequest) (resp *ateworkerpb.RestoreWorkloadResponse, retErr error) { - // Taken before the lock, so prep covers the wait for it. - tStart := time.Now() - if err := validateActorDirs(req.GetActorDirs()); err != nil { + if err := ateworkervalidation.ValidateRestoreWorkloadRequest(ctx, req); err != nil { return nil, err } + // Taken before the lock, so prep covers the wait for it. + tStart := time.Now() if err := validateFidelity(req.GetFidelity()); err != nil { return nil, err } @@ -1059,7 +1060,7 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb } }() checkpointDir := req.GetActorDirs().GetRestoreDir() - restoreArgs, err := fsRestoreArgs(checkpointDir) + restoreArgs, err := fsRestoreArgs(checkpointDir, fidelity) if err != nil { return nil, err } @@ -1080,8 +1081,8 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb } defer pw.Close() switch fidelity { - case ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: - // Create and start pause container (cold boot with durable-dir volumes restored from fs/) + case ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS: + // Create and start pause container (cold boot with filesystem checkpoint bundles restored) containersToDelete = append(containersToDelete, ocispec.PauseContainer) err := rcmd.cmdCreate(ctx, pw, ocispec.PauseContainer, restoreArgs) timing.pauseCreate = lap(&tLast) @@ -1094,7 +1095,7 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb return nil, fmt.Errorf("while starting pause container: %w", err) } case ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: - // Create and restore pause container (restores both process/rootfs and fs/ durable-dir volumes) + // Create and restore pause container (restores process state and filesystem checkpoint bundles) containersToDelete = append(containersToDelete, ocispec.PauseContainer) err := rcmd.cmdCreate(ctx, pw, ocispec.PauseContainer, restoreArgs) timing.pauseCreate = lap(&tLast) @@ -1118,7 +1119,7 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb return nil, fmt.Errorf("while starting json log pipe for %q: %w", ac.GetName(), err) } defer pw.Close() - err = imagecache.SetupBundleRootfs(ociBundlePath(req.GetActorDirs(), ac.GetName())) + err = setupAppRootfs(req.GetActorDirs(), ac.GetName()) timing.appRootfs += lap(&tLast) if err != nil { return nil, fmt.Errorf("while composing %q rootfs: %w", ac.GetName(), err) @@ -1130,7 +1131,7 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb return nil, fmt.Errorf("while creating %q application container: %w", ac.GetName(), err) } switch fidelity { - case ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: + case ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS: err = rcmd.cmdStart(ctx, pw, ac.GetName()) timing.appRestore += lap(&tLast) if err != nil { @@ -1163,7 +1164,7 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb } } if len(restoreArgs) > 0 { - for _, ctr := range fsRestoreContainers(containers) { + for _, ctr := range fsRestoreContainers(fidelity, containers) { if err := rcmd.cmdWaitFSRestore(waitCtx, ctr); err != nil { return err } @@ -1180,7 +1181,7 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb } func (s *AteWorkerService) TerminateWorkload(ctx context.Context, req *ateworkerpb.TerminateWorkloadRequest) (*ateworkerpb.TerminateWorkloadResponse, error) { - if err := validateActorDirs(req.GetActorDirs()); err != nil { + if err := ateworkervalidation.ValidateTerminateWorkloadRequest(ctx, req); err != nil { return nil, err } if err := validateRunscPath(req.GetActorUid(), req.GetActorDirs(), req.GetRunscPath()); err != nil { diff --git a/cmd/ateworker-gvisor/main_test.go b/cmd/ateworker-gvisor/main_test.go index b2f5fcc539..84b4c5d2ac 100644 --- a/cmd/ateworker-gvisor/main_test.go +++ b/cmd/ateworker-gvisor/main_test.go @@ -39,8 +39,9 @@ import ( "go.ate.dev/substrate/internal/resources" ) -// Every RPC rejects a request without ActorDirs before touching any state. -func TestRPCsRejectMissingActorDirs(t *testing.T) { +// Every RPC with a request body rejects an empty one before touching any +// state: the generated validators run first in each handler. +func TestRPCsRejectInvalidRequests(t *testing.T) { s := &AteWorkerService{} ctx := context.Background() for name, call := range map[string]func() error{ @@ -60,6 +61,10 @@ func TestRPCsRejectMissingActorDirs(t *testing.T) { _, err := s.TerminateWorkload(ctx, &ateworkerpb.TerminateWorkloadRequest{}) return err }, + "GetWorkloadStats": func() error { + _, err := s.GetWorkloadStats(ctx, &ateworkerpb.GetWorkloadStatsRequest{}) + return err + }, } { if got := apierror.Code(call()); got != codes.InvalidArgument { t.Errorf("%s() code = %v, want %v", name, got, codes.InvalidArgument) @@ -70,73 +75,93 @@ func TestRPCsRejectMissingActorDirs(t *testing.T) { func TestRPCsRejectUntrustedRunscPath(t *testing.T) { s := &AteWorkerService{} ctx := context.Background() - dirs := &ateworkerpb.ActorDirs{ - RootDir: "/node/actors/actor-a", - OciBundleDir: "/node/actors/actor-a/bundle", - CheckpointDir: "/node/actors/actor-a/checkpoint-state", - RestoreDir: "/node/actors/actor-a/restore", - DurableDirVolumeMountsDir: "/node/actors/actor-a/durable-dirs", - SystemInfoVolumeRootsDir: "/node/actors/actor-a/system-info", - ExternalVolumeMountsDir: "/node/actors/actor-a/volumes", - SandboxAssetsDir: "/node/actors/actor-a/sandbox-assets", - } for name, call := range map[string]func() error{ "RunWorkload": func() error { - _, err := s.RunWorkload(ctx, &ateworkerpb.RunWorkloadRequest{ActorDirs: dirs, RunscPath: "/bin/sh"}) + req := validRunRequest() + req.RunscPath = "/bin/sh" + _, err := s.RunWorkload(ctx, req) return err }, "RestoreWorkload": func() error { - _, err := s.RestoreWorkload(ctx, &ateworkerpb.RestoreWorkloadRequest{ActorDirs: dirs, RunscPath: "/bin/sh"}) + req := validRestoreRequest() + req.RunscPath = "/bin/sh" + _, err := s.RestoreWorkload(ctx, req) return err }, "CheckpointWorkload": func() error { - _, err := s.CheckpointWorkload(ctx, &ateworkerpb.CheckpointWorkloadRequest{ActorDirs: dirs, RunscPath: "/bin/sh"}) + req := validCheckpointRequest() + req.RunscPath = "/bin/sh" + _, err := s.CheckpointWorkload(ctx, req) return err }, "TerminateWorkload": func() error { - _, err := s.TerminateWorkload(ctx, &ateworkerpb.TerminateWorkloadRequest{ActorDirs: dirs, RunscPath: "/bin/sh"}) + req := validTerminateRequest() + req.RunscPath = "/bin/sh" + _, err := s.TerminateWorkload(ctx, req) return err }, } { - if got := apierror.Code(call()); got != codes.InvalidArgument { - t.Errorf("%s() code = %v, want %v", name, got, codes.InvalidArgument) + err := call() + if apierror.Code(err) != codes.InvalidArgument || !strings.Contains(err.Error(), "runsc_path") { + t.Errorf("%s() = %v, want InvalidArgument naming runsc_path", name, err) } } } func TestRestoreWorkloadRejectsUnsupportedFidelity(t *testing.T) { - // runsc must sit in the actor's own sandbox assets dir. - sandboxAssetsDir := t.TempDir() - runscPath := filepath.Join(sandboxAssetsDir, "runsc") - if err := os.WriteFile(runscPath, nil, 0o755); err != nil { - t.Fatal(err) + // A zero service cannot acquire locks, configure networking, or open log pipes. + s := &AteWorkerService{} + for _, fidelity := range []ateworkerpb.SnapshotFidelity{0, 99} { + t.Run(fidelity.String(), func(t *testing.T) { + // Valid but for the fidelity, so the rejection is the fidelity's; + // the fidelity check runs before the runsc path is looked at. + req := validRestoreRequest() + req.Fidelity = fidelity + resp, err := s.RestoreWorkload(context.Background(), req) + if resp != nil || apierror.Code(err) != codes.InvalidArgument || !strings.Contains(err.Error(), "fidelity") { + t.Errorf("RestoreWorkload() = (%v, %v), want InvalidArgument naming the fidelity", resp, err) + } + }) } - dirs := &ateworkerpb.ActorDirs{ +} + +// The requests below describe testActor and pass the generated validators, so +// a test aimed at one of the hand checks that run after them (the runsc path, +// the fidelity) sees that check's error rather than an unrelated rejection. + +func testDirs() *ateworkerpb.ActorDirs { + return &ateworkerpb.ActorDirs{ RootDir: "/node/actors/actor-a", OciBundleDir: "/node/actors/actor-a/bundle", CheckpointDir: "/node/actors/actor-a/checkpoint-state", - RestoreDir: "/node/actors/actor-a/restore", + RestoreDir: "/node/actors/actor-a/restore-state", DurableDirVolumeMountsDir: "/node/actors/actor-a/durable-dirs", SystemInfoVolumeRootsDir: "/node/actors/actor-a/system-info", ExternalVolumeMountsDir: "/node/actors/actor-a/volumes", - SandboxAssetsDir: sandboxAssetsDir, - } - // A zero service cannot acquire locks, configure networking, or open log pipes. - s := &AteWorkerService{} - for _, fidelity := range []ateworkerpb.SnapshotFidelity{0, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, 99} { - t.Run(fidelity.String(), func(t *testing.T) { - resp, err := s.RestoreWorkload(context.Background(), &ateworkerpb.RestoreWorkloadRequest{ - ActorDirs: dirs, - RunscPath: runscPath, - Fidelity: fidelity, - }) - if resp != nil || err == nil || apierror.Code(err) != codes.InvalidArgument { - t.Errorf("RestoreWorkload() = (%v, %v), want InvalidArgument for unsupported fidelity", resp, err) - } - }) + SandboxAssetsDir: nodepath.ActorSandboxAssetsDir(testActor.UID), } } +func testSpec() *ateworkerpb.WorkloadSpec { + return &ateworkerpb.WorkloadSpec{Containers: []*ateworkerpb.Container{{Name: "app"}}} +} + +func validRunRequest() *ateworkerpb.RunWorkloadRequest { + return &ateworkerpb.RunWorkloadRequest{Atespace: testActor.Ref.Atespace, ActorName: testActor.Ref.Name, ActorUid: testActor.UID, Spec: testSpec(), ActorDirs: testDirs()} +} + +func validCheckpointRequest() *ateworkerpb.CheckpointWorkloadRequest { + return &ateworkerpb.CheckpointWorkloadRequest{Atespace: testActor.Ref.Atespace, ActorName: testActor.Ref.Name, ActorUid: testActor.UID, Spec: testSpec(), ActorDirs: testDirs(), Fidelity: ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY} +} + +func validRestoreRequest() *ateworkerpb.RestoreWorkloadRequest { + return &ateworkerpb.RestoreWorkloadRequest{Atespace: testActor.Ref.Atespace, ActorName: testActor.Ref.Name, ActorUid: testActor.UID, Spec: testSpec(), ActorDirs: testDirs(), Fidelity: ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY} +} + +func validTerminateRequest() *ateworkerpb.TerminateWorkloadRequest { + return &ateworkerpb.TerminateWorkloadRequest{Atespace: testActor.Ref.Atespace, ActorName: testActor.Ref.Name, ActorUid: testActor.UID, Spec: testSpec(), ActorDirs: testDirs()} +} + // TestPauseContainerLogAttribution covers the actor and _pause labels on runsc // CLI output, JSON field preservation, and rejection of forged attribution. func TestPauseContainerLogAttribution(t *testing.T) { diff --git a/cmd/ateworker-gvisor/phaselog.go b/cmd/ateworker-gvisor/phaselog.go index 7d390bc1c3..78ab0f2740 100644 --- a/cmd/ateworker-gvisor/phaselog.go +++ b/cmd/ateworker-gvisor/phaselog.go @@ -45,9 +45,9 @@ const ( phaseWakeupProbe = "wakeup_probe" phaseActivate = "activate" - phaseCheckpoint = "checkpoint" - phaseDurableDir = "durable_dir" - phaseTeardown = "teardown" + phaseCheckpoint = "checkpoint" + phaseFSCheckpoint = "fs_checkpoint" + phaseTeardown = "teardown" phaseTotal = ateattr.SnapshotPhaseTotal ) @@ -57,8 +57,8 @@ const ( const containerCountKey = "ate.actor.container.count" // restoreTiming is the elapsed time of each RestoreWorkload phase. A phase -// left at zero never ran. Under a Data scope pauseRestore and appRestore time -// the cold start that stands in for the restore. +// left at zero never ran. Under VOLUMES and ROOTFS fidelity, pauseRestore and +// appRestore time the cold start that stands in for the restore. type restoreTiming struct { prep, egressPrepare, netSetup time.Duration pauseRootfs, pauseCreate, pauseRestore time.Duration @@ -83,17 +83,19 @@ func (t restoreTiming) phases() []phaselog.Phase { } } -// checkpointTiming is the elapsed time of each CheckpointWorkload phase. A -// Full scope runs checkpoint; a Data scope runs durable_dir capture instead. +// checkpointTiming is the elapsed time of each CheckpointWorkload phase. +// MEMORY fidelity runs runsc checkpoint (recorded under checkpoint). VOLUMES +// and ROOTFS both run runsc fscheckpoint (recorded under fs_checkpoint); +// ate.snapshot.fidelity tells them apart. type checkpointTiming struct { - prep, checkpoint, durableDir, teardown, total time.Duration + prep, checkpoint, fsCheckpoint, teardown, total time.Duration } func (t checkpointTiming) phases() []phaselog.Phase { return []phaselog.Phase{ {Name: phasePrep, D: t.prep}, {Name: phaseCheckpoint, D: t.checkpoint}, - {Name: phaseDurableDir, D: t.durableDir}, + {Name: phaseFSCheckpoint, D: t.fsCheckpoint}, {Name: phaseTeardown, D: t.teardown}, {Name: phaseTotal, D: t.total}, } diff --git a/cmd/ateworker-gvisor/phaselog_test.go b/cmd/ateworker-gvisor/phaselog_test.go index e647c4c26a..11efbe3973 100644 --- a/cmd/ateworker-gvisor/phaselog_test.go +++ b/cmd/ateworker-gvisor/phaselog_test.go @@ -104,38 +104,55 @@ func TestRestoreTimingPhases(t *testing.T) { } } -// TestCheckpointTimingPhases: a Data checkpoint runs the durable capture and -// never runs the full runsc checkpoint, and the record says so. +// TestCheckpointTimingPhases: VOLUMES and ROOTFS checkpoints both record their +// runsc fscheckpoint under fs_checkpoint, the fidelity attribute tells them +// apart, and phases that did not run stay off the record. func TestCheckpointTimingPhases(t *testing.T) { t.Parallel() - timing := checkpointTiming{ - prep: 20 * time.Millisecond, - durableDir: 300 * time.Millisecond, - teardown: 150 * time.Millisecond, - total: 500 * time.Millisecond, - } - rec := renderPhaseRecord(t, phaselog.SnapshotPhaseAttrs(resources.ActorAttribution{UID: "uid-abc"}, - ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, phaselog.CheckpointDurationKey, nil, timing.phases())) - - for k, want := range map[string]float64{ - "ateworker.actor.checkpoint.duration.prep": 0.02, - "ateworker.actor.checkpoint.duration.durable_dir": 0.3, - "ateworker.actor.checkpoint.duration.teardown": 0.15, - "ateworker.actor.checkpoint.duration.total": 0.5, + for _, tc := range []struct { + fidelity ateworkerpb.SnapshotFidelity + want string + }{ + {ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, "volumes"}, + {ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, "rootfs"}, } { - got, ok := rec[k].(float64) - if !ok { - t.Errorf("%s = %v (%T), want a number", k, rec[k], rec[k]) - } else if got != want { - t.Errorf("%s = %v, want %v", k, got, want) - } - } - if v, ok := rec["ateworker.actor.checkpoint.duration.checkpoint"]; ok { - t.Errorf("checkpoint present with %v, want absent", v) - } - if rec["ate.snapshot.fidelity"] != "volumes" { - t.Errorf("ate.snapshot.fidelity = %v, want volumes", rec["ate.snapshot.fidelity"]) + t.Run(tc.want, func(t *testing.T) { + t.Parallel() + timing := checkpointTiming{ + prep: 20 * time.Millisecond, + fsCheckpoint: 300 * time.Millisecond, + teardown: 150 * time.Millisecond, + total: 500 * time.Millisecond, + } + rec := renderPhaseRecord(t, phaselog.SnapshotPhaseAttrs(resources.ActorAttribution{UID: "uid-abc"}, + tc.fidelity, phaselog.CheckpointDurationKey, nil, timing.phases())) + + for k, want := range map[string]float64{ + "ateworker.actor.checkpoint.duration.prep": 0.02, + "ateworker.actor.checkpoint.duration.fs_checkpoint": 0.3, + "ateworker.actor.checkpoint.duration.teardown": 0.15, + "ateworker.actor.checkpoint.duration.total": 0.5, + } { + got, ok := rec[k].(float64) + if !ok { + t.Errorf("%s = %v (%T), want a number", k, rec[k], rec[k]) + } else if got != want { + t.Errorf("%s = %v, want %v", k, got, want) + } + } + for _, absent := range []string{ + "ateworker.actor.checkpoint.duration.checkpoint", + "ateworker.actor.checkpoint.duration.durable_dir", + } { + if v, ok := rec[absent]; ok { + t.Errorf("%s present with %v, want absent", absent, v) + } + } + if rec["ate.snapshot.fidelity"] != tc.want { + t.Errorf("ate.snapshot.fidelity = %v, want %s", rec["ate.snapshot.fidelity"], tc.want) + } + }) } } diff --git a/cmd/ateworker-gvisor/rootfs.go b/cmd/ateworker-gvisor/rootfs.go new file mode 100644 index 0000000000..0e8073bf71 --- /dev/null +++ b/cmd/ateworker-gvisor/rootfs.go @@ -0,0 +1,36 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "path/filepath" + + "go.ate.dev/substrate/internal/ateworker/actornet" + "go.ate.dev/substrate/internal/imagecache" + "go.ate.dev/substrate/internal/ocispec" + "go.ate.dev/substrate/internal/proto/ateworkerpb" +) + +// setupAppRootfs composes an application container's bundle rootfs, the base +// layer under the sandbox's restored rootfs writes, and writes its /etc files. +func setupAppRootfs(actorDirs *ateworkerpb.ActorDirs, containerName string) error { + bundle := ociBundlePath(actorDirs, containerName) + if err := imagecache.SetupBundleRootfs(bundle); err != nil { + return err + } + return actornet.WriteRootfsEtcFiles(filepath.Join(bundle, "rootfs"), ocispec.Hostname) +} diff --git a/cmd/ateworker-gvisor/runsc.go b/cmd/ateworker-gvisor/runsc.go index a9ab7675de..f41e6e7daf 100644 --- a/cmd/ateworker-gvisor/runsc.go +++ b/cmd/ateworker-gvisor/runsc.go @@ -24,13 +24,11 @@ import ( "log/slog" "os" "os/exec" - "path/filepath" "slices" "strings" "go.ate.dev/substrate/internal/ocispec" "go.ate.dev/substrate/internal/proto/ateworkerpb" - "go.ate.dev/substrate/internal/resources" "go.ate.dev/substrate/internal/sizing" ) @@ -111,7 +109,6 @@ func (r *runsc) writeSpec(containerName string) error { ContainerName: containerName, DurableVolumes: r.durableVolumes, Size: r.size, - ResolvConf: resolvConfPath(r.actorDirs), }) return ocispec.Save(ociBundlePath(r.actorDirs, containerName), spec) } @@ -164,9 +161,10 @@ func (r *runsc) cmdStart(ctx context.Context, out io.Writer, containerName strin // checkpointArgs builds the argv for `runsc checkpoint `. Factored // out so the argument construction can be unit-tested without executing runsc. func (r *runsc) checkpointArgs(containerName, checkpointPath string, fsCheckpointPaths []string) []string { - extra := []string{"-image-path", checkpointPath} - if len(fsCheckpointPaths) > 0 { - extra = append(extra, "-fs-checkpoint-paths", strings.Join(fsCheckpointPaths, ",")) + extra := make([]string, 0, 2+2*len(fsCheckpointPaths)) + extra = append(extra, "-image-path", checkpointPath) + for _, p := range fsCheckpointPaths { + extra = append(extra, "-fs-checkpoint-paths", p) } return r.buildArgs("checkpoint", containerName, extra...) } @@ -189,7 +187,7 @@ func (r *runsc) cmdCheckpoint(ctx context.Context, containerName, checkpointPath // executing runsc. func (r *runsc) fsCheckpointArgs(containerName, checkpointPath string, fsCheckpointPaths []string) []string { extra := make([]string, 0, 2+2*len(fsCheckpointPaths)) - extra = append(extra, "-image-path", filepath.Join(checkpointPath, resources.FSCheckpointSubdir)) + extra = append(extra, "-image-path", checkpointPath) for _, ddv := range fsCheckpointPaths { extra = append(extra, "-path", ddv) } diff --git a/cmd/ateworker-gvisor/runsc_test.go b/cmd/ateworker-gvisor/runsc_test.go index 8f1489851c..badafe71be 100644 --- a/cmd/ateworker-gvisor/runsc_test.go +++ b/cmd/ateworker-gvisor/runsc_test.go @@ -23,6 +23,7 @@ import ( "path/filepath" "reflect" "slices" + "strings" "testing" "github.com/opencontainers/runtime-spec/specs-go" @@ -214,14 +215,15 @@ func TestCheckpointArgs(t *testing.T) { } checkpointDir := "/node/actors/test-actor-123/checkpoints/snap-1" - got := r.checkpointArgs(ocispec.PauseContainer, checkpointDir, []string{"app:/var/data", "app:/var/cache"}) + got := r.checkpointArgs(ocispec.PauseContainer, checkpointDir, []string{"rootfs/=app:/", "fs/=app:/var/data"}) want := []string{ "-global1", "-global2", "-root", "/node/actors/test-actor-123/runsc-state", "checkpoint", "-image-path", checkpointDir, - "-fs-checkpoint-paths", "app:/var/data,app:/var/cache", + "-fs-checkpoint-paths", "rootfs/=app:/", + "-fs-checkpoint-paths", "fs/=app:/var/data", ocispec.PauseContainer, } @@ -239,15 +241,15 @@ func TestFsCheckpointArgs(t *testing.T) { } checkpointDir := "/node/actors/test-actor-123/checkpoints/snap-1" - got := r.fsCheckpointArgs(ocispec.PauseContainer, checkpointDir, []string{"app:/var/data", "app:/var/cache"}) + got := r.fsCheckpointArgs(ocispec.PauseContainer, checkpointDir, []string{"rootfs/=app:/", "fs/=app:/var/data"}) want := []string{ "-global1", "-global2", "-root", "/node/actors/test-actor-123/runsc-state", "fscheckpoint", - "-image-path", filepath.Join(checkpointDir, "fs"), - "-path", "app:/var/data", - "-path", "app:/var/cache", + "-image-path", checkpointDir, + "-path", "rootfs/=app:/", + "-path", "fs/=app:/var/data", ocispec.PauseContainer, } @@ -291,34 +293,22 @@ func TestRestoreWorkloadRejectsEmptyRestoreDir(t *testing.T) { inFlight: actorlock.NewInFlight(), } + noDirs := validRestoreRequest() + noDirs.ActorDirs = nil + noRestoreDir := validRestoreRequest() + noRestoreDir.ActorDirs.RestoreDir = "" for _, tc := range []struct { name string req *ateworkerpb.RestoreWorkloadRequest + want string }{ - { - name: "nil actor_dirs", - req: &ateworkerpb.RestoreWorkloadRequest{ActorUid: "actor-a"}, - }, - { - name: "empty restore_dir", - req: &ateworkerpb.RestoreWorkloadRequest{ - ActorUid: "actor-a", - ActorDirs: &ateworkerpb.ActorDirs{ - RootDir: "/node/actors/actor-a", - OciBundleDir: "/node/actors/actor-a/bundle", - CheckpointDir: "/node/actors/actor-a/checkpoint-state", - DurableDirVolumeMountsDir: "/node/actors/actor-a/durable-dirs", - SystemInfoVolumeRootsDir: "/node/actors/actor-a/system-info", - ExternalVolumeMountsDir: "/node/actors/actor-a/volumes", - SandboxAssetsDir: "/node/actors/actor-a/sandbox-assets", - }, - }, - }, + {name: "nil actor_dirs", req: noDirs, want: "actor_dirs"}, + {name: "empty restore_dir", req: noRestoreDir, want: "restore_dir"}, } { t.Run(tc.name, func(t *testing.T) { _, err := s.RestoreWorkload(context.Background(), tc.req) - if apierror.Code(err) != codes.InvalidArgument { - t.Fatalf("RestoreWorkload() code = %v, want %v (err: %v)", apierror.Code(err), codes.InvalidArgument, err) + if apierror.Code(err) != codes.InvalidArgument || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("RestoreWorkload() = %v, want InvalidArgument naming %s", err, tc.want) } }) } diff --git a/cmd/ateworker-gvisor/sandboxnet.go b/cmd/ateworker-gvisor/sandboxnet.go deleted file mode 100644 index d1b368de54..0000000000 --- a/cmd/ateworker-gvisor/sandboxnet.go +++ /dev/null @@ -1,57 +0,0 @@ -//go:build linux - -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "errors" - "fmt" - "io/fs" - "log/slog" - "os" - "path/filepath" - - "go.ate.dev/substrate/internal/ateworker/actornet" - "go.ate.dev/substrate/internal/proto/ateworkerpb" -) - -// actorResolvConf writes the resolver bind source outside the actor's rootfs. -func actorResolvConf(actorDirs *ateworkerpb.ActorDirs) (string, error) { - pod, err := os.ReadFile("/etc/resolv.conf") - if err != nil { - return "", fmt.Errorf("reading the worker pod resolv.conf: %w", err) - } - path := resolvConfPath(actorDirs) - if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { - return "", fmt.Errorf("creating the actor directory: %w", err) - } - if err := os.WriteFile(path, actornet.SandboxResolvConf(actornet.ActorVethGateway, pod), 0o644); err != nil { - return "", fmt.Errorf("writing the actor resolv.conf: %w", err) - } - return path, nil -} - -// removeActorResolvConf drops the file with the actor's network; atelet's -// per-activation reset clears directories under the actor's path, not files. -func removeActorResolvConf(ctx context.Context, path string) { - if path == "" { - return - } - if err := os.Remove(path); err != nil && !errors.Is(err, fs.ErrNotExist) { - slog.WarnContext(ctx, "Failed to remove the actor resolv.conf", slog.Any("err", err)) - } -} diff --git a/cmd/ateworker-gvisor/stats.go b/cmd/ateworker-gvisor/stats.go index 5a2a71b7ab..dce914cd1d 100644 --- a/cmd/ateworker-gvisor/stats.go +++ b/cmd/ateworker-gvisor/stats.go @@ -27,6 +27,7 @@ import ( "go.ate.dev/substrate/cmd/ateworker-gvisor/internal/cgroupstats" "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/ateworkervalidation" "go.ate.dev/substrate/internal/ocispec" "go.ate.dev/substrate/internal/proto/ateworkerpb" "go.ate.dev/substrate/internal/resources" @@ -94,8 +95,8 @@ const sandboxCgroupContainer = ocispec.PauseContainer // boots and checkpoints: polls would stall behind runsc, and a checkpoint // would wait on telemetry. The cgroup files are read with no lock held. func (s *AteWorkerService) GetWorkloadStats(ctx context.Context, req *ateworkerpb.GetWorkloadStatsRequest) (*ateworkerpb.GetWorkloadStatsResponse, error) { - if req.GetActorUid() == "" { - return nil, apierror.InvalidArgument("actor_uid is required") + if err := ateworkervalidation.ValidateGetWorkloadStatsRequest(ctx, req); err != nil { + return nil, err } // NOT_FOUND rather than FAILED_PRECONDITION: the requested actor is not diff --git a/cmd/ateworker-gvisor/stats_test.go b/cmd/ateworker-gvisor/stats_test.go index 2b75483651..7a211acc7c 100644 --- a/cmd/ateworker-gvisor/stats_test.go +++ b/cmd/ateworker-gvisor/stats_test.go @@ -46,11 +46,15 @@ import ( var testActor = resources.ActorAttribution{ Ref: resources.ActorRef{Atespace: "space-a", Name: "actor-a"}, - UID: "uid-a", + UID: "aaaaaaaa-0000-4000-8000-000000000001", TemplateAtespace: "ns-a", TemplateName: "template-a", } +// otherActorUID is a second actor's UID, for requests and targets that must +// not match testActor. +const otherActorUID = "bbbbbbbb-0000-4000-8000-000000000002" + // healthyCPUUsec is the usage_usec in healthyCgroup. const healthyCPUUsec = 1234567 @@ -108,7 +112,7 @@ func TestGetWorkloadStats(t *testing.T) { setHostedActor(s, &testActor) before := time.Now().UnixNano() - got, err := s.GetWorkloadStats(context.Background(), &ateworkerpb.GetWorkloadStatsRequest{ActorUid: "uid-a"}) + got, err := s.GetWorkloadStats(context.Background(), &ateworkerpb.GetWorkloadStatsRequest{ActorUid: testActor.UID}) after := time.Now().UnixNano() if err != nil { t.Fatalf("GetWorkloadStats() error = %v, want nil", err) @@ -123,7 +127,7 @@ func TestGetWorkloadStats(t *testing.T) { want := &ateworkerpb.GetWorkloadStatsResponse{Sample: &ateworkerpb.WorkloadStatsSample{ Atespace: "space-a", ActorName: "actor-a", - ActorUid: "uid-a", + ActorUid: testActor.UID, ActorTemplateAtespace: "ns-a", ActorTemplateName: "template-a", SandboxClass: "gvisor", @@ -157,13 +161,20 @@ func TestGetWorkloadStatsErrors(t *testing.T) { actorUID: "", want: codes.InvalidArgument, }, + { + name: "malformed actor_uid", + files: healthyCgroup, + active: &testActor, + actorUID: "not-a-uuid", + want: codes.InvalidArgument, + }, { // Not here at all. NOT_FOUND rather than FAILED_PRECONDITION, because // what the caller should do about it is re-resolve, not retry. name: "ateworker is available", files: healthyCgroup, active: nil, - actorUID: "uid-a", + actorUID: testActor.UID, want: codes.NotFound, }, { @@ -173,7 +184,7 @@ func TestGetWorkloadStatsErrors(t *testing.T) { name: "actor_uid does not match the executing workload", files: healthyCgroup, active: &testActor, - actorUID: "uid-b", + actorUID: otherActorUID, want: codes.NotFound, }, { @@ -184,7 +195,7 @@ func TestGetWorkloadStatsErrors(t *testing.T) { name: "no sandbox cgroup to measure", files: nil, active: &testActor, - actorUID: "uid-a", + actorUID: testActor.UID, want: codes.FailedPrecondition, }, { @@ -193,7 +204,7 @@ func TestGetWorkloadStatsErrors(t *testing.T) { name: "sandbox cgroup is malformed", files: map[string]string{"memory.current": "max\n"}, active: &testActor, - actorUID: "uid-a", + actorUID: testActor.UID, want: codes.Internal, }, } { @@ -227,7 +238,7 @@ func TestGetWorkloadStatsDoesNotTakeLock(t *testing.T) { } defer s.locks.Unlock(testActor.UID) - if _, err := s.GetWorkloadStats(context.Background(), &ateworkerpb.GetWorkloadStatsRequest{ActorUid: "uid-a"}); err != nil { + if _, err := s.GetWorkloadStats(context.Background(), &ateworkerpb.GetWorkloadStatsRequest{ActorUid: testActor.UID}); err != nil { t.Errorf("GetWorkloadStats() error = %v, want nil", err) } } @@ -257,7 +268,7 @@ func TestGetActiveWorkloadStats(t *testing.T) { // The keyed read against the same fixture is the reference: the discovery // read must produce the identical sample, since both are the same // measurement with a different addressing mode. - want, err := s.GetWorkloadStats(context.Background(), &ateworkerpb.GetWorkloadStatsRequest{ActorUid: "uid-a"}) + want, err := s.GetWorkloadStats(context.Background(), &ateworkerpb.GetWorkloadStatsRequest{ActorUid: testActor.UID}) if err != nil { t.Fatalf("GetWorkloadStats() error = %v, want nil", err) } @@ -330,7 +341,7 @@ func TestGetActiveWorkloadStatsNoCgroup(t *testing.T) { // activation they were read for. func TestGetActiveWorkloadStatsTransition(t *testing.T) { otherActor := testActor - otherActor.UID = "uid-b" + otherActor.UID = otherActorUID otherTemplate := testActor otherTemplate.TemplateName = "template-b" @@ -381,7 +392,7 @@ func TestGetWorkloadStatsTransition(t *testing.T) { return cgroupstats.Read(dir) } - _, err := s.GetWorkloadStats(context.Background(), &ateworkerpb.GetWorkloadStatsRequest{ActorUid: "uid-a"}) + _, err := s.GetWorkloadStats(context.Background(), &ateworkerpb.GetWorkloadStatsRequest{ActorUid: testActor.UID}) if got := apierror.Code(err); got != codes.NotFound { t.Errorf("GetWorkloadStats() during transition: code = %v, want %v (err: %v)", got, codes.NotFound, err) } @@ -389,7 +400,7 @@ func TestGetWorkloadStatsTransition(t *testing.T) { func TestGetActiveWorkloadStatsSeveralActors(t *testing.T) { second := testActor - second.UID = "uid-b" + second.UID = otherActorUID s := newStatsService(t, healthyCgroup) // Give the second actor its own cgroup fixture. @@ -563,7 +574,7 @@ func TestSweepUsageDeadSandbox(t *testing.T) { t.Fatalf("got %d dead-sandbox warnings, want %d; logs:\n%s", n, tc.wantWarns, logs) } if tc.wantWarns > 0 { - for _, want := range []string{`"ate.actor.uid":"uid-a"`, `"ate.actor.name":"actor-a"`, `"ate.atespace":"space-a"`, `"ate.template.name":"template-a"`, `"ate.sandbox.oom_kills":3`} { + for _, want := range []string{`"ate.actor.uid":"` + testActor.UID + `"`, `"ate.actor.name":"actor-a"`, `"ate.atespace":"space-a"`, `"ate.template.name":"template-a"`, `"ate.sandbox.oom_kills":3`} { if !strings.Contains(logs.String(), want) { t.Errorf("warning missing %s; logs:\n%s", want, logs) } diff --git a/cmd/ateworker-microvm/checkpoint.go b/cmd/ateworker-microvm/checkpoint.go index b4e80fa60f..68de889094 100644 --- a/cmd/ateworker-microvm/checkpoint.go +++ b/cmd/ateworker-microvm/checkpoint.go @@ -31,6 +31,7 @@ import ( "go.ate.dev/substrate/cmd/ateworker-microvm/internal/ch" "go.ate.dev/substrate/cmd/ateworker-microvm/internal/kata" "go.ate.dev/substrate/internal/ateworker/phaselog" + "go.ate.dev/substrate/internal/ateworkervalidation" "go.ate.dev/substrate/internal/imagecache" "go.ate.dev/substrate/internal/proto/ateworkerpb" "go.ate.dev/substrate/internal/workerstats" @@ -66,7 +67,7 @@ import ( // Allow checkpointing even if the pod is shutting down. This will allow actors // (or the harness) to suspend on shutdown. func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworkerpb.CheckpointWorkloadRequest) (_ *ateworkerpb.CheckpointWorkloadResponse, err error) { - if err := validateActorDirs(req.GetActorDirs()); err != nil { + if err := ateworkervalidation.ValidateCheckpointWorkloadRequest(ctx, req); err != nil { return nil, err } if err := validateFidelity(req.GetFidelity()); err != nil { @@ -407,7 +408,7 @@ func (s *AteWorkerService) teardownActor(ctx context.Context, id string, actorDi // TerminateWorkload stops the running actor, tears down its VMM, and cleans up // networking and overlays. func (s *AteWorkerService) TerminateWorkload(ctx context.Context, req *ateworkerpb.TerminateWorkloadRequest) (*ateworkerpb.TerminateWorkloadResponse, error) { - if err := validateActorDirs(req.GetActorDirs()); err != nil { + if err := ateworkervalidation.ValidateTerminateWorkloadRequest(ctx, req); err != nil { return nil, err } if !s.locks.Lock(ctx, req.GetActorUid()) { diff --git a/cmd/ateworker-microvm/hosted.go b/cmd/ateworker-microvm/hosted.go index 61978a513e..3447e9ed9b 100644 --- a/cmd/ateworker-microvm/hosted.go +++ b/cmd/ateworker-microvm/hosted.go @@ -73,9 +73,6 @@ func (s *AteWorkerService) admitActor(attribution resources.ActorAttribution, re // is as for admitActor. func (s *AteWorkerService) hostActor(ctx context.Context, attribution resources.ActorAttribution, resumesGuest bool) (*hostedActor, error) { uid := attribution.UID - if uid == "" { - return nil, fmt.Errorf("actor UID is required") - } hosted, stale, err := s.admitActor(attribution, resumesGuest) if err != nil { diff --git a/cmd/ateworker-microvm/internal/ch/createvm.go b/cmd/ateworker-microvm/internal/ch/createvm.go index 9019294cf2..634f498e87 100644 --- a/cmd/ateworker-microvm/internal/ch/createvm.go +++ b/cmd/ateworker-microvm/internal/ch/createvm.go @@ -28,6 +28,7 @@ type VmConfig struct { Payload PayloadConfig `json:"payload"` Pmem []PmemConfig `json:"pmem,omitempty"` Fs []FsConfig `json:"fs,omitempty"` + Balloon *BalloonConfig `json:"balloon,omitempty"` Rng *RngConfig `json:"rng,omitempty"` Serial *ConsoleConfig `json:"serial,omitempty"` Console *ConsoleConfig `json:"console,omitempty"` @@ -35,6 +36,12 @@ type VmConfig struct { Platform *PlatformConfig `json:"platform,omitempty"` } +// BalloonConfig configures the virtio-balloon device. +type BalloonConfig struct { + Size int64 `json:"size"` + FreePageReporting bool `json:"free_page_reporting,omitempty"` +} + // FsConfig is a virtio-fs device backed by a vhost-user (virtiofsd) socket. The // overlay rootfs path uses it as the RO lower; the guest mounts it via the FsTag. type FsConfig struct { diff --git a/cmd/ateworker-microvm/main.go b/cmd/ateworker-microvm/main.go index df3131257d..0bb2cd9a42 100644 --- a/cmd/ateworker-microvm/main.go +++ b/cmd/ateworker-microvm/main.go @@ -414,14 +414,6 @@ func (s *AteWorkerService) beginRPC(actorUID, name string, cancel context.Cancel return release, nil } -// validateActorDirs rejects a request whose actor directories are unusable. -func validateActorDirs(actorDirs *ateworkerpb.ActorDirs) error { - if errs := resources.ValidateActorDirs(actorDirs, field.NewPath("actor_dirs")); len(errs) > 0 { - return apierror.InvalidArgument("%v", errs.ToAggregate()) - } - return nil -} - // supportedFidelities are the snapshot fidelities this runtime serves: the // rootfs upper is host-backed, so it ships with or without guest memory. var supportedFidelities = []ateworkerpb.SnapshotFidelity{ @@ -433,17 +425,15 @@ var supportedFidelities = []ateworkerpb.SnapshotFidelity{ // validateFidelity rejects a checkpoint or restore request whose fidelity // this runtime cannot serve. func validateFidelity(fidelity ateworkerpb.SnapshotFidelity) error { - if errs := resources.ValidateSnapshotFidelity(fidelity, supportedFidelities, field.NewPath("fidelity")); len(errs) > 0 { - return apierror.InvalidArgument("%v", errs.ToAggregate()) - } - return nil + errs := resources.ValidateSnapshotFidelity(fidelity, supportedFidelities, field.NewPath("fidelity")) + return resources.ToAPIError(errs) } // validateRuntimeAssetPaths ensures we only run assets from the actor's own // sandbox assets dir, where atelet staged them. The dir is derived from the // actor UID here and the request's sandbox_assets_dir must match it, so a // request cannot name a root of its own choosing. actorDirs must have passed -// validateActorDirs. +// the generated validator. func validateRuntimeAssetPaths(actorUID string, actorDirs *ateworkerpb.ActorDirs, paths map[string]string) error { root := actorDirs.GetSandboxAssetsDir() if errs := resources.ValidateSandboxAssetsDir(actorUID, root, field.NewPath("actor_dirs", "sandbox_assets_dir")); len(errs) > 0 { @@ -453,10 +443,7 @@ func validateRuntimeAssetPaths(actorUID string, actorDirs *ateworkerpb.ActorDirs for name, p := range paths { errs = append(errs, resources.ValidateRuntimeAssetPath(root, p, field.NewPath("runtime_asset_paths").Key(name))...) } - if len(errs) > 0 { - return resources.ToAPIError(errs) - } - return nil + return resources.ToAPIError(errs) } // rejectIfDraining returns a codes.Unavailable error if ateworker has begun graceful diff --git a/cmd/ateworker-microvm/main_test.go b/cmd/ateworker-microvm/main_test.go new file mode 100644 index 0000000000..35620cad37 --- /dev/null +++ b/cmd/ateworker-microvm/main_test.go @@ -0,0 +1,91 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "testing" + + "google.golang.org/grpc/codes" + + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/proto/ateworkerpb" +) + +// Every RPC with a request body rejects an empty one before touching any +// state: the generated validators run first in each handler. +func TestRPCsRejectInvalidRequests(t *testing.T) { + s := &AteWorkerService{} + ctx := context.Background() + for name, call := range map[string]func() error{ + "RunWorkload": func() error { + _, err := s.RunWorkload(ctx, &ateworkerpb.RunWorkloadRequest{}) + return err + }, + "RestoreWorkload": func() error { + _, err := s.RestoreWorkload(ctx, &ateworkerpb.RestoreWorkloadRequest{}) + return err + }, + "CheckpointWorkload": func() error { + _, err := s.CheckpointWorkload(ctx, &ateworkerpb.CheckpointWorkloadRequest{}) + return err + }, + "TerminateWorkload": func() error { + _, err := s.TerminateWorkload(ctx, &ateworkerpb.TerminateWorkloadRequest{}) + return err + }, + "GetWorkloadStats": func() error { + _, err := s.GetWorkloadStats(ctx, &ateworkerpb.GetWorkloadStatsRequest{}) + return err + }, + } { + if got := apierror.Code(call()); got != codes.InvalidArgument { + t.Errorf("%s() code = %v, want %v", name, got, codes.InvalidArgument) + } + } +} + +// The requests below describe testActor and pass the generated validators, so +// a test aimed at the asset-path check that runs after them sees that check's +// error rather than an unrelated rejection. Only RunWorkload and +// RestoreWorkload read the asset map, so only those two are built. + +func testDirs() *ateworkerpb.ActorDirs { + return &ateworkerpb.ActorDirs{ + RootDir: "/node/actors/actor-a", + OciBundleDir: "/node/actors/actor-a/bundle", + CheckpointDir: "/node/actors/actor-a/checkpoint-state", + RestoreDir: "/node/actors/actor-a/restore-state", + DurableDirVolumeMountsDir: "/node/actors/actor-a/durable-dirs", + SystemInfoVolumeRootsDir: "/node/actors/actor-a/system-info", + ExternalVolumeMountsDir: "/node/actors/actor-a/volumes", + SandboxAssetsDir: nodepath.ActorSandboxAssetsDir(testActor.UID), + } +} + +func testSpec() *ateworkerpb.WorkloadSpec { + return &ateworkerpb.WorkloadSpec{Containers: []*ateworkerpb.Container{{Name: "app"}}} +} + +func validRunRequest() *ateworkerpb.RunWorkloadRequest { + return &ateworkerpb.RunWorkloadRequest{Atespace: testActor.Ref.Atespace, ActorName: testActor.Ref.Name, ActorUid: testActor.UID, Spec: testSpec(), ActorDirs: testDirs()} +} + +func validRestoreRequest() *ateworkerpb.RestoreWorkloadRequest { + return &ateworkerpb.RestoreWorkloadRequest{Atespace: testActor.Ref.Atespace, ActorName: testActor.Ref.Name, ActorUid: testActor.UID, Spec: testSpec(), ActorDirs: testDirs(), Fidelity: ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY} +} diff --git a/cmd/ateworker-microvm/restore.go b/cmd/ateworker-microvm/restore.go index b7c7ff1032..ca50611339 100644 --- a/cmd/ateworker-microvm/restore.go +++ b/cmd/ateworker-microvm/restore.go @@ -35,6 +35,7 @@ import ( "go.ate.dev/substrate/cmd/ateworker-microvm/internal/ch" "go.ate.dev/substrate/cmd/ateworker-microvm/internal/kata" + "go.ate.dev/substrate/internal/ateworkervalidation" "go.ate.dev/substrate/internal/imagecache" "go.ate.dev/substrate/internal/proto/ateworkerpb" "go.ate.dev/substrate/internal/resources" @@ -105,7 +106,7 @@ func newReseedNonce() ([]byte, error) { // Contract with atelet: the snapshot's files are in ActorDirs.restore_dir, // and the durable-dir volume directories re-created (empty). func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb.RestoreWorkloadRequest) (resp *ateworkerpb.RestoreWorkloadResponse, retErr error) { - if err := validateActorDirs(req.GetActorDirs()); err != nil { + if err := ateworkervalidation.ValidateRestoreWorkloadRequest(ctx, req); err != nil { return nil, err } if err := validateFidelity(req.GetFidelity()); err != nil { @@ -292,12 +293,6 @@ func (s *AteWorkerService) restoreMemoryFidelity(ctx context.Context, p actorBoo // deterministic unpack of the same image at the same /rootfs path // (plus, for merged rootfs, the upper re-materialized from the tar). containers := p.containers - if len(containers) == 0 { - return apierror.InvalidArgument("actor spec has no containers") - } - if len(containers) > maxActorContainers { - return apierror.Unimplemented("ateworker-microvm supports at most %d containers, got %d", maxActorContainers, len(containers)) - } ctrs, err := s.buildActorContainers(p.actorUID, p.actorDirs, containers) if err != nil { return err diff --git a/cmd/ateworker-microvm/restore_test.go b/cmd/ateworker-microvm/restore_test.go index 32bbb2674a..8572dba7a1 100644 --- a/cmd/ateworker-microvm/restore_test.go +++ b/cmd/ateworker-microvm/restore_test.go @@ -322,34 +322,22 @@ func TestRestoreWorkloadRejectsEmptyRestoreDir(t *testing.T) { inFlight: actorlock.NewInFlight(), } + noDirs := validRestoreRequest() + noDirs.ActorDirs = nil + noRestoreDir := validRestoreRequest() + noRestoreDir.ActorDirs.RestoreDir = "" for _, tc := range []struct { name string req *ateworkerpb.RestoreWorkloadRequest + want string }{ - { - name: "nil actor_dirs", - req: &ateworkerpb.RestoreWorkloadRequest{ActorUid: "actor-a"}, - }, - { - name: "empty restore_dir", - req: &ateworkerpb.RestoreWorkloadRequest{ - ActorUid: "actor-a", - ActorDirs: &ateworkerpb.ActorDirs{ - RootDir: "/node/actors/actor-a", - OciBundleDir: "/node/actors/actor-a/bundle", - CheckpointDir: "/node/actors/actor-a/checkpoint-state", - DurableDirVolumeMountsDir: "/node/actors/actor-a/durable-dirs", - SystemInfoVolumeRootsDir: "/node/actors/actor-a/system-info", - ExternalVolumeMountsDir: "/node/actors/actor-a/volumes", - SandboxAssetsDir: "/node/actors/actor-a/sandbox-assets", - }, - }, - }, + {name: "nil actor_dirs", req: noDirs, want: "actor_dirs"}, + {name: "empty restore_dir", req: noRestoreDir, want: "restore_dir"}, } { t.Run(tc.name, func(t *testing.T) { _, err := s.RestoreWorkload(context.Background(), tc.req) - if apierror.Code(err) != codes.InvalidArgument { - t.Fatalf("RestoreWorkload() code = %v, want %v (err: %v)", apierror.Code(err), codes.InvalidArgument, err) + if apierror.Code(err) != codes.InvalidArgument || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("RestoreWorkload() = %v, want InvalidArgument naming %s", err, tc.want) } }) } @@ -505,29 +493,24 @@ func TestNewReseedNonce(t *testing.T) { func TestRPCsRejectUntrustedRuntimeAssetPaths(t *testing.T) { s := &AteWorkerService{} ctx := context.Background() - dirs := &ateworkerpb.ActorDirs{ - RootDir: "/node/actors/actor-a", - OciBundleDir: "/node/actors/actor-a/bundle", - CheckpointDir: "/node/actors/actor-a/checkpoint-state", - RestoreDir: "/node/actors/actor-a/restore", - DurableDirVolumeMountsDir: "/node/actors/actor-a/durable-dirs", - SystemInfoVolumeRootsDir: "/node/actors/actor-a/system-info", - ExternalVolumeMountsDir: "/node/actors/actor-a/volumes", - SandboxAssetsDir: "/node/actors/actor-a/sandbox-assets", - } assets := map[string]string{assetCH: "/bin/sh"} for name, call := range map[string]func() error{ "RunWorkload": func() error { - _, err := s.RunWorkload(ctx, &ateworkerpb.RunWorkloadRequest{ActorDirs: dirs, RuntimeAssetPaths: assets}) + req := validRunRequest() + req.RuntimeAssetPaths = assets + _, err := s.RunWorkload(ctx, req) return err }, "RestoreWorkload": func() error { - _, err := s.RestoreWorkload(ctx, &ateworkerpb.RestoreWorkloadRequest{ActorDirs: dirs, RuntimeAssetPaths: assets}) + req := validRestoreRequest() + req.RuntimeAssetPaths = assets + _, err := s.RestoreWorkload(ctx, req) return err }, } { - if got := apierror.Code(call()); got != codes.InvalidArgument { - t.Errorf("%s() code = %v, want %v", name, got, codes.InvalidArgument) + err := call() + if apierror.Code(err) != codes.InvalidArgument || !strings.Contains(err.Error(), "runtime_asset_paths") { + t.Errorf("%s() = %v, want InvalidArgument naming runtime_asset_paths", name, err) } } } diff --git a/cmd/ateworker-microvm/run.go b/cmd/ateworker-microvm/run.go index 0b60f7376d..ce61de6c12 100644 --- a/cmd/ateworker-microvm/run.go +++ b/cmd/ateworker-microvm/run.go @@ -29,7 +29,6 @@ import ( "syscall" "time" - "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/workerstats" "go.ate.dev/substrate/internal/ateworker/actornet" @@ -38,6 +37,7 @@ import ( "go.ate.dev/substrate/cmd/ateworker-microvm/internal/ch" "go.ate.dev/substrate/cmd/ateworker-microvm/internal/kata" "go.ate.dev/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb" + "go.ate.dev/substrate/internal/ateworkervalidation" "go.ate.dev/substrate/internal/imagecache" "go.ate.dev/substrate/internal/ocispec" "go.ate.dev/substrate/internal/proto/ateworkerpb" @@ -161,10 +161,6 @@ const vmmMemReserveMiB = 128 // binary needs more, and this floor cannot know how much. const minGuestMemMiB = 128 -// maxActorContainers is a sanity cap on containers per actor (all share the one -// micro-VM + virtiofsd). 25 is far above any real pod. -const maxActorContainers = 25 - // workloadIDs returns the guest container ids for the actor's containers, in // order. Recorded on runningActor so the SIGTERM handler knows which guest // workloads to signal and wait on, and it must name what the guest actually @@ -236,7 +232,7 @@ func (s *AteWorkerService) resolveRuntime(paths map[string]string) resolvedRunti // - Each container's bundle holds the overlay spec its rootfs/ is composed // from. ateworker builds the OCI spec itself. func (s *AteWorkerService) RunWorkload(ctx context.Context, req *ateworkerpb.RunWorkloadRequest) (resp *ateworkerpb.RunWorkloadResponse, retErr error) { - if err := validateActorDirs(req.GetActorDirs()); err != nil { + if err := ateworkervalidation.ValidateRunWorkloadRequest(ctx, req); err != nil { return nil, err } if err := validateRuntimeAssetPaths(req.GetActorUid(), req.GetActorDirs(), req.GetRuntimeAssetPaths()); err != nil { @@ -386,12 +382,6 @@ func (s *AteWorkerService) coldBootActor(ctx context.Context, p actorBootParams) // CreateContainer/StartContainer, driven below after the shared boot + // CreateSandbox + guest networking. containers := p.containers - if len(containers) == 0 { - return apierror.InvalidArgument("actor spec has no containers") - } - if len(containers) > maxActorContainers { - return apierror.Unimplemented("ateworker-microvm supports at most %d containers, got %d", maxActorContainers, len(containers)) - } // ateworker builds the CH vm.create itself, so it needs the guest kernel + image // paths directly. @@ -644,7 +634,7 @@ func (s *AteWorkerService) coldBootActor(ctx context.Context, p actorBootParams) } // buildActorContainers prepares each of the actor's containers for the shared -// micro-VM: it builds the OCI spec from the request, injects guest DNS, +// micro-VM: it builds the OCI spec from the request, injects guest DNS and hosts, // and records the bundle rootfs that backs the overlay's RO lower. No host disk is // mounted here — the merged overlays are assembled in stageMergedRootfs after the // sandbox state is clean. Both RunWorkload and RestoreWorkload go through here. @@ -659,7 +649,7 @@ func (s *AteWorkerService) buildActorContainers(actorUID string, actorDirs *atew } // Compose the bundle rootfs from the node's cached image layers (an // overlay mounted in this pod's namespace; no-op for bundles without an - // overlay spec). Everything downstream — the resolv.conf write below, + // overlay spec). Everything downstream — the /etc writes below, // the bind into virtiofsd's shared dir, the read-only remount — then // sees the composed tree, with host-side writes landing in the bundle's // private upper. The guest still builds its own writable upper on top. @@ -667,9 +657,9 @@ func (s *AteWorkerService) buildActorContainers(actorUID string, actorDirs *atew return nil, fmt.Errorf("while composing rootfs for %q: %w", cn, err) } bundleRootfs := filepath.Join(bundle, "rootfs") - // Set guest DNS before serving the rootfs over virtio-fs, on boot and restore. - if err := writeActorResolvConf(bundleRootfs); err != nil { - return nil, fmt.Errorf("while writing guest resolv.conf for %q: %w", cn, err) + // Set guest DNS and hosts before serving the rootfs over virtio-fs, on boot and restore. + if err := actornet.WriteRootfsEtcFiles(bundleRootfs, ocispec.Hostname); err != nil { + return nil, fmt.Errorf("while writing guest /etc files for %q: %w", cn, err) } ctrs[i] = actorContainer{ name: cn, @@ -824,6 +814,7 @@ func buildVMConfig(id, kernel, image, kparams, consoleLog string, memMiB, vcpus // with DAX, so its file data stays in the host page cache instead of being copied into guest RAM. Pmem: []ch.PmemConfig{{ID: guestImagePmemID, File: image, DiscardWrites: true}}, Fs: buildFsConfigs(id), + Balloon: &ch.BalloonConfig{FreePageReporting: true}, Platform: &ch.PlatformConfig{NumPciSegments: 2}, Rng: &ch.RngConfig{Src: "/dev/urandom"}, Console: &ch.ConsoleConfig{Mode: "File", File: consoleLog}, diff --git a/cmd/ateworker-microvm/run_test.go b/cmd/ateworker-microvm/run_test.go index 34699b0f96..48a166324c 100644 --- a/cmd/ateworker-microvm/run_test.go +++ b/cmd/ateworker-microvm/run_test.go @@ -18,6 +18,7 @@ package main import ( "context" + "encoding/json" "errors" "net" "path/filepath" @@ -222,3 +223,13 @@ func TestKernelParamsDebugConsole(t *testing.T) { } } } + +func TestBuildVMConfigBalloon(t *testing.T) { + cfg := buildVMConfig("actor-1", "/vmlinux", "/rootfs.img", "", "/console.log", 256, 1, true, false) + if cfg.Balloon == nil || !cfg.Balloon.FreePageReporting || cfg.Balloon.Size != 0 { + t.Fatalf("cfg.Balloon = %+v, want size 0 with free page reporting", cfg.Balloon) + } + if b, err := json.Marshal(cfg); err != nil || !strings.Contains(string(b), `"balloon":{"size":0,"free_page_reporting":true}`) { + t.Errorf("VmConfig JSON = %s, %v; want the balloon section cloud-hypervisor expects", b, err) + } +} diff --git a/cmd/ateworker-microvm/stats.go b/cmd/ateworker-microvm/stats.go index 2bc2d5c2a4..eec4baf9dd 100644 --- a/cmd/ateworker-microvm/stats.go +++ b/cmd/ateworker-microvm/stats.go @@ -29,6 +29,7 @@ import ( "go.ate.dev/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb" "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/ateworkervalidation" "go.ate.dev/substrate/internal/proto/ateworkerpb" "go.ate.dev/substrate/internal/resources" atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" @@ -102,8 +103,8 @@ type guestStatsTarget struct { // cold boot, snapshot, or restore; blocking there would silence the poller // through the phases whose usage matters most. func (s *AteWorkerService) GetWorkloadStats(ctx context.Context, req *ateworkerpb.GetWorkloadStatsRequest) (*ateworkerpb.GetWorkloadStatsResponse, error) { - if req.GetActorUid() == "" { - return nil, apierror.InvalidArgument("actor_uid is required") + if err := ateworkervalidation.ValidateGetWorkloadStatsRequest(ctx, req); err != nil { + return nil, err } // NOT_FOUND rather than FAILED_PRECONDITION: the requested actor is not @@ -397,9 +398,11 @@ func (s *AteWorkerService) sampleGuest(ctx context.Context, h *hostedActor) (*at // every container answered, so a deadline never yields a partial sum. // // The loop as a whole is bounded by ctx — the caller's RPC deadline — with -// maxActorContainers * statsCallTimeout as the ceiling when the caller set -// none. statsCallTimeout is per call so that one hung container read cannot -// eat the budget the remaining containers still need. +// the WorkloadSpec.containers bound (ten, its +k8s:maxItems tag) times +// statsCallTimeout as the ceiling when the caller set none, since validation +// caps the spec before anything boots. statsCallTimeout is per call so that +// one hung container read cannot eat the budget the remaining containers +// still need. func sumContainerStats(ctx context.Context, target *guestStatsTarget) (agentstats.Sample, map[string]uint64, error) { var ( total agentstats.Sample diff --git a/cmd/ateworker-microvm/stats_test.go b/cmd/ateworker-microvm/stats_test.go index 85977fa069..40303e1960 100644 --- a/cmd/ateworker-microvm/stats_test.go +++ b/cmd/ateworker-microvm/stats_test.go @@ -52,13 +52,13 @@ func TestActorBootParamsAttribution(t *testing.T) { name: "fully populated", p: actorBootParams{ actorRef: resources.ActorRef{Atespace: "atespace-a", Name: "actor-b"}, - actorUID: "uid-c", + actorUID: "cccccccc-0000-4000-8000-000000000003", templateAtespace: "template-ns-d", templateName: "template-name-e", }, want: resources.ActorAttribution{ Ref: resources.ActorRef{Atespace: "atespace-a", Name: "actor-b"}, - UID: "uid-c", + UID: "cccccccc-0000-4000-8000-000000000003", TemplateAtespace: "template-ns-d", TemplateName: "template-name-e", }, @@ -88,7 +88,7 @@ func TestActorBootParamsAttributionMatchesRequest(t *testing.T) { req := &ateworkerpb.RunWorkloadRequest{ Atespace: "atespace-a", ActorName: "actor-b", - ActorUid: "uid-c", + ActorUid: "cccccccc-0000-4000-8000-000000000003", ActorTemplateAtespace: "template-ns-d", ActorTemplateName: "template-name-e", } @@ -113,11 +113,15 @@ func TestActorBootParamsAttributionMatchesRequest(t *testing.T) { var testActor = resources.ActorAttribution{ Ref: resources.ActorRef{Atespace: "space-a", Name: "actor-a"}, - UID: "uid-a", + UID: "aaaaaaaa-0000-4000-8000-000000000001", TemplateAtespace: "ns-a", TemplateName: "template-a", } +// otherActorUID is a second actor's UID, for requests and targets that must +// not match testActor. +const otherActorUID = "bbbbbbbb-0000-4000-8000-000000000002" + // fakeAgent stands in for the kata-agent client: a canned reply per container // id, or a canned error for the ones that are meant to fail. type fakeAgent struct { @@ -226,7 +230,7 @@ func TestGetWorkloadStats(t *testing.T) { s := newStatsService(agent, "app_ovl") before := time.Now().UnixNano() - got, err := s.GetWorkloadStats(context.Background(), &ateworkerpb.GetWorkloadStatsRequest{ActorUid: "uid-a"}) + got, err := s.GetWorkloadStats(context.Background(), &ateworkerpb.GetWorkloadStatsRequest{ActorUid: testActor.UID}) after := time.Now().UnixNano() if err != nil { t.Fatalf("GetWorkloadStats() error = %v, want nil", err) @@ -241,7 +245,7 @@ func TestGetWorkloadStats(t *testing.T) { want := &ateworkerpb.GetWorkloadStatsResponse{Sample: &ateworkerpb.WorkloadStatsSample{ Atespace: "space-a", ActorName: "actor-a", - ActorUid: "uid-a", + ActorUid: testActor.UID, ActorTemplateAtespace: "ns-a", ActorTemplateName: "template-a", SandboxClass: "microvm", @@ -273,7 +277,7 @@ func TestGetWorkloadStatsSumsContainers(t *testing.T) { }} s := newStatsService(agent, "app_ovl", "sidecar_ovl") - got, err := s.GetWorkloadStats(context.Background(), &ateworkerpb.GetWorkloadStatsRequest{ActorUid: "uid-a"}) + got, err := s.GetWorkloadStats(context.Background(), &ateworkerpb.GetWorkloadStatsRequest{ActorUid: testActor.UID}) if err != nil { t.Fatalf("GetWorkloadStats() error = %v, want nil", err) } @@ -310,7 +314,7 @@ func TestGetWorkloadStatsSkipsUnreadableContainer(t *testing.T) { } s := newStatsService(agent, "app_ovl", "exited_ovl") - got, err := s.GetWorkloadStats(context.Background(), &ateworkerpb.GetWorkloadStatsRequest{ActorUid: "uid-a"}) + got, err := s.GetWorkloadStats(context.Background(), &ateworkerpb.GetWorkloadStatsRequest{ActorUid: testActor.UID}) if err != nil { t.Fatalf("GetWorkloadStats() error = %v, want nil", err) } @@ -329,7 +333,7 @@ func TestGetWorkloadStatsCountsAnsweredContainer(t *testing.T) { agent := &fakeAgent{stats: map[string]*agentpb.CgroupStats{"app_ovl": nil}} s := newStatsService(agent, "app_ovl") - got, err := s.GetWorkloadStats(context.Background(), &ateworkerpb.GetWorkloadStatsRequest{ActorUid: "uid-a"}) + got, err := s.GetWorkloadStats(context.Background(), &ateworkerpb.GetWorkloadStatsRequest{ActorUid: testActor.UID}) if err != nil { t.Fatalf("GetWorkloadStats() error = %v, want nil", err) } @@ -357,12 +361,18 @@ func TestGetWorkloadStatsErrors(t *testing.T) { actorUID: "", want: codes.InvalidArgument, }, + { + name: "malformed actor_uid", + service: func() *AteWorkerService { return newStatsService(healthy, "app_ovl") }, + actorUID: "not-a-uuid", + want: codes.InvalidArgument, + }, { // Not here at all. NOT_FOUND rather than FAILED_PRECONDITION, because // what the caller should do about it is re-resolve, not retry. name: "ateworker is available", service: func() *AteWorkerService { return &AteWorkerService{} }, - actorUID: "uid-a", + actorUID: testActor.UID, want: codes.NotFound, }, { @@ -371,7 +381,7 @@ func TestGetWorkloadStatsErrors(t *testing.T) { // another's name, and it is the same "not here" as the case above. name: "actor_uid does not match the executing workload", service: func() *AteWorkerService { return newStatsService(healthy, "app_ovl") }, - actorUID: "uid-b", + actorUID: otherActorUID, want: codes.NotFound, }, { @@ -384,7 +394,7 @@ func TestGetWorkloadStatsErrors(t *testing.T) { hostTestActor(s, testActor, nil) return s }, - actorUID: "uid-a", + actorUID: testActor.UID, want: codes.FailedPrecondition, }, { @@ -394,10 +404,10 @@ func TestGetWorkloadStatsErrors(t *testing.T) { name: "guest agent connection belongs to another actor", service: func() *AteWorkerService { s := newStatsService(healthy, "app_ovl") - hostTestActor(s, testActor, &guestStatsTarget{actorUID: "uid-b", agent: healthy, workloadIDs: []string{"app_ovl"}}) + hostTestActor(s, testActor, &guestStatsTarget{actorUID: otherActorUID, agent: healthy, workloadIDs: []string{"app_ovl"}}) return s }, - actorUID: "uid-a", + actorUID: testActor.UID, want: codes.Internal, }, { @@ -414,7 +424,7 @@ func TestGetWorkloadStatsErrors(t *testing.T) { }} return newStatsService(agent, "app_ovl", "sidecar_ovl") }, - actorUID: "uid-a", + actorUID: testActor.UID, want: codes.FailedPrecondition, }, { @@ -422,7 +432,7 @@ func TestGetWorkloadStatsErrors(t *testing.T) { // confident zero would be reporting a state we do not understand. name: "no containers to measure", service: func() *AteWorkerService { return newStatsService(healthy) }, - actorUID: "uid-a", + actorUID: testActor.UID, want: codes.FailedPrecondition, }, } { @@ -451,7 +461,7 @@ func TestGetWorkloadStatsDoesNotTakeLock(t *testing.T) { } defer s.locks.Unlock(testActor.UID) - if _, err := s.GetWorkloadStats(context.Background(), &ateworkerpb.GetWorkloadStatsRequest{ActorUid: "uid-a"}); err != nil { + if _, err := s.GetWorkloadStats(context.Background(), &ateworkerpb.GetWorkloadStatsRequest{ActorUid: testActor.UID}); err != nil { t.Errorf("GetWorkloadStats() error = %v, want nil", err) } } @@ -488,7 +498,7 @@ func TestGetActiveWorkloadStats(t *testing.T) { // The keyed read against the same fake is the reference: the discovery read // must produce the identical sample, since both are the same measurement // with a different addressing mode. - want, err := s.GetWorkloadStats(context.Background(), &ateworkerpb.GetWorkloadStatsRequest{ActorUid: "uid-a"}) + want, err := s.GetWorkloadStats(context.Background(), &ateworkerpb.GetWorkloadStatsRequest{ActorUid: testActor.UID}) if err != nil { t.Fatalf("GetWorkloadStats() error = %v, want nil", err) } @@ -556,7 +566,7 @@ func TestGetActiveWorkloadStatsBooting(t *testing.T) { func TestGetActiveWorkloadStatsSeveralActors(t *testing.T) { second := testActor - second.UID = "uid-b" + second.UID = otherActorUID second.Ref.Name = "actor-b" agent := &fakeAgent{stats: map[string]*agentpb.CgroupStats{ @@ -591,7 +601,7 @@ func TestGetActiveWorkloadStatsSeveralActors(t *testing.T) { // pending entry alongside their samples. func TestGetActiveWorkloadStatsOneBooting(t *testing.T) { booting := testActor - booting.UID = "uid-b" + booting.UID = otherActorUID agent := &fakeAgent{stats: map[string]*agentpb.CgroupStats{ "app_ovl": containerStats(1000, 2000, 100, 5000), @@ -624,7 +634,7 @@ func TestGetWorkloadStatsTransition(t *testing.T) { s := newStatsService(agent, "app_ovl") agent.onCall = func() { unhostTestActor(s, testActor.UID) } - _, err := s.GetWorkloadStats(context.Background(), &ateworkerpb.GetWorkloadStatsRequest{ActorUid: "uid-a"}) + _, err := s.GetWorkloadStats(context.Background(), &ateworkerpb.GetWorkloadStatsRequest{ActorUid: testActor.UID}) if got := apierror.Code(err); got != codes.NotFound { t.Errorf("GetWorkloadStats() during transition: code = %v, want %v (err: %v)", got, codes.NotFound, err) } @@ -638,7 +648,7 @@ func TestGetActiveWorkloadStatsStaleTarget(t *testing.T) { "app_ovl": containerStats(1000, 2000, 100, 5000), }} s := newStatsService(agent, "app_ovl") - hostTestActor(s, testActor, &guestStatsTarget{actorUID: "uid-b", agent: agent, workloadIDs: []string{"app_ovl"}}) + hostTestActor(s, testActor, &guestStatsTarget{actorUID: otherActorUID, agent: agent, workloadIDs: []string{"app_ovl"}}) got, err := sweepAndList(s) if err != nil { @@ -701,7 +711,7 @@ func TestGetActiveWorkloadStatsSamplesGuestsConcurrently(t *testing.T) { // activation they were read for. func TestGetActiveWorkloadStatsTransition(t *testing.T) { otherActor := testActor - otherActor.UID = "uid-b" + otherActor.UID = otherActorUID otherTemplate := testActor otherTemplate.TemplateName = "template-b" diff --git a/cmd/benchmarking/boomer-worker/main.go b/cmd/benchmarking/boomer-worker/main.go index 29b77a4a63..c36ca3790a 100644 --- a/cmd/benchmarking/boomer-worker/main.go +++ b/cmd/benchmarking/boomer-worker/main.go @@ -48,7 +48,7 @@ func main() { routerURL = flag.String("router-url", "http://atenet-router.ate-system.svc.cluster.local", "atenet HTTP router base URL (no trailing slash).") atespace = flag.String("atespace", "benchmark", "Atespace every actor this worker creates lives in. Ensured (CreateAtespace, AlreadyExists is ok) at startup.") promAddr = flag.String("prometheus-addr", ":8001", "Address for the Prometheus /metrics endpoint.") - configJSON = flag.String("config-json", "", "Initial dynconfig as a JSON object (keys: trace_probability, min_wait_time, max_wait_time, min_live_time, max_live_time in seconds, durdir_file_size_bytes, resume_mode, lifecycle_mode, durdir_read_mode, durdir_template, mem_target, mem_churn, mem_read, cpu_cores, cpu_duty_cycle, max_pings_per_wake, target_rps, total_actors). Unset fields keep their built-in defaults.") + configJSON = flag.String("config-json", "", "Initial dynconfig as a JSON object keyed by locust flag name in snake_case (trace_probability, min_wait_time, ...); the selected user class reads the keys it documents and ignores the rest. Unset keys keep the class's built-in defaults. Each payload the master serves later stands on its own in the same way, so a key it leaves null is back at the default.") masterWebPort = flag.Int("master-web-port", 0, "If non-zero, fetch dynconfig from http://{master-host}:{master-web-port}/boomer-config on each spawn message. Exits if the first fetch fails; later failures keep the last fetched values. {master-host} comes from boomer's existing --master-host flag.") configPollInterval = flag.Duration("config-poll-interval", 10*time.Second, "With --master-web-port, also fetch dynconfig on this interval. A spawn message comes only when the number of users or the spawn rate changes, thus a load shape that changes the sample rate alone needs this. Zero stops the polling.") userClass = flag.String("user-class", "glutton", fmt.Sprintf("Locust user class to run, lowercase; one of %s.", strings.Join(userclass.Names(), "|"))) @@ -103,17 +103,27 @@ func main() { return } - initialCfg, err := dynconfig.Parse([]byte(*configJSON), dynconfig.Config{ - MaxWait: 500 * time.Millisecond, - MaxPingsPerWake: 1, - }) - if err != nil { - slog.Error("failed to parse --config-json", slog.String("err", err.Error())) + entry, ok := userclass.Lookup(class) + if !ok { + slog.Error("fatal: unknown --user-class value", + slog.String("user_class", *userClass), + slog.String("known", strings.Join(userclass.Names(), ","))) os.Exit(1) } + // The holder is built from the class's own codec, so a --config-json + // the class cannot run on fails here, and a bad value from the master + // later is refused in favor of the last good config. + dyn := dynconfig.NewHolder(entry.Config) + if *configJSON != "" { + if _, _, err := dyn.Apply([]byte(*configJSON)); err != nil { + slog.Error("failed to parse --config-json", slog.String("err", err.Error())) + os.Exit(1) + } + } + ctx := context.Background() - sampler := btrace.NewUpdatableSampler(initialCfg.TraceProbability) + sampler := btrace.NewUpdatableSampler(dyn.Common().TraceProbability) tp, err := btrace.Init(ctx, "substrate-boomer", sampler) if err != nil { slog.Error("failed to initialize tracing", slog.String("err", err.Error())) @@ -145,8 +155,6 @@ func main() { transport.IdleConnTimeout = 5 * time.Minute httpClient := &http.Client{Timeout: 30 * time.Second, Transport: transport} - dyn := dynconfig.NewHolder(initialCfg) - if *masterWebPort > 0 { masterHost := flag.Lookup("master-host").Value.String() configURL := fmt.Sprintf("http://%s:%d/boomer-config", masterHost, *masterWebPort) @@ -196,13 +204,6 @@ func main() { ActorDeadline: time.Duration(*actorDeadline * float64(time.Second)), } - entry, ok := userclass.Lookup(class) - if !ok { - slog.Error("fatal: unknown --user-class value", - slog.String("user_class", *userClass), - slog.String("known", strings.Join(userclass.Names(), ","))) - os.Exit(1) - } taskFn, shutdownFn := entry.Init(cfg) slog.Info("registered boomer task", diff --git a/cmd/kubectl-ate/README.md b/cmd/kubectl-ate/README.md index 6c07170a49..4a0396b821 100644 --- a/cmd/kubectl-ate/README.md +++ b/cmd/kubectl-ate/README.md @@ -158,7 +158,7 @@ kubectl ate delete atespace ### Actor Templates An **actor template** describes what an actor runs: containers, volumes, -snapshot policy, sandbox runtime, and worker selection. Templates live in an +snapshot policy, and worker selection. Templates live in an atespace and are immutable — there is no update; delete and recreate to change one. @@ -186,7 +186,6 @@ for a complete manifest example. |---|---| | `ATESPACE` | The atespace the template belongs to. | | `NAME` | The template's name. | -| `SANDBOX CLASS` | The sandbox runtime family (`gvisor` or `microvm`). | | `GOLDEN TAG` | The golden tag's name once it exists; `DISABLED` when the template opts out of golden snapshot creation; empty while the golden build is still running. | | `ERROR` | `ERROR` when the golden build failed; `-o yaml` shows the full message. | | `AGE` | Time elapsed since the template was created. | @@ -202,10 +201,15 @@ kubectl ate create actor my-actor --template= -a # Resume an actor (assigns it to a free worker and restores its state) kubectl ate resume actor my-actor -a -# Suspend an actor (snapshots its state to storage and frees the worker) +# Suspend an actor (snapshots its state and frees the worker). By default the +# snapshot stays on the worker's node and is uploaded to storage in the background. kubectl ate suspend actor my-actor -a -# Revert an actor to its last external snapshot (discards live, paused, or crashed state and returns to SUSPENDED) +# Suspend an actor and wait for its snapshot to reach storage. +kubectl ate suspend actor my-actor -a --mode=sync + +# Revert an actor to its last external snapshot (discards live or crashed state, or suspended state with a node-local snapshot, and returns to SUSPENDED) +# A suspended actor without a node-local snapshot is already at its external snapshot, so it cannot be reverted kubectl ate revert actor my-actor -a # Delete an actor (by default, requires the actor to be SUSPENDED or CRASHED). @@ -229,6 +233,7 @@ kubectl ate get tags -A kubectl ate get tag [ ...] -a # Tag the snapshot a suspended actor holds, then publish or unpublish the tag. +# Tagging fails until the snapshot is in storage: suspend with --mode=sync to tag right away. kubectl ate create tag -a --actor [--scope published] kubectl ate update tag -a --scope published kubectl ate update tag -a --scope atespace diff --git a/cmd/kubectl-ate/internal/cmd/actor.go b/cmd/kubectl-ate/internal/cmd/actor.go index be7b57c711..07e3932930 100644 --- a/cmd/kubectl-ate/internal/cmd/actor.go +++ b/cmd/kubectl-ate/internal/cmd/actor.go @@ -49,9 +49,9 @@ var ( createActorTagFlag string deleteActorAtespaceFlag string deleteActorAnyStateFlag bool - pauseActorAtespaceFlag string resumeActorAtespaceFlag string suspendActorAtespaceFlag string + suspendActorModeFlag string logsActorAtespaceFlag string logsActorFollowFlag bool logsActorContainerFlag string @@ -165,30 +165,6 @@ var deleteActorCmd = &cobra.Command{ }, } -var pauseActorCmd = &cobra.Command{ - Use: "actor ", - Short: "Pause an actor", - Args: exactlyOneArg("", "atespace"), - RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - apiClient, err := newAPIClient(cmd) - if err != nil { - return fmt.Errorf("failed to connect to ate-api-server: %w", err) - } - defer apiClient.Close() - - actorRef := resources.ActorRef{Atespace: pauseActorAtespaceFlag, Name: args[0]} - resp, err := apiClient.PauseActor(ctx, &ateapipb.PauseActorRequest{ - Actor: actorRef.ToObjectRef(), - }) - if err != nil { - return fmt.Errorf("failed to pause actor: %w", err) - } - - return printer.PrintActorTo(cmd.OutOrStdout(), resp.GetActor(), outputFmt) - }, -} - var resumeActorCmd = &cobra.Command{ Use: "actor ", Short: "Resume an actor", @@ -218,6 +194,10 @@ var suspendActorCmd = &cobra.Command{ Short: "Suspend an actor", Args: exactlyOneArg("", "atespace"), RunE: func(cmd *cobra.Command, args []string) error { + mode, err := parseSuspendMode(suspendActorModeFlag) + if err != nil { + return wrapUsageError(err) + } ctx := cmd.Context() apiClient, err := newAPIClient(cmd) if err != nil { @@ -226,9 +206,7 @@ var suspendActorCmd = &cobra.Command{ defer apiClient.Close() actorRef := resources.ActorRef{Atespace: suspendActorAtespaceFlag, Name: args[0]} - resp, err := apiClient.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ - Actor: actorRef.ToObjectRef(), - }) + resp, err := apiClient.SuspendActor(ctx, buildSuspendActorRequest(actorRef, mode)) if err != nil { return fmt.Errorf("failed to suspend actor: %w", err) } @@ -244,6 +222,32 @@ var logsActorsCmd = &cobra.Command{ RunE: runLogsActor, } +// Values of kubectl ate suspend actor --mode. +const ( + suspendModeAsync = "async" + suspendModeSync = "sync" +) + +// parseSuspendMode maps the --mode flag value to a durability mode. +func parseSuspendMode(mode string) (ateapipb.DurabilityMode, error) { + switch mode { + case suspendModeAsync: + return ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS, nil + case suspendModeSync: + return ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS, nil + default: + return ateapipb.DurabilityMode_DURABILITY_MODE_UNSPECIFIED, fmt.Errorf("invalid --mode %q: must be one of %s, %s", mode, suspendModeAsync, suspendModeSync) + } +} + +// buildSuspendActorRequest builds the SuspendActor request for an actor. +func buildSuspendActorRequest(actorRef resources.ActorRef, mode ateapipb.DurabilityMode) *ateapipb.SuspendActorRequest { + return &ateapipb.SuspendActorRequest{ + Actor: actorRef.ToObjectRef(), + Options: &ateapipb.SuspendOptions{Mode: mode}, + } +} + func buildCreateActorRequest(actorName, atespace, template, tag string) (*ateapipb.CreateActorRequest, error) { templateRef, err := parseAtespacedName(template, atespace) if err != nil { @@ -746,13 +750,11 @@ func init() { deleteActorCmd.Flags().BoolVar(&deleteActorAnyStateFlag, "any-state", false, "delete the actor regardless of current state") deleteCmd.AddCommand(deleteActorCmd) - pauseActorCmd.Flags().StringVarP(&pauseActorAtespaceFlag, "atespace", "a", "", "the actor's atespace") - pauseCmd.AddCommand(pauseActorCmd) - resumeActorCmd.Flags().StringVarP(&resumeActorAtespaceFlag, "atespace", "a", "", "the actor's atespace") resumeCmd.AddCommand(resumeActorCmd) suspendActorCmd.Flags().StringVarP(&suspendActorAtespaceFlag, "atespace", "a", "", "the actor's atespace") + suspendActorCmd.Flags().StringVar(&suspendActorModeFlag, "mode", suspendModeAsync, "how the snapshot is made durable: async (return once the snapshot is on the worker's node, and upload it in the background) or sync (return once the snapshot is in object storage)") suspendCmd.AddCommand(suspendActorCmd) logsActorsCmd.Flags().BoolVarP(&logsActorFollowFlag, "follow", "f", false, "stream logs continuously") diff --git a/cmd/kubectl-ate/internal/cmd/actor_test.go b/cmd/kubectl-ate/internal/cmd/actor_test.go index 634369c2b8..a6d0954687 100644 --- a/cmd/kubectl-ate/internal/cmd/actor_test.go +++ b/cmd/kubectl-ate/internal/cmd/actor_test.go @@ -108,6 +108,71 @@ func TestBuildCreateActorRequest(t *testing.T) { } } +func TestBuildSuspendActorRequest(t *testing.T) { + tests := []struct { + name string + mode ateapipb.DurabilityMode + }{ + {name: "async", mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, + {name: "sync", mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := buildSuspendActorRequest(resources.ActorRef{Atespace: "demo", Name: "my-counter"}, test.mode) + want := &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: "demo", Name: "my-counter"}, + Options: &ateapipb.SuspendOptions{Mode: test.mode}, + } + if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { + t.Errorf("request mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestParseSuspendMode(t *testing.T) { + tests := []struct { + name string + mode string + want ateapipb.DurabilityMode + wantErr string + }{ + {name: "async", mode: "async", want: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, + {name: "sync", mode: "sync", want: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, + {name: "empty", mode: "", wantErr: `invalid --mode "": must be one of async, sync`}, + {name: "unknown", mode: "durable", wantErr: `invalid --mode "durable": must be one of async, sync`}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := parseSuspendMode(test.mode) + if test.wantErr != "" { + if err == nil || err.Error() != test.wantErr { + t.Fatalf("got err %v, want %q", err, test.wantErr) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != test.want { + t.Errorf("got %v, want %v", got, test.want) + } + }) + } +} + +func TestSuspendActorModeFlagDefault(t *testing.T) { + flag := suspendActorCmd.Flags().Lookup("mode") + if flag == nil { + t.Fatal("suspend actor has no --mode flag") + } + if got, want := flag.DefValue, suspendModeAsync; got != want { + t.Errorf("--mode default = %q, want %q", got, want) + } +} + func TestFilterAndDisplayLogLine(t *testing.T) { tests := []struct { name string diff --git a/cmd/kubectl-ate/internal/cmd/actortemplate_test.go b/cmd/kubectl-ate/internal/cmd/actortemplate_test.go index ec0033e2e9..c84f9c4d46 100644 --- a/cmd/kubectl-ate/internal/cmd/actortemplate_test.go +++ b/cmd/kubectl-ate/internal/cmd/actortemplate_test.go @@ -59,9 +59,6 @@ resources: snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://ate-snapshots/ate-demo-counter/ -sandboxConfig: - sandboxClass: gvisor - configName: gvisor-default volumes: - name: data durableDir: {} @@ -93,10 +90,6 @@ func TestActorTemplateFromManifest(t *testing.T) { PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, StorageLocation: "gs://ate-snapshots/ate-demo-counter/", }, - SandboxConfig: &ateapipb.SandboxConfig{ - SandboxClass: "gvisor", - ConfigName: "gvisor-default", - }, Volumes: []*ateapipb.Volume{{ Name: "data", DurableDir: &ateapipb.DurableDirVolumeSource{}, @@ -114,9 +107,6 @@ func TestActorTemplateFromManifest_SnakeCase(t *testing.T) { name: counter snapshot_config: storage_location: gs://ate-snapshots/ate-demo-counter/ -sandbox_config: - sandbox_class: microvm - config_name: microvm ` got, err := actorTemplateFromManifest([]byte(manifest)) if err != nil { @@ -125,9 +115,6 @@ sandbox_config: if got.GetSnapshotConfig().GetStorageLocation() != "gs://ate-snapshots/ate-demo-counter/" { t.Errorf("storage_location = %q", got.GetSnapshotConfig().GetStorageLocation()) } - if got.GetSandboxConfig().GetSandboxClass() != "microvm" { - t.Errorf("sandbox_class = %v", got.GetSandboxConfig().GetSandboxClass()) - } } func TestActorTemplateFromManifest_Errors(t *testing.T) { @@ -157,19 +144,16 @@ func TestActorTemplateFromManifest_DemoManifests(t *testing.T) { manifest string atespace string name string - class string }{ { manifest: "counter-template.yaml.tmpl", atespace: "ate-demo-counter", name: "counter", - class: "gvisor", }, { manifest: "counter-microvm-template.yaml.tmpl", atespace: "ate-demo-counter-microvm", name: "counter-microvm", - class: "microvm", }, } for _, test := range tests { @@ -198,9 +182,6 @@ func TestActorTemplateFromManifest_DemoManifests(t *testing.T) { t.Errorf("metadata = %s/%s, want %s/%s", got.GetMetadata().GetAtespace(), got.GetMetadata().GetName(), test.atespace, test.name) } - if got.GetSandboxConfig().GetSandboxClass() != test.class { - t.Errorf("sandbox class = %v, want %v", got.GetSandboxConfig().GetSandboxClass(), test.class) - } if len(got.GetContainers()) == 0 || got.GetSnapshotConfig().GetStorageLocation() == "" { t.Errorf("missing required fields: %v", got) } diff --git a/cmd/kubectl-ate/internal/cmd/revert_actor.go b/cmd/kubectl-ate/internal/cmd/revert_actor.go index 87ce5290c7..8928ac8852 100644 --- a/cmd/kubectl-ate/internal/cmd/revert_actor.go +++ b/cmd/kubectl-ate/internal/cmd/revert_actor.go @@ -31,7 +31,9 @@ var revertActorCmd = &cobra.Command{ Long: `Revert an actor to SUSPENDED, discarding its current execution. The actor's external snapshot is left untouched, so a later resume restores -from it. Only crashed, running, or paused actors can be reverted.`, +from it. Only crashed or running actors, or suspended actors with a node-local +snapshot, can be reverted. A suspended actor without a node-local snapshot is +already at its durable snapshot, so there is nothing to revert.`, Args: exactlyOneArg("", "atespace"), RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() diff --git a/cmd/kubectl-ate/internal/cmd/verbs.go b/cmd/kubectl-ate/internal/cmd/verbs.go index 87d332d442..ebe169dbcf 100644 --- a/cmd/kubectl-ate/internal/cmd/verbs.go +++ b/cmd/kubectl-ate/internal/cmd/verbs.go @@ -25,7 +25,6 @@ var ( createCmd = &cobra.Command{Use: "create", Short: "Create a resource"} updateCmd = &cobra.Command{Use: "update", Short: "Update a resource"} deleteCmd = &cobra.Command{Use: "delete", Short: "Delete a resource"} - pauseCmd = &cobra.Command{Use: "pause", Short: "Pause a resource"} resumeCmd = &cobra.Command{Use: "resume", Short: "Resume a resource"} suspendCmd = &cobra.Command{Use: "suspend", Short: "Suspend a resource"} logsCmd = &cobra.Command{Use: "logs", Short: "Print the logs for a resource"} @@ -33,7 +32,7 @@ var ( ) func init() { - for _, c := range []*cobra.Command{getCmd, createCmd, updateCmd, deleteCmd, pauseCmd, resumeCmd, suspendCmd, logsCmd, topCmd} { + for _, c := range []*cobra.Command{getCmd, createCmd, updateCmd, deleteCmd, resumeCmd, suspendCmd, logsCmd, topCmd} { requireSubcommand(c) rootCmd.AddCommand(c) } diff --git a/cmd/kubectl-ate/internal/printer/printer.go b/cmd/kubectl-ate/internal/printer/printer.go index 7b7cf83412..c41c6d6663 100644 --- a/cmd/kubectl-ate/internal/printer/printer.go +++ b/cmd/kubectl-ate/internal/printer/printer.go @@ -292,7 +292,7 @@ func PrintActorTemplatesTo(out io.Writer, templates []*ateapipb.ActorTemplate, f return printProto(out, &ateapipb.ListActorTemplatesResponse{ActorTemplates: templates}, format) case "table": w := tabwriter.NewWriter(out, 0, 0, 3, ' ', 0) - fmt.Fprintln(w, "ATESPACE\tNAME\tSANDBOX CLASS\tGOLDEN TAG\tERROR\tAGE") + fmt.Fprintln(w, "ATESPACE\tNAME\tGOLDEN TAG\tERROR\tAGE") for _, t := range templates { gss := t.GetStatus().GetGoldenSnapshotStatus() goldenTag := gss.GetGoldenTag().GetName() @@ -304,9 +304,8 @@ func PrintActorTemplatesTo(out io.Writer, templates []*ateapipb.ActorTemplate, f if gss.GetErrorMessage() != "" { errFlag = "ERROR" } - fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", t.GetMetadata().GetAtespace(), t.GetMetadata().GetName(), - t.GetSandboxConfig().GetSandboxClass(), goldenTag, errFlag, formatAge(t.GetMetadata().GetCreateTime())) } diff --git a/cmd/kubectl-ate/internal/printer/printer_test.go b/cmd/kubectl-ate/internal/printer/printer_test.go index fb78ec1261..3137bc69b8 100644 --- a/cmd/kubectl-ate/internal/printer/printer_test.go +++ b/cmd/kubectl-ate/internal/printer/printer_test.go @@ -434,13 +434,9 @@ func TestPrintActorTemplatesTo_Table(t *testing.T) { Version: 1, CreateTime: timestamppb.New(now.Add(-5 * time.Minute)), }, - SandboxConfig: &ateapipb.SandboxConfig{ - SandboxClass: "gvisor", - ConfigName: "gvisor-default", - }, Status: &ateapipb.ActorTemplateStatus{ GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ - GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden-tag"}, + GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-demo-counter-substrate", Name: "golden-tag"}, }, }, }, @@ -451,10 +447,6 @@ func TestPrintActorTemplatesTo_Table(t *testing.T) { Version: 1, CreateTime: timestamppb.New(now.Add(-5 * time.Hour)), }, - SandboxConfig: &ateapipb.SandboxConfig{ - SandboxClass: "microvm", - ConfigName: "microvm", - }, Status: &ateapipb.ActorTemplateStatus{ GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ ErrorMessage: "golden actor failed to start", @@ -468,10 +460,6 @@ func TestPrintActorTemplatesTo_Table(t *testing.T) { Version: 1, CreateTime: timestamppb.New(now.Add(-72 * time.Hour)), }, - SandboxConfig: &ateapipb.SandboxConfig{ - SandboxClass: "gvisor", - ConfigName: "gvisor-default", - }, }, { Metadata: &ateapipb.ResourceMetadata{ @@ -479,7 +467,6 @@ func TestPrintActorTemplatesTo_Table(t *testing.T) { Name: "counter-3", CreateTime: timestamppb.New(now.Add(-15 * time.Minute)), }, - SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor"}, SnapshotConfig: &ateapipb.SnapshotConfig{ GoldenSnapshotConfig: &ateapipb.GoldenSnapshotConfig{Mode: ateapipb.GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_DISABLED}, }, @@ -492,11 +479,11 @@ func TestPrintActorTemplatesTo_Table(t *testing.T) { // Sorted by atespace, then name. The ERROR column only flags that an // error message exists; the full text is available via json/yaml. - expected := `ATESPACE NAME SANDBOX CLASS GOLDEN TAG ERROR AGE -ate-demo-counter-substrate counter gvisor golden-tag 5m -ate-demo-counter-substrate counter-2 gvisor 3d -ate-demo-counter-substrate counter-3 gvisor DISABLED 15m -ate-demo-counter-substrate-microvm counter-microvm microvm ERROR 5h + expected := `ATESPACE NAME GOLDEN TAG ERROR AGE +ate-demo-counter-substrate counter golden-tag 5m +ate-demo-counter-substrate counter-2 3d +ate-demo-counter-substrate counter-3 DISABLED 15m +ate-demo-counter-substrate-microvm counter-microvm ERROR 5h ` if diff := cmp.Diff(expected, buf.String()); diff != "" { t.Errorf("output mismatch (-want +got):\n%s", diff) diff --git a/demos/autoscaled-workerpool/autoscaled-workerpool-template.yaml.tmpl b/demos/autoscaled-workerpool/autoscaled-workerpool-template.yaml.tmpl index 04ac32f934..3eeda134bf 100644 --- a/demos/autoscaled-workerpool/autoscaled-workerpool-template.yaml.tmpl +++ b/demos/autoscaled-workerpool/autoscaled-workerpool-template.yaml.tmpl @@ -46,9 +46,6 @@ resources: snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_VOLUMES storageLocation: gs://${BUCKET_NAME}/ate-demo-autoscaled-workerpool/ -sandboxConfig: - sandboxClass: gvisor - configName: gvisor-default volumes: - name: data durableDir: {} diff --git a/demos/claude-code-multiplex/agent-luna-template.yaml.tmpl b/demos/claude-code-multiplex/agent-luna-template.yaml.tmpl index 69eb72c48f..ce154f3d8b 100644 --- a/demos/claude-code-multiplex/agent-luna-template.yaml.tmpl +++ b/demos/claude-code-multiplex/agent-luna-template.yaml.tmpl @@ -51,6 +51,3 @@ resources: snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/claude-multiplex-demo/ -sandboxConfig: - sandboxClass: gvisor - configName: gvisor-default diff --git a/demos/claude-code-multiplex/agent-mars-template.yaml.tmpl b/demos/claude-code-multiplex/agent-mars-template.yaml.tmpl index 080202ec43..19b48d2a47 100644 --- a/demos/claude-code-multiplex/agent-mars-template.yaml.tmpl +++ b/demos/claude-code-multiplex/agent-mars-template.yaml.tmpl @@ -43,6 +43,3 @@ resources: snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/claude-multiplex-demo/ -sandboxConfig: - sandboxClass: gvisor - configName: gvisor-default diff --git a/demos/claude-code-multiplex/agent-orion-template.yaml.tmpl b/demos/claude-code-multiplex/agent-orion-template.yaml.tmpl index 7a162bde47..8ecc7ac2f2 100644 --- a/demos/claude-code-multiplex/agent-orion-template.yaml.tmpl +++ b/demos/claude-code-multiplex/agent-orion-template.yaml.tmpl @@ -43,6 +43,3 @@ resources: snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/claude-multiplex-demo/ -sandboxConfig: - sandboxClass: gvisor - configName: gvisor-default diff --git a/demos/counter/README.md b/demos/counter/README.md index ea6ececc19..23db9d7ad4 100644 --- a/demos/counter/README.md +++ b/demos/counter/README.md @@ -2,7 +2,7 @@ This directory contains a demo of a stateful counter application running on Agent Substrate. -It deploys a simple Go HTTP server (`counter.go`) that increments two counters on every request — one in process memory, one in a file on a durable volume — and preserves both across suspends and resumes: the template's `MEMORY` `preferredFidelity` snapshot captures process memory alongside the durable volumes, so the in-memory count continues from where it left off. (With a `VOLUMES` fidelity, only the durable-volume counter would survive and the in-memory counter would restart from a cold boot.) +It deploys a simple Go HTTP server (`counter.go`) that increments two counters on every request — one in process memory, one in a file on a durable volume — and preserves both across suspends and resumes: the template's `MEMORY` `preferredFidelity` snapshot captures process memory alongside the durable volumes, so the in-memory count continues from where it left off. (With a `VOLUMES` fidelity, only the durable-volume counter would survive and the in-memory counter would restart from a cold boot. `ROOTFS` also keeps each container's rootfs writes, but not process memory, so the in-memory counter would restart under it too.) The demo uses two kinds of resources: the `WorkerPool` is a Kubernetes CRD, while the actor template is a Substrate `ActorTemplate` resource — an `ateapipb.ActorTemplate` living in an **atespace** rather than a Kubernetes namespace, managed through the ate API with `kubectl ate`. @@ -22,6 +22,42 @@ The demo uses two kinds of resources: the `WorkerPool` is a Kubernetes CRD, whil > automatically injects your `${BUCKET_NAME}` environment variable during > deployment. +
+Installation on MicroVM + +The in-RAM-counter suspend/resume-continuity demo be run on the micro-VM +sandbox class (`ateworker-microvm`: a Kata guest on Cloud Hypervisor). + +- [`demos/counter/counter-microvm.yaml.tmpl`](counter-microvm.yaml.tmpl) — + the `WorkerPool` for the micro-VM sandbox class. +- [`demos/counter/counter-microvm-template.yaml.tmpl`](counter-microvm-template.yaml.tmpl) — + the `counter-microvm` actor template, created through the ate API in the + `ate-demo-counter-microvm` atespace. +- [`hack/run-microvm-demo.sh`](../../hack/run-microvm-demo.sh) — one-shot bring-up + that builds the micro-VM worker image, stages the guest assets, deploys the + control plane, and applies the manifests above. Like the other hack scripts it + reads `.ate-dev-env.sh` for GKE; use the kind wrapper for a local cluster. + +Run it and follow the printed next steps: + +```bash +# GKE (uses .ate-dev-env.sh, uploads assets to GCS): +./hack/run-microvm-demo.sh + +# local kind (local registry + in-cluster rustfs): +KIND_CLUSTER_NAME= ./hack/run-microvm-demo-kind.sh +``` + +On a cluster that already has the micro-VM deps (`hack/install-microvm-deps.sh`), +deploy directly instead: + +```bash +./hack/install-ate.sh --deploy-demo-counter-microvm +``` +
+ +
+Installation on GVisor Use the core installation script to build the image and deploy the demo to your cluster: ```bash @@ -48,11 +84,27 @@ Inspect the deployed template with: kubectl ate get actor-templates -a ate-demo-counter kubectl ate get actor-template counter -a ate-demo-counter -o yaml ``` +
### 2. Create a Counter Actor Create the counter actor with a chosen ID (e.g., `my-counter-1`) using `--template` (the template's name, resolved in the actor's atespace — so the actor lives in the demo's atespace, which the deploy step already created): +
+Creation on MicroVM + +```bash +# Install the CLI as a kubectl plugin if not already installed +go install ./cmd/kubectl-ate + +# Create the actor from the counter template. +kubectl ate create actor my-counter-microvm-1 -a ate-demo-counter-microvm --template counter-microvm +``` +
+ +
+Creation on GVisor + ```bash # Install the CLI as a kubectl plugin if not already installed go install ./cmd/kubectl-ate @@ -60,6 +112,7 @@ go install ./cmd/kubectl-ate # Create the actor from the counter template. kubectl ate create actor my-counter-1 -a ate-demo-counter --template counter ``` +
### 3. Port-Forward Services @@ -81,21 +134,56 @@ The two `-H` options below are the routing key; the request URL and `Host` are ordinary application metadata. 1. Send an HTTP POST request to increment the counter: +
+Send to MicroVM +```bash +curl -X POST \ + -H "ate-target-actor: ate-demo-counter-microvm/my-counter-microvm-1" \ + http://localhost:8000 +``` +
+ +
+Send to GVisor ```bash curl -X POST \ -H "ate-target-actor: ate-demo-counter/my-counter-1" \ http://localhost:8000 ``` +
2. Verify that the actor is now in a `RUNNING` state and assigned to a worker pod: +
+Verify on MicroVM +```bash +kubectl ate get actor my-counter-microvm-1 -a ate-demo-counter-microvm +``` +
+ +
+Verify on GVisor ```bash kubectl ate get actor my-counter-1 -a ate-demo-counter ``` +
+ +3. When finished, you can manually suspend the actor: +
+Suspend on MicroVM +```bash +kubectl ate suspend actor my-counter-microvm-1 -a ate-demo-counter-microvm +``` +
-3. When finished, you can manually suspend the actor back to snapshot storage: +
+Suspend on GVisor ```bash kubectl ate suspend actor my-counter-1 -a ate-demo-counter ``` +
+ +The snapshot stays on the worker's node and is uploaded to snapshot storage in +the background. Add `--mode=sync` to wait for the upload instead. Repeat the `curl` from step 1 and the actor resumes from its snapshot — possibly on a different worker — with **both** counters continuing from where @@ -103,9 +191,19 @@ they left off: the memory count comes back from the `MEMORY` snapshot's process memory, the file counter from the durable volume. 4. To permanently delete the suspended actor: +
+Delete from MicroVM +```bash +kubectl ate delete actor my-counter-microvm-1 -a ate-demo-counter-microvm +``` +
+ +
+Delete from GVisor ```bash kubectl ate delete actor my-counter-1 -a ate-demo-counter ``` +
## Reaching a non-default port @@ -120,11 +218,23 @@ through it to the named port. `curl -p` forces a tunnel even for a plain (non-TLS) target, which its default proxy behavior wouldn't do: +
+Non standard port with MicroVM +```bash +curl -p -x http://localhost:8001 \ + --proxy-header "ate-target-actor: ate-demo-counter-microvm/my-counter-microvm-1" \ + http://my-counter-microvm-1:9090/ +``` +
+ +
+Non standard port with GVisor ```bash curl -p -x http://localhost:8001 \ --proxy-header "ate-target-actor: ate-demo-counter/my-counter-1" \ http://my-counter-1:9090/ ``` +
This reaches the same actor's second listener and resumes it exactly like any other request — the only difference from the default-port examples above is @@ -137,49 +247,6 @@ this way today — if your use case needs that, please open an issue or reach out; the CONNECT tunnel itself is protocol-agnostic, so it's a matter of prioritizing it, not a fundamental limitation of the design. -## Micro-VM variant - -The same in-RAM-counter suspend/resume-continuity demo also runs on the micro-VM -sandbox class (`ateworker-microvm`: a Kata guest on Cloud Hypervisor), proving that -the guest-memory snapshot round-trips just as gVisor's process snapshot does. -The micro-VM class is also the one that serves `SNAPSHOT_FIDELITY_ROOTFS`, a -snapshot of the volumes and each container's rootfs writes without guest -memory; this template keeps `MEMORY` because the in-RAM counter is the point of -the demo. - -- [`demos/counter/counter-microvm.yaml.tmpl`](counter-microvm.yaml.tmpl) — - the `WorkerPool` for the micro-VM sandbox class. -- [`demos/counter/counter-microvm-template.yaml.tmpl`](counter-microvm-template.yaml.tmpl) — - the `counter-microvm` actor template, created through the ate API in the - `ate-demo-counter-microvm` atespace. -- [`hack/run-microvm-demo.sh`](../../hack/run-microvm-demo.sh) — one-shot bring-up - that builds the micro-VM worker image, stages the guest assets, deploys the - control plane, and applies the manifests above. Like the other hack scripts it - reads `.ate-dev-env.sh` for GKE; use the kind wrapper for a local cluster. - -Run it and follow the printed next steps: - -```bash -# GKE (uses .ate-dev-env.sh, uploads assets to GCS): -./hack/run-microvm-demo.sh - -# local kind (local registry + in-cluster rustfs): -KIND_CLUSTER_NAME= ./hack/run-microvm-demo-kind.sh -``` - -On a cluster that already has the micro-VM deps (`hack/install-microvm-deps.sh`), -deploy directly instead: - -```bash -./hack/install-ate.sh --deploy-demo-counter-microvm -``` - -Then create an actor (`--template counter-microvm`, in the -`ate-demo-counter-microvm` atespace), increment the counter, suspend -it, resume it (even on a different worker), and confirm the count continues — -the actor's counter lives in guest RAM, so a continuing count proves the -guest-memory snapshot survived the round trip. - ## Settings The installer reads these settings in addition to those in @@ -195,10 +262,19 @@ flag, environment variable, and configuration document channels. To remove the counter demo resources from your cluster, run: +
+Delete from MicroVM ```bash ./hack/install-ate.sh --delete-demo-counter +``` +
+ +
+Delete from GVisor +```bash ./hack/install-ate.sh --delete-demo-counter-microvm ``` +
This deletes the actors created from the demo templates, the templates themselves (along with their golden actors and golden snapshots, server-side), diff --git a/demos/counter/counter-microvm-csi-test-template.yaml b/demos/counter/counter-microvm-csi-test-template.yaml index 293d0c407d..6d1e7bb280 100644 --- a/demos/counter/counter-microvm-csi-test-template.yaml +++ b/demos/counter/counter-microvm-csi-test-template.yaml @@ -43,9 +43,6 @@ resources: snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_VOLUMES storageLocation: gs://ate-snapshots/ate-demo-counter-microvm-csi/ -sandboxConfig: - sandboxClass: microvm - configName: microvm volumes: - name: data externalVolumeTemplate: diff --git a/demos/counter/counter-microvm-template.yaml.tmpl b/demos/counter/counter-microvm-template.yaml.tmpl index e694e3d11d..c5d7dde843 100644 --- a/demos/counter/counter-microvm-template.yaml.tmpl +++ b/demos/counter/counter-microvm-template.yaml.tmpl @@ -66,9 +66,6 @@ snapshotConfig: # file counter and any rootfs writes but cold-boot the memory counter. preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-counter-microvm/ -sandboxConfig: - sandboxClass: microvm - configName: microvm volumes: - name: data durableDir: {} diff --git a/demos/counter/counter-template.yaml.tmpl b/demos/counter/counter-template.yaml.tmpl index 9aca738597..fabdfe60f3 100644 --- a/demos/counter/counter-template.yaml.tmpl +++ b/demos/counter/counter-template.yaml.tmpl @@ -61,9 +61,6 @@ snapshotConfig: # parameterized templates, not by this one. preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-counter/ -sandboxConfig: - sandboxClass: gvisor - configName: gvisor-default volumes: - name: data durableDir: {} diff --git a/demos/egress/README.md b/demos/egress/README.md index 685cefefd5..9535e699d3 100644 --- a/demos/egress/README.md +++ b/demos/egress/README.md @@ -79,7 +79,8 @@ rejects that option with agentgateway rather than silently omitting it. - **Egress app (`main.go`)** — the Actor: `POST /` fetches an HTTP(S) URL and returns the status, body, and observed protocol. `POST /websocket` probes a `ws` or `wss` origin; `POST /grpc` makes - unary and streaming gRPC calls, described below. + unary and streaming gRPC calls, described below; `POST /connect` sends a forward proxy an HTTP + CONNECT, in the clear or over TLS, and returns its answer. - **Egress gateway** — the `atenet-egress` Deployment. Envoy uses a co-located atenet `ext_proc` container started with `--mode=egress`; agentgateway uses its built-in `substrateEgress` policy and does not need that sidecar. The installer renders the matching configuration and container. diff --git a/demos/egress/egress-microvm-template.yaml.tmpl b/demos/egress/egress-microvm-template.yaml.tmpl index dc0e70a9ff..888ee22b48 100644 --- a/demos/egress/egress-microvm-template.yaml.tmpl +++ b/demos/egress/egress-microvm-template.yaml.tmpl @@ -70,11 +70,6 @@ resources: quantity: "1" - name: memory quantity: 512Mi -sandboxConfig: - sandboxClass: microvm - # Installed cluster-wide by hack/install-microvm-deps.sh; naming it - # explicitly makes a missing or stale install fail loudly. - configName: microvm snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-egress-microvm/ diff --git a/demos/egress/egress-template.yaml.tmpl b/demos/egress/egress-template.yaml.tmpl index 09cf6b9775..23480ead7e 100644 --- a/demos/egress/egress-template.yaml.tmpl +++ b/demos/egress/egress-template.yaml.tmpl @@ -61,9 +61,6 @@ resources: quantity: "1" - name: memory quantity: 512Mi -sandboxConfig: - sandboxClass: gvisor - configName: gvisor-default snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-egress/ diff --git a/demos/egress/main.go b/demos/egress/main.go index 4e32cdedec..8664c6254f 100644 --- a/demos/egress/main.go +++ b/demos/egress/main.go @@ -53,6 +53,7 @@ const ( maxWebSocketMessages = 16 maxWebSocketMessageSize = 64 << 10 maxWebSocketHandshakeBody = 4 << 10 + maxConnectAnswerBody = 4 << 10 muxRequestCount = 250 muxBodySize = 64 // Keep connection around longer than ext_proc cache freshness (10s) @@ -131,6 +132,29 @@ type streamedMessage struct { Index int32 `json:"index"` } +// connectRequest parameters for an HTTP CONNECT. +type connectRequest struct { + // Proxy is the host:port the actor dials and sends the CONNECT to. + Proxy string `json:"proxy"` + // Target is the host:port named in the CONNECT request line. + Target string `json:"target"` + // TLS wraps the connection to Proxy in TLS before the CONNECT goes out, + // which is how a client talks to an HTTPS proxy. The proxy's certificate + // is verified against RootCA when set, else the actor's roots. + TLS bool `json:"tls,omitempty"` + RootCA string `json:"rootCA,omitempty"` +} + +// connectResponse is the proxy's answer to the CONNECT. +type connectResponse struct { + StatusCode int `json:"statusCode,omitempty"` + Protocol string `json:"protocol,omitempty"` + Body string `json:"body,omitempty"` + // TLS is true if the CONNECT was sent over TLS. + TLS bool `json:"tls"` + Error string `json:"error,omitempty"` +} + func main() { slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, nil))) @@ -210,6 +234,7 @@ func newHandler(client *http.Client) http.Handler { mux.HandleFunc("/mux", handleMux) mux.HandleFunc("/websocket", handleWebSocket) mux.HandleFunc("/grpc", handleGRPC) + mux.HandleFunc("/connect", handleConnect) return mux } @@ -741,7 +766,7 @@ func handleGRPC(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusBadRequest, grpcResponse{Error: fmt.Sprintf("invalid JSON payload: %v", err)}) return } - if err := validateTarget(input.Target); err != nil { + if err := validateTarget(input.Target, "target"); err != nil { writeJSON(w, http.StatusBadRequest, grpcResponse{Error: err.Error()}) return } @@ -862,6 +887,107 @@ func writeGRPCFailure(w http.ResponseWriter, rpc string, err error) { }) } +// handleConnect sends one HTTP CONNECT, the request a client makes of a +// forward proxy, and reports the answer. +func handleConnect(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.Header().Set("Allow", http.MethodPost) + writeJSON(w, http.StatusMethodNotAllowed, connectResponse{Error: "method must be POST"}) + return + } + + var input connectRequest + decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxRequestBody)) + if err := decoder.Decode(&input); err != nil { + writeJSON(w, http.StatusBadRequest, connectResponse{Error: fmt.Sprintf("invalid JSON payload: %v", err)}) + return + } + if err := validateTarget(input.Proxy, "proxy"); err != nil { + writeJSON(w, http.StatusBadRequest, connectResponse{Error: err.Error()}) + return + } + if err := validateTarget(input.Target, "target"); err != nil { + writeJSON(w, http.StatusBadRequest, connectResponse{Error: err.Error()}) + return + } + // Built before anything is dialed, so a bad rootCA is the caller's + // mistake and no connection is opened for it. + var tlsCfg *tls.Config + if input.TLS { + host, _, _ := net.SplitHostPort(input.Proxy) + tlsCfg = &tls.Config{ServerName: host, MinVersion: tls.VersionTLS12} + if input.RootCA != "" { + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM([]byte(input.RootCA)) { + writeJSON(w, http.StatusBadRequest, connectResponse{Error: "rootCA does not contain a valid certificate"}) + return + } + tlsCfg.RootCAs = roots + } + } + + ctx, cancel := context.WithTimeout(r.Context(), requestTimeout) + defer cancel() + response, err := performConnect(ctx, input, tlsCfg) + if err != nil { + response.Error = err.Error() + writeJSON(w, http.StatusBadGateway, response) + return + } + writeJSON(w, http.StatusOK, response) +} + +// performConnect dials the proxy, completes a TLS handshake with it when +// tlsCfg is set, sends the CONNECT and reads its answer. +func performConnect(ctx context.Context, input connectRequest, tlsCfg *tls.Config) (connectResponse, error) { + var response connectResponse + var dialer net.Dialer + conn, err := dialer.DialContext(ctx, "tcp", input.Proxy) + if err != nil { + return response, fmt.Errorf("dialing proxy %s: %w", input.Proxy, err) + } + // Through the variable, so the TLS connection that may replace it below + // is the one closed and gets to send its close_notify. + defer func() { _ = conn.Close() }() + if deadline, ok := ctx.Deadline(); ok { + if err := conn.SetDeadline(deadline); err != nil { + return response, fmt.Errorf("setting deadline: %w", err) + } + } + + if tlsCfg != nil { + tlsConn := tls.Client(conn, tlsCfg) + if err := tlsConn.HandshakeContext(ctx); err != nil { + return response, fmt.Errorf("TLS handshake with proxy %s: %w", input.Proxy, err) + } + response.TLS = tlsConn.ConnectionState().VerifiedChains != nil + conn = tlsConn + } + + request := fmt.Sprintf("CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n", input.Target, input.Target) + if _, err := io.WriteString(conn, request); err != nil { + return response, fmt.Errorf("sending CONNECT %s: %w", input.Target, err) + } + answer, err := http.ReadResponse(bufio.NewReader(conn), &http.Request{Method: http.MethodConnect}) + if err != nil { + return response, fmt.Errorf("reading the answer to CONNECT %s: %w", input.Target, err) + } + defer answer.Body.Close() + response.StatusCode = answer.StatusCode + response.Protocol = answer.Proto + // A 2xx turns the connection into the tunnel, so there is no body to + // read, and reading one would wait on bytes the proxy will never send. + if answer.StatusCode/100 == 2 { + return response, nil + } + body, err := io.ReadAll(io.LimitReader(answer.Body, maxConnectAnswerBody)) + if err != nil { + return response, fmt.Errorf("reading the body of the answer to CONNECT %s: %w", input.Target, err) + } + response.Body = string(body) + return response, nil +} + func validateURL(raw string) error { parsed, err := url.Parse(raw) if err != nil { @@ -876,21 +1002,23 @@ func validateURL(raw string) error { return nil } -// validateTarget checks that raw is the host:port a gRPC dial needs. Unlike a -// URL there is no scheme to reject, so a caller that passes one -- or passes a -// bare hostname and lets the dial default the port -- finds out here rather -// than in a connection error from somewhere along the egress path. -func validateTarget(raw string) error { +// validateTarget checks that raw is a host:port, which is what a gRPC dial, a +// forward proxy, and a CONNECT's authority all take. Unlike a URL there is no +// scheme to reject, so a caller that passes one -- or passes a bare hostname +// and lets the dial default the port -- finds out here rather than in a +// connection error from somewhere along the egress path. what names the +// field in the error. +func validateTarget(raw, what string) error { host, port, err := net.SplitHostPort(raw) if err != nil { - return fmt.Errorf("target must be host:port: %w", err) + return fmt.Errorf("%s must be host:port: %w", what, err) } if host == "" { - return fmt.Errorf("target must include a host") + return fmt.Errorf("%s must include a host", what) } number, err := strconv.Atoi(port) if err != nil || number < 1 || number > 65535 { - return fmt.Errorf("target port must be a number between 1 and 65535, got %q", port) + return fmt.Errorf("%s port must be a number between 1 and 65535, got %q", what, port) } return nil } diff --git a/demos/egress/main_test.go b/demos/egress/main_test.go index 80778bbcdc..c6760f6c54 100644 --- a/demos/egress/main_test.go +++ b/demos/egress/main_test.go @@ -15,6 +15,7 @@ package main import ( + "bufio" "bytes" "context" "crypto/tls" @@ -25,7 +26,9 @@ import ( "net" "net/http" "net/http/httptest" + "slices" "strings" + "sync" "sync/atomic" "testing" "time" @@ -850,3 +853,178 @@ func TestSlowHeadersInvalidRequests(t *testing.T) { } }) } + +// startProxyStub listens on loopback and answers the first CONNECT on each +// connection with answer, verbatim, then holds the connection open until the +// test ends -- as a proxy that opened the tunnel would. It returns its +// host:port and a function listing the request lines it received so far. +func startProxyStub(t *testing.T, answer string) (string, func() []string) { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listening on loopback: %v", err) + } + var mu sync.Mutex + var received []string + done := make(chan struct{}) + t.Cleanup(func() { listener.Close(); close(done) }) + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + go func() { + defer conn.Close() + request, err := http.ReadRequest(bufio.NewReader(conn)) + if err != nil { + return + } + mu.Lock() + received = append(received, request.Method+" "+request.Host) + mu.Unlock() + _, _ = io.WriteString(conn, answer) + <-done + }() + } + }() + return listener.Addr().String(), func() []string { + mu.Lock() + defer mu.Unlock() + return slices.Clone(received) + } +} + +func TestConnectReportsRefusal(t *testing.T) { + proxy, received := startProxyStub(t, "HTTP/1.1 403 Forbidden\r\nContent-Length: 6\r\n\r\ndenied") + recorder := postHandler(t, newHandler(http.DefaultClient), "/connect", connectRequest{Proxy: proxy, Target: "example.com:443"}) + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusOK, recorder.Body.String()) + } + var got connectResponse + decodeRecorder(t, recorder, &got) + if got.StatusCode != http.StatusForbidden || got.Body != "denied" || got.Protocol != "HTTP/1.1" || got.TLS || got.Error != "" { + t.Errorf("response = %+v, want the proxy's 403 and body in the clear", got) + } + if got, want := received(), []string{"CONNECT example.com:443"}; !slices.Equal(got, want) { + t.Errorf("proxy received %q, want %q", got, want) + } +} + +// A 2xx has no body: the connection is the tunnel now, and the proxy stub +// sends nothing further, so a handler that tried to read one would hang. +func TestConnectReportsOpenedTunnelWithoutWaitingForBody(t *testing.T) { + proxy, _ := startProxyStub(t, "HTTP/1.1 200 Connection established\r\n\r\n") + start := time.Now() + recorder := postHandler(t, newHandler(http.DefaultClient), "/connect", connectRequest{Proxy: proxy, Target: "example.com:443"}) + if elapsed := time.Since(start); elapsed > requestTimeout/2 { + t.Errorf("answering a 200 took %v; the handler waited on a body the tunnel never sends", elapsed) + } + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusOK, recorder.Body.String()) + } + var got connectResponse + decodeRecorder(t, recorder, &got) + if got.StatusCode != http.StatusOK || got.Body != "" || got.Error != "" { + t.Errorf("response = %+v, want an opened tunnel with no body", got) + } +} + +func TestConnectOverTLS(t *testing.T) { + server, rootCA := startIPTLSServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodConnect { + http.Error(w, "not a CONNECT", http.StatusBadRequest) + return + } + http.Error(w, "no tunnels here", http.StatusForbidden) + })) + proxy := strings.TrimPrefix(server.URL, "https://") + + recorder := postHandler(t, newHandler(http.DefaultClient), "/connect", connectRequest{Proxy: proxy, Target: "example.com:443", TLS: true, RootCA: rootCA}) + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusOK, recorder.Body.String()) + } + var got connectResponse + decodeRecorder(t, recorder, &got) + if got.StatusCode != http.StatusForbidden || !got.TLS || strings.TrimSpace(got.Body) != "no tunnels here" || got.Error != "" { + t.Errorf("response = %+v, want the proxy's 403 over verified TLS", got) + } + + // Without the root the handshake fails, and that is a failure to get an + // answer, not an answer. + recorder = postHandler(t, newHandler(http.DefaultClient), "/connect", connectRequest{Proxy: proxy, Target: "example.com:443", TLS: true}) + if recorder.Code != http.StatusBadGateway { + t.Fatalf("status without a root = %d, want %d; body = %s", recorder.Code, http.StatusBadGateway, recorder.Body.String()) + } + var unverified connectResponse + decodeRecorder(t, recorder, &unverified) + if unverified.StatusCode != 0 || !strings.Contains(unverified.Error, "TLS handshake") { + t.Errorf("response without a root = %+v, want a TLS handshake error and no status", unverified) + } +} + +// A rootCA that is not a certificate is refused before anything is dialed: +// the proxy here refuses connections, so a handler that dialed first would +// report that instead. +func TestConnectRejectsBadRootCABeforeDialing(t *testing.T) { + closed, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listening on loopback: %v", err) + } + proxy := closed.Addr().String() + closed.Close() + + recorder := postHandler(t, newHandler(http.DefaultClient), "/connect", connectRequest{Proxy: proxy, Target: "example.com:443", TLS: true, RootCA: "not-a-pem"}) + if recorder.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusBadRequest, recorder.Body.String()) + } + var got connectResponse + decodeRecorder(t, recorder, &got) + if !strings.Contains(got.Error, "rootCA") || strings.Contains(got.Error, "dialing") { + t.Errorf("error = %q, want the rootCA rejection and no dial", got.Error) + } +} + +func TestConnectReportsUnreachableProxy(t *testing.T) { + // A port that was just listening and is now closed refuses the connection. + closed, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listening on loopback: %v", err) + } + proxy := closed.Addr().String() + closed.Close() + + recorder := postHandler(t, newHandler(http.DefaultClient), "/connect", connectRequest{Proxy: proxy, Target: "example.com:443"}) + if recorder.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusBadGateway, recorder.Body.String()) + } + var got connectResponse + decodeRecorder(t, recorder, &got) + if !strings.Contains(got.Error, "dialing proxy "+proxy) { + t.Errorf("error = %q, want the dial failure", got.Error) + } +} + +func TestConnectRejectsInvalidRequests(t *testing.T) { + handler := newHandler(http.DefaultClient) + for name, tc := range map[string]struct { + method string + body string + want int + }{ + "GET": {http.MethodGet, `{"proxy":"127.0.0.1:3128","target":"example.com:443"}`, http.StatusMethodNotAllowed}, + "not JSON": {http.MethodPost, `proxy`, http.StatusBadRequest}, + "no proxy": {http.MethodPost, `{"target":"example.com:443"}`, http.StatusBadRequest}, + "proxy without port": {http.MethodPost, `{"proxy":"127.0.0.1","target":"example.com:443"}`, http.StatusBadRequest}, + "no target": {http.MethodPost, `{"proxy":"127.0.0.1:3128"}`, http.StatusBadRequest}, + "target with scheme": {http.MethodPost, `{"proxy":"127.0.0.1:3128","target":"https://example.com:443"}`, http.StatusBadRequest}, + } { + t.Run(name, func(t *testing.T) { + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest(tc.method, "/connect", strings.NewReader(tc.body))) + if recorder.Code != tc.want { + t.Errorf("status = %d, want %d; body = %s", recorder.Code, tc.want, recorder.Body.String()) + } + }) + } +} diff --git a/demos/jupyter/jupyter-template.yaml.tmpl b/demos/jupyter/jupyter-template.yaml.tmpl index f7edabde18..452d6a7b2c 100644 --- a/demos/jupyter/jupyter-template.yaml.tmpl +++ b/demos/jupyter/jupyter-template.yaml.tmpl @@ -50,6 +50,3 @@ resources: snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-jupyter/ -sandboxConfig: - sandboxClass: gvisor - configName: gvisor-default diff --git a/demos/multi-template/counter-template.yaml.tmpl b/demos/multi-template/counter-template.yaml.tmpl index a84a2469fe..ea1815b247 100644 --- a/demos/multi-template/counter-template.yaml.tmpl +++ b/demos/multi-template/counter-template.yaml.tmpl @@ -38,6 +38,3 @@ resources: snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-multi-template-counter/ -sandboxConfig: - sandboxClass: gvisor - configName: gvisor-default diff --git a/demos/multi-template/fspersist-template.yaml.tmpl b/demos/multi-template/fspersist-template.yaml.tmpl index 9ceb6881b2..4215abe360 100644 --- a/demos/multi-template/fspersist-template.yaml.tmpl +++ b/demos/multi-template/fspersist-template.yaml.tmpl @@ -38,6 +38,3 @@ resources: snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-multi-template-fspersist/ -sandboxConfig: - sandboxClass: gvisor - configName: gvisor-default diff --git a/demos/parking/parking-template.yaml.tmpl b/demos/parking/parking-template.yaml.tmpl index eac6c76796..80c9b67f32 100644 --- a/demos/parking/parking-template.yaml.tmpl +++ b/demos/parking/parking-template.yaml.tmpl @@ -37,6 +37,3 @@ resources: snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-parking/ -sandboxConfig: - sandboxClass: gvisor - configName: gvisor-default diff --git a/demos/sandbox/manual-test-multi-template.yaml b/demos/sandbox/manual-test-multi-template.yaml index 3811c37684..1e35e877ba 100644 --- a/demos/sandbox/manual-test-multi-template.yaml +++ b/demos/sandbox/manual-test-multi-template.yaml @@ -45,9 +45,6 @@ resources: snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://ate-snapshots/ate-manual-test-multi/ -sandboxConfig: - sandboxClass: gvisor - configName: gvisor-default volumes: - name: vol-hostpath externalVolumeTemplate: diff --git a/demos/sandbox/sandbox-template.yaml.tmpl b/demos/sandbox/sandbox-template.yaml.tmpl index c638376bb7..87ad72eb58 100644 --- a/demos/sandbox/sandbox-template.yaml.tmpl +++ b/demos/sandbox/sandbox-template.yaml.tmpl @@ -40,6 +40,3 @@ resources: snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-sandbox/ -sandboxConfig: - sandboxClass: gvisor - configName: gvisor-default diff --git a/docs/api-guide.md b/docs/api-guide.md index 315f127537..d8c3070740 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -12,7 +12,7 @@ The `WorkerPool` defines the pool of physical "warm" compute capacity. It manage | :--- | :--- | :--- | | `replicas` | `int32` | **Required.** Number of physical standby pods to maintain in the cluster. | | `workerImage` | `string` | **Required.** The container image for the `ateworker` herder process (e.g. `ko://go.ate.dev/substrate/cmd/ateworker-gvisor`). | -| `sandboxClasses` | `[]WorkerPoolSandboxClass` | **Required.** The sandbox runtime families the pool runs; exactly one entry today. Each entry has `name` (**required**, `gvisor` or `microvm`, immutable), which drives the worker pod shape (e.g. KVM device mounts, node placement), and `configRef.name` (optional), which names a cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) of the same class. Without `configRef`, the pool uses the class default (see [Class default](#class-default)). The controller records the result in `status.sandboxClasses`. The sandbox binaries an actor boots with still come from the `SandboxConfig` its `ActorTemplate` selects. | +| `sandboxClasses` | `[]WorkerPoolSandboxClass` | **Required.** The sandbox runtime families the pool runs; exactly one entry today. Each entry has `name` (**required**, `gvisor` or `microvm`, immutable), which drives the worker pod shape (e.g. KVM device mounts, node placement), and `configRef.name` (optional), which names a cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) of the same class. Without `configRef`, the pool uses the class default (see [Class default](#class-default)). The controller records the result in `status.sandboxClasses`. The sandbox binaries an actor boots with come from the `SandboxConfig` the pool of its assigned worker resolved. | | `template` | `WorkerPoolPodTemplate` | **Optional.** Metadata, scheduling, and resource settings for worker workloads. | #### `WorkerPoolPodTemplate` (`spec.template`) @@ -99,15 +99,12 @@ The `ActorTemplate` defines the code, environment, and state-management policies | Field | Type | Description | | :--- | :--- | :--- | | `containers` | `[]Container` | **Required.** The workload definition — see [Container Fields](#container-fields) below. Each container may also declare an optional `wakeupProbe` HTTP probe — see [Container Wakeup Probe](#container-wakeup-probe-wakeupprobe). | -| `sandboxConfig` | `SandboxConfig` | **Required.** The sandbox runtime selection: `sandboxClass` (**required**, `gvisor` or `microvm`) picks the runtime family this template's actors require — only `WorkerPool`s whose `sandboxClasses` entry matches are eligible — and `configName` (**required**) names the cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) object supplying the sandbox binaries. It must reference an existing config of the matching class; `CreateActorTemplate` rejects the template otherwise. | -| `workerSelector` | `*Selector` | Optional. Gates which `WorkerPool`s actors from this template may use, by matching against each pool's labels (`matchLabels`). If unset, all pools are eligible (subject to the actor's own `worker_selector`). | -| `snapshotConfig` | `SnapshotConfig` | **Required.** The base object-storage location snapshots are written under, the snapshot fidelity (`preferredFidelity`) every Pause and Suspend captures (`SNAPSHOT_FIDELITY_ROOTFS` requires `sandboxClass: microvm`), and automated golden snapshot creation (`goldenSnapshotConfig.mode`). See [Snapshot Storage Layout](#snapshot-storage-layout). | -| `volumes` | `[]Volume` | Optional. Volumes the containers may mount, each a `durableDir`, an `externalVolumeTemplate` (see [CSI Volumes Guide](csi-volumes.md)), or a `systemInfo` volume (see [SystemInfo Volumes](#systeminfo-volumes)). Every declared volume must be mounted by at least one container. A `microvm` template may declare several `durableDir` volumes; a `gvisor` template is limited to one. A `durableDir` is created `0777`, as Kubernetes creates an emptyDir, so a container running as any user can write it. A snapshot restores it with the mode it had, so the container's own chmod survives suspend and resume. | +| `workerSelector` | `*Selector` | Optional. Gates which `WorkerPool`s actors from this template may use, by matching against each pool's labels (`matchLabels`). If unset, all pools are eligible (subject to the actor's own `worker_selector`). The pool decides the sandbox class, so this is also how a template pins one: if your cluster has pools of more than one sandbox class and the workload needs a particular class, make sure the selector points only at pools of that class. An actor booting with no snapshot and no selector may land on any class, and keeps it from its first checkpoint on. | +| `snapshotConfig` | `SnapshotConfig` | **Required.** The base object-storage location snapshots are written under, the snapshot fidelity (`preferredFidelity`) every Pause and Suspend captures, and automated golden snapshot creation (`goldenSnapshotConfig.mode`). See [Snapshot Storage Layout](#snapshot-storage-layout). | +| `volumes` | `[]Volume` | Optional. Volumes the containers may mount, each a `durableDir`, an `externalVolumeTemplate` (see [CSI Volumes Guide](csi-volumes.md)), or a `systemInfo` volume (see [SystemInfo Volumes](#systeminfo-volumes)). Every declared volume must be mounted by at least one container. A `durableDir` is created `0777`, as Kubernetes creates an emptyDir, so a container running as any user can write it. A snapshot restores it with the mode it had, so the container's own chmod survives suspend and resume. | | `resources` | `*ResourceRequirements` | Required. Declares each actor's compute size via `limits`, which must set both `cpu` and `memory` — see [Sandbox Right-Sizing](#sandbox-right-sizing-resources). Immutable, like the rest of the template. | -The sandbox itself — the binaries (e.g. the gVisor `runsc` binary) and the `pauseImage` holding the sandbox's namespaces — comes from the cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) object the template names via `sandboxConfig.configName`. An actor always resolves the config from its current template — repointing the actor at another template requires the same config. - -Because a snapshot is not restorable across sandbox runtimes, `sandboxClass` is a **hard scheduling gate**: an actor is only ever placed on a `WorkerPool` whose `sandboxClasses` entry matches. It is AND'd with `workerSelector` (and the actor's `worker_selector`), which can only narrow the eligible pools further. It has no default — `sandboxConfig` is required and its `sandboxClass` must be set — and, like the rest of the template, is immutable, so each template's class is fixed at creation. +The sandbox runtime and its binaries (e.g. the gVisor `runsc` binary and `pauseImage`, or a micro-VM kernel/firmware/config) are configured on the [`WorkerPool`](#1-workerpool-the-compute-foundation) via its `sandboxClasses` and the cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) it resolves, rather than on the `ActorTemplate`. An `ActorTemplate` selects its `WorkerPool`s via `workerSelector` (AND'd with the actor's `worker_selector`). When an actor runs on a worker, the worker's active `SandboxRuntime` is stamped onto the actor's assignment and snapshots so subsequent resumes prefer runtime-compatible workers. ### Sandbox Right-Sizing (`resources`) @@ -263,10 +260,9 @@ The sandbox — gVisor or micro-VM — remains the isolation boundary; capabilit ### Per-container limits -A container may cap its own CPU and memory so it cannot starve or kill its siblings in the same actor: +A container may cap its own CPU and memory so it cannot starve or kill its siblings in the same actor (on a template whose `workerSelector` targets a micro-VM pool): ```yaml -sandboxClass: microvm containers: - name: trainer resources: @@ -278,7 +274,7 @@ containers: A container that exceeds its memory limit is OOM-killed on its own; the actor's other containers are unaffected. A `cpu` limit below `10m` is raised to `10m`, because the kernel rejects a CFS quota under 1ms. -Per-container limits are micro-VM only today. gVisor applies cgroup limits at the sandbox level: one sentry backs every container in the actor, so a per-container cgroup is created and then stays empty ([google/gvisor#190](https://github.com/google/gvisor/issues/190)). A template that sets `resources` with `sandboxClass: gvisor` is rejected. +Per-container limits are micro-VM only today. gVisor applies cgroup limits at the sandbox level: one sentry backs every container in the actor, so a per-container cgroup is created and then stays empty ([google/gvisor#190](https://github.com/google/gvisor/issues/190)). On a gVisor worker the per-container limits are accepted but not enforced; only the sandbox-level size applies. These limits subdivide the sandbox that [`resources`](#sandbox-right-sizing-resources) already sized; a container that declares none is bounded by the guest as a whole, not by a copy of the actor's total. A micro-VM guest's memory is `resources.limits.memory` minus the VMM reserve, and its vCPU count is `resources.limits.cpu` rounded up to whole cores. A container limit above either ceiling can never bind, so the actor fails to start with an error naming both the limit and the ceiling. @@ -326,13 +322,6 @@ containers: workerSelector: matchLabels: workload: secret-agent -# sandboxClass (required) picks the runtime family (set microvm to require -# micro-VM pools); configName (required) names the cluster-scoped -# SandboxConfig supplying the sandbox binaries (see section 3). -# gvisor-default is the SandboxConfig that manifests/ate-install ships. -sandboxConfig: - sandboxClass: gvisor - configName: gvisor-default # The actor's size, booked against worker capacity. resources: limits: @@ -353,7 +342,7 @@ snapshotConfig: /atespaces//tags/ ``` -The objects of a snapshot (its manifest, memory image, durable-data tar) are named below it. So for the template above, a snapshot of an actor in atespace `team-a` is stored at `gs://my-bucket/secret-agent/atespaces/team-a/actors/3f8b…/snapshots/f47ac10b-…`, and the template's golden snapshot — the golden tag lives in the reserved `ate-golden` atespace — under `gs://my-bucket/secret-agent/atespaces/ate-golden/tags/`. +The objects of a snapshot (its manifest, memory image, durable-data tar) are named below it. So for the template above, a snapshot of an actor in atespace `team-a` is stored at `gs://my-bucket/secret-agent/atespaces/team-a/actors/3f8b…/snapshots/f47ac10b-…`, and the template's golden snapshot under `gs://my-bucket/secret-agent/atespaces/team-a/tags/`. An actor takes a series of snapshots over its life, so it gets a prefix of its own and each snapshot sits below it. A tag holds exactly one, so the tag's @@ -385,16 +374,17 @@ a `Snapshot` with the following fields: sandbox image cannot be resumed under another. - `locality`: The node that holds the snapshot's local files. It is set when a pause completes and cleared once those files are deleted from the node (when - the actor is next paused or suspended, or is reverted). A resume restores from - the files without deleting them, so an actor resumed from pause still reports - where they are. A paused actor is an `ACTOR_STATE_SUSPENDED` actor whose - latest snapshot has `locality` and no `durableSnapshot`; it is pinned to that - node (`status.assignedNode`) until it is suspended or reverted. + atelet detaches the actor in the background, the actor is next paused or + suspended, or it is reverted). A resume restores from the files without + deleting them, so an actor resumed from pause still reports where they are. A + paused actor is an `ACTOR_STATE_SUSPENDED` actor whose latest snapshot has + `locality`; it is pinned to that node (`status.assignedNode`) until atelet + detaches it or it is reverted. - `durableSnapshot`: The copy in object storage, with its URI in `object.snapshotUri` and a `status` that is `PENDING` while the copy is being - written and `COMPLETED` once it can be restored from. A snapshot that exists - only on a node has no `durableSnapshot`. A tag's snapshot always has one, and - the tag can seed actors only once it is `COMPLETED`. + written and `COMPLETED` once it can be restored from. A pause records the copy + as `PENDING`, and atelet uploads it in the background. A tag's snapshot always + has one, and the tag can seed actors only once it is `COMPLETED`. - `sandboxRuntime`: The sandbox runtime and hardware compatibility attributes the guest state was captured under, copied from `status.workerAssignment.sandboxRuntime` when the checkpoint is taken. A resume that restores memory or rootfs state @@ -411,11 +401,11 @@ a `Snapshot` with the following fields: worker and downgrades the restore to `SNAPSHOT_FIDELITY_VOLUMES`, preserving the snapshot's volume data while booting the guest fresh under the new worker's `defaultRuntime`. - - A local pause checkpoint (with `locality` and no `durableSnapshot`) is not - relaxed; the resume returns `RESOURCE_EXHAUSTED` and the actor remains - `PAUSED` on its node until a compatible worker there has room. Suspending - the actor uploads the checkpoint and lifts the node pin, after which its - next resume may fall back like a durable snapshot. + - A local pause checkpoint (with `locality`) is not relaxed; the resume + returns `RESOURCE_EXHAUSTED` and the actor stays paused on its node until a + compatible worker there has room, or until atelet detaches it. A detach + uploads the checkpoint, deletes the local files, and lifts the node pin, + after which the next resume may fall back like a durable snapshot. A snapshot without `sandboxRuntime` or with `SNAPSHOT_FIDELITY_VOLUMES` imposes no runtime placement constraint. @@ -424,7 +414,7 @@ An `ActorTemplate` references its golden tag with the `ObjectRef` in `status.goldenSnapshotStatus.goldenTag`. These status fields are server-owned and ignored on input. Parse a URI only against the scheme above. -An `ActorTemplate` belongs to one atespace, but one `storageLocation` still holds snapshots for many atespaces: the golden actor lives in the reserved `ate-golden` atespace, and a `PUBLISHED` snapshot may be cloned from other atespaces. The `` level exists so that access can be granted per tenant: an object-storage policy can only condition on an **object-name prefix**, and cannot read the identity recorded inside a snapshot's manifest. Binding a per-atespace grant on GCS looks like: +An `ActorTemplate` belongs to one atespace, but one `storageLocation` still holds snapshots for many atespaces: the golden actor and tag live in the template's atespace, and a `PUBLISHED` snapshot may be cloned from other atespaces. The `` level exists so that access can be granted per tenant: an object-storage policy can only condition on an **object-name prefix**, and cannot read the identity recorded inside a snapshot's manifest. Binding a per-atespace grant on GCS looks like: ```yaml # Read-only on team-a's snapshots for this template, and nothing else. @@ -443,15 +433,13 @@ One consequence worth planning for: **a published snapshot is read from the ates ## 3. SandboxConfig: The Sandbox Itself -`SandboxConfig` is a **cluster-scoped** resource that decouples the sandbox — its binaries (the gVisor `runsc` binary, or a micro-VM kernel/firmware/config) and the `pauseImage` that holds the sandbox's namespaces — from the workload definition in the `ActorTemplate`. An actor's cold boot resolves the sandbox binaries from the default version of the config its `ActorTemplate` names via `sandboxConfig.configName`. - -This means a single, cluster-managed config pins the sandbox runtime version for many templates, and operators upgrade the runtime in one place. A resume also resolves the default version at restore time: the snapshot manifest records the assets it was captured with, but a restore runs the current default, so changing `defaultVersion` applies to every existing snapshot on its next resume. gVisor does not promise that a checkpoint restores under a different release, so after a version change recut the golden snapshots and expect snapshots captured under the old version to fail on their next resume. +`SandboxConfig` is a **cluster-scoped** resource that decouples the sandbox — its binaries (the gVisor `runsc` binary, or a micro-VM kernel/firmware/config) and the `pauseImage` that holds the sandbox's namespaces — from the workload definition in the `ActorTemplate`. A `WorkerPool`'s `sandboxClasses` entry resolves its `SandboxConfig` (either by explicit `configRef` or via the class default), and `atelet` provisions each worker pod with the enabled versions of that config, using `defaultVersion` for cold boots and keeping other enabled versions in `restorableRuntimes` so existing snapshots can still restore across upgrades. ### Specification (`SandboxConfigSpec`) | Field | Type | Description | | :--- | :--- | :--- | -| `sandboxClass` | `string` | Optional. Runtime family this config applies to: `gvisor` (default) or `microvm`. An `ActorTemplate` only uses `SandboxConfig`s whose `sandboxClass` matches its own. | +| `sandboxClass` | `string` | Optional. Runtime family this config applies to: `gvisor` (default) or `microvm`. A `WorkerPool` only uses `SandboxConfig`s whose `sandboxClass` matches its entry's `name`. | | `defaultVersion` | `string` | **Required.** Name of the entry in `versions` that actors boot with. It must name an existing version that is not `Disabled`. | | `versions` | `[]SandboxVersionConfig` | **Required.** 1–16 sandbox versions, unique by `name`. | @@ -464,7 +452,7 @@ Each entry of `versions` (`SandboxVersionConfig`): | `pauseImage` | `string` | **Required for `gvisor`; not allowed for `microvm`**, which runs no pause container. The image for the sandbox's root container (e.g. `registry.k8s.io/pause`, or `gcr.io/gke-release/pause` on GKE). Must include a digest (`...@sha256:...`) — it is recorded in each snapshot's manifest so a restore rebuilds the sandbox from the same image. | | `assets` | `map[arch]map[name]AssetFile` | Content-addressed files atelet fetches, keyed by architecture (`amd64`, `arm64`) then asset name. gVisor expects a `gvisor` asset (the release's `gvisor.tar.zstd`), which atelet auto-extracts. A micro-VM backend expects several. Each `AssetFile` is a `{ url, sha256 }` pair; `url` is at most 2048 bytes. A `ValidatingAdmissionPolicy` enforces each class's required assets on every version. atelet holds no storage credentials: it fetches every URL through the node's object-store plugin (`AssetProvider`), with the plugin's credentials. Which schemes work depends on the plugin; the in-tree one serves only its storage backend's scheme, `gs://` on GCS and `s3://` on S3, as `/`, and rejects other schemes without reading them. A `gs://` URL the plugin cannot serve (it has no such object, rejects the URL, does not implement `AssetProvider`, is unreachable, or fails with an error it does not recognize, such as GCS refusing its credentials for a public bucket) is then read anonymously, which serves public buckets such as `gs://gvisor`; any other refusal by the plugin is final. The plugin returns an object only if its content matches `sha256`, and refuses any URL under a snapshot location (an `ActorTemplate`'s `snapshotConfig.location` followed by `/atespaces/`), so assets may share the snapshot bucket but must not be stored there. atelet verifies the sha256 again before using an asset. Its logs and errors show an asset URL as `scheme://host/path`. | -A cluster-wide gVisor `SandboxConfig` (`gvisor-default`) is installed with the platform, so gVisor templates can name it via `sandboxConfig.configName` without any extra setup. +A cluster-wide gVisor `SandboxConfig` (`gvisor-default`, annotated as the class default) is installed with the platform, so gVisor `WorkerPool`s use it automatically without any extra setup. ### Class default @@ -501,25 +489,25 @@ metadata: sandboxconfig.ate.dev/is-class-default: "true" spec: sandboxClass: gvisor - defaultVersion: v3 + defaultVersion: v4 versions: - - name: v3 + - name: v4 state: Enabled pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" assets: amd64: gvisor: - url: "gs://gvisor/releases/nightly/2026-10-10/x86_64/gvisor.tar.zstd" - sha256: "4ae7fab988c3647d2a835d6c3c70c6a99021c562db97827592562b6c51c3e88c" + url: "gs://gvisor/releases/nightly/2026-10-17/x86_64/gvisor.tar.zstd" + sha256: "fa72f0d45756c69a30beec21222c78241b0d330c36f4f2185648617cc23415c8" arm64: gvisor: - url: "gs://gvisor/releases/nightly/2026-10-10/aarch64/gvisor.tar.zstd" - sha256: "aaf414f06eea7960238e43165c691f0a36cefc0593fb141e011e96211c7b6f57" + url: "gs://gvisor/releases/nightly/2026-10-17/aarch64/gvisor.tar.zstd" + sha256: "238634978d99f0dd3e631c4f88eae7b19fb36937065428467f0b6d6b10697689" ``` ### Micro-VM SandboxConfig -A `microvm` `SandboxConfig` supplies the [Kata Containers](https://katacontainers.io/) + [Cloud Hypervisor](https://www.cloudhypervisor.org/) toolchain instead of `runsc`. Each architecture of each version must define the full asset set — `cloud-hypervisor`, `virtiofsd`, `kata-kernel`, and `kata-image` — which a `ValidatingAdmissionPolicy` enforces at apply time. Worker pods for a micro-VM pool require `/dev/kvm` and nested-virtualization-capable nodes. The controller requests those devices on the pod automatically, and atelet advertises them only where they exist, so placement follows the hardware rather than a node label. Clusters that reserve nested-virt nodes with an `ate.dev/sandboxClass=microvm` taint are still tolerated: advertising a device attracts these pods to capable nodes but repels nothing else from them. The same convention applies to every class: worker pods of a pool tolerate `ate.dev/sandboxClass=:NoSchedule`, so a cluster can reserve a node pool per sandbox class with that taint, and the atelet DaemonSet tolerates the key for any value. +A `microvm` `SandboxConfig` supplies the [Kata Containers](https://katacontainers.io/) + [Cloud Hypervisor](https://www.cloudhypervisor.org/) toolchain instead of `runsc`. Each architecture of each version must define the full asset set — `cloud-hypervisor`, `virtiofsd`, `kata-kernel`, and `kata-image` — which a `ValidatingAdmissionPolicy` enforces at apply time. Worker pods for a micro-VM pool require `/dev/kvm` and nested-virtualization-capable nodes. The controller requests those devices on the pod automatically, and atelet advertises them only where they exist, so placement follows the hardware rather than a node label. Clusters that reserve nested-virt nodes with an `ate.dev/sandboxClass=microvm` taint are still tolerated: advertising a device attracts these pods to capable nodes but repels nothing else from them. The same convention applies to every class: worker pods of a pool tolerate `ate.dev/sandboxClass=:NoSchedule`, so a cluster can reserve a node pool per sandbox class with that taint, and the atelet DaemonSet tolerates the key for any value. On GKE, a pool can also select a ComputeClass with `template.nodeSelector` (`cloud.google.com/compute-class: `): GKE taints the nodes it creates for the class `cloud.google.com/compute-class=:NoSchedule` and adds the matching toleration to the worker pods, and the atelet DaemonSet tolerates that key as well. See [`hack/microvm-assets/`](../hack/microvm-assets/) for scripts that assemble and stage these assets, plus a worked counter demo (`demos/counter/counter-microvm.yaml.tmpl`) that suspends and resumes an in-RAM counter across worker pods. @@ -540,22 +528,42 @@ pause, suspend, and resume using their own snapshots. for these templates. When an `ActorTemplate` is created with golden snapshots enabled: -1. Substrate creates and resumes a temporary golden actor in `ate-golden`. -2. It waits for readiness (or the warm-up interval), then suspends the actor. -3. It creates a published tag named after the template UID, copying the snapshot into tag-owned storage. +1. Substrate creates and resumes a temporary golden actor in the template's atespace, using the same atespace-scoped policies and storage grants as ordinary actors. +2. It waits for readiness plus a 20-second warmup, then suspends the actor at the template's `preferredFidelity`. The extra interval is zero when every container has a wakeup probe, since Resume already waits for readiness. +3. It creates a published tag named after the template UID, copying the snapshot into tag-owned storage in the same atespace. The golden actor uses that name too. 4. It deletes the golden actor and records the tag reference in the template status. -`CreateActor` uses an explicit `sourceTag` when supplied; otherwise it resolves the template's golden tag and records that snapshot on the new actor. If the golden tag is not ready yet, the actor starts without a snapshot and cold-boots even if the tag becomes ready before its first resume. The default does not populate the caller-owned `sourceTag` field. Deleting the template collects its golden tag and any unfinished golden actor. +Golden actors follow ordinary lifecycle and ingress policies and appear in +Actor lists. Leave these resources to the controller: modifying or deleting +them can interrupt Golden creation, and deleting a published Golden tag +invalidates the template's recorded reference. `DeleteActorTemplate` collects +its Golden tag and any unfinished Golden actor. + +Golden capture follows `preferredFidelity`. MEMORY preserves processes and memory; +VOLUMES preserves startup data in mounted volumes and starts new processes on +restore. ROOTFS preserves a micro-VM's filesystem and also starts new processes. + +`CreateActor` uses an explicit `sourceTag` when supplied; otherwise it resolves the template's golden tag and records that snapshot on the new actor. If the golden tag is not ready yet, the actor starts without a snapshot and cold-boots even if the tag becomes ready before its first resume. The default does not populate the caller-owned `sourceTag` field. ### Resumption Lifecycle -Once a template is `Ready`, creating an actor logically (via `kubectl ate create actor`) allows it to be resumed instantly on any free worker in the referenced `WorkerPool`. Substrate bypasses the standard container boot and restores the process directly from its last saved state. +CreateActor fixes the Actor's initial snapshot source at creation. Resume places +it on an eligible worker. A MEMORY snapshot restores saved processes; VOLUMES and +ROOTFS snapshots restore their persisted data and start new processes. An Actor +created without a snapshot boots from its template's images. ### Eviction A worker pod can go away while it hosts running actors: when its `WorkerPool` rolls to a new worker image or pod template, as in an [upgrade](upgrade.md), or scales down, when its node is drained, or when its Spot VM is reclaimed. -When the pod gets `SIGTERM`, its worker stops accepting actors and sends `SIGTERM` to the main process of each container of every actor it hosts. Each actor then has 30 minutes, counted from the pod's `SIGTERM`, to be suspended. An actor suspended in that window keeps its state and resumes on another worker like any suspended actor. +When the pod gets `SIGTERM`, its worker stops accepting actors and sends `SIGTERM` to the main process of each container of every actor it hosts. Each actor then has 30 minutes, counted from the pod's `SIGTERM`, to be suspended. The 30 minutes is the worker's shutdown budget, fixed in ateworker and kept shorter than the pod's one-hour termination grace period so that the worker, not the kubelet, kills what is left. An actor suspended in that window keeps its state and resumes on another worker like any suspended actor. + +Substrate does not suspend the actor for you. Whatever drives the actor, usually its harness, has to call `SuspendActor` with `DURABILITY_MODE_SYNCHRONOUS` (`kubectl ate suspend actor --mode=sync`) before those 30 minutes are up. An asynchronous suspend can lose the snapshot if the node goes away: + +- It returns once the snapshot is on the node. +- atelet schedules the upload to object storage about `--snapshot-upload-delay` (5 minutes by default) after the snapshot is taken. Neither when the upload starts nor when it finishes is promised. +- Until atelet detaches the actor, `--snapshot-detach-delay` (30 minutes by default, unrelated to the worker's 30 minutes), the actor stays pinned to the node and resumes there. +- If the node goes away before the upload completes, the snapshot goes with it. -Substrate does not suspend the actor for you. Whatever drives the actor, usually its harness, has to call `SuspendActor` (`kubectl ate suspend`) within the 30 minutes. Only the actor gets the `SIGTERM`, so it has to pass that on, for example on an endpoint the harness polls. +Only the actor gets the `SIGTERM`, so it has to pass that on, for example on an endpoint the harness polls. Every actor that must keep its state through an eviction needs a `SIGTERM` handler that: @@ -571,7 +579,7 @@ The node can cut the 30 minutes short. GKE node upgrades and cluster autoscaler ## 5. Best Practices * **Startup Logic:** Place expensive initialization (loading large models, establishing baseline connections) in your application's entry point. When golden snapshots are enabled, this initialization can be captured in the Golden Snapshot. Actors that cold-boot run initialization again. -* **Placement:** Ensure your `ActorTemplate`'s `sandboxClass` matches your `WorkerPool`'s `sandboxClasses[].name`, and use the template's `workerSelector` to target specific pools — pool selection is by label match, not by namespace or RBAC. +* **Placement:** The `WorkerPool` decides the sandbox class (`sandboxClasses[].name`), so use the template's `workerSelector` to target the pools whose class your workload needs — pool selection is by label match, not by namespace or RBAC. Without a selector an actor may land on any pool. * **Version Management:** When updating code, create a new `ActorTemplate` (e.g. `v2`). Substrate treats each template as an immutable state root. * **Eviction:** Give every actor a `SIGTERM` handler that does not exit, and have the actor suspended within 30 minutes of it. See [Eviction](#eviction). @@ -594,7 +602,7 @@ Registers a new logical actor in the system. Replaces the mutable fields of an existing actor with the ones in the request. * **Request:** `UpdateActorRequest` * `actor`: `Actor` — the complete replacement actor. `metadata.atespace` and `metadata.name` identify the resource; `metadata.uid` and `metadata.version` are **required** preconditions. `metadata` and `status` are server-owned and whatever the request carries in them is ignored. `source_tag` is immutable. - * `actor_template` may change only while the actor is `SUSPENDED`. The new template must name the same `SandboxConfig` and declare the same volumes and volume mounts. Once the actor owns an external snapshot, the new template must also store snapshots under the same `snapshotConfig.storageLocation`. + * `actor_template` may change only while the actor is `SUSPENDED`. The new template must declare the same volumes and volume mounts. Once the actor owns an external snapshot, the new template must also store snapshots under the same `snapshotConfig.storageLocation`. * **Response:** the updated `Actor`. * **Errors:** `INVALID_ARGUMENT` if `uid` or `version` is unset, or if the request changes an immutable field — including by leaving one unset; `FAILED_PRECONDITION` if a change to `actor_template` breaks one of the rules above; `ABORTED` if either guard no longer matches the stored resource. @@ -610,8 +618,10 @@ Activates a suspended or paused actor by restoring it onto a physical worker. Hibernate a running actor, capturing its current RAM and disk state into a snapshot. * **Request:** `SuspendActorRequest` * `actor`: `ObjectRef` of the actor to suspend. + * `options.mode`: (Optional) `DURABILITY_MODE_SYNCHRONOUS` returns once the snapshot is `COMPLETED` in object storage. `DURABILITY_MODE_ASYNCHRONOUS` (a pause) returns once the snapshot is captured on the worker's node, and atelet uploads it in the background. Unset runs an asynchronous suspend. * **Response:** `SuspendActorResponse` containing the `Actor` object in `ACTOR_STATE_SUSPENDED`, with its snapshot in `status.snapshots`. -* A successful suspend releases the actor's previous external snapshot: an actor keeps one, and only tags outlive it. To keep the snapshot a suspend just wrote, tag it with `CreateTag` while the actor is still suspended. +* Called on an actor that is already `ACTOR_STATE_SUSPENDED`, it returns the actor as it is, in either mode. atelet uploads a paused actor's node-local snapshot in the background, so `status.snapshots` may still show it with `locality` and a `PENDING` `durableSnapshot`. +* A successful suspend releases the actor's previous external snapshot: an actor keeps one, and only tags outlive it. To keep the snapshot a suspend just wrote, tag it with `CreateTag` while the actor is still suspended. After an asynchronous suspend, `CreateTag` fails with `FAILED_PRECONDITION` until atelet has uploaded the snapshot. Suspend with `DURABILITY_MODE_SYNCHRONOUS` to tag right away. #### Snapshot lifetime diff --git a/docs/architecture.md b/docs/architecture.md index a0f6f193bd..6583be2059 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -331,11 +331,11 @@ The node-level subsystem manages the physical execution of sandboxes and the mov ### Sandbox Classes -A `WorkerPool` selects a **sandbox class** (`spec.sandboxClasses[].name`), and each class has a matching `ateworker` herder image. The sandbox binaries themselves are not baked into the worker image — they, and the pause image holding the sandbox's namespaces, come at runtime from a cluster-scoped [`SandboxConfig`](api-guide.md#3-sandboxconfig-the-sandbox-itself) the `ActorTemplate` names in its sandbox config (naming one is currently required; per-class cluster defaults are planned) and are pinned into each snapshot's manifest so restores stay reproducible across runtime upgrades. +A `WorkerPool` selects a **sandbox class** (`spec.sandboxClasses[].name`), and each class has a matching `ateworker` herder image. The sandbox binaries themselves are not baked into the worker image — they, and the pause image holding the sandbox's namespaces, come at runtime from the cluster-scoped [`SandboxConfig`](api-guide.md#3-sandboxconfig-the-sandbox-itself) the `WorkerPool` resolves for its class (an explicit `configRef`, else the class default) and are pinned into each snapshot's manifest so restores stay reproducible across runtime upgrades. - * **gVisor** (`ateworker-gvisor`, the default): Runs the workload under `runsc` for kernel-level sandboxing. Suspend and resume leverage gVisor's native checkpoint/restore of the sandboxed process tree. + * **gVisor** (`ateworker-gvisor`, the default): Runs the workload under `runsc` for kernel-level sandboxing. Suspend and resume leverage gVisor's native checkpoint/restore of the sandboxed process tree. `DurableDir` volumes and each container's rootfs overlay are saved as separate filesystem checkpoint bundles (`fs/` and `rootfs/`). A `MEMORY` snapshot splits them out of the process checkpoint; `VOLUMES` and `ROOTFS` snapshots save only the bundles they need with `runsc fscheckpoint`, without memory, and restore them on a cold boot. - * **micro-VM** (`ateworker-microvm`): Runs the workload inside a [Kata Containers](https://katacontainers.io/) guest on the [Cloud Hypervisor](https://www.cloudhypervisor.org/) VMM. Suspend and resume capture a memory-only VM snapshot and restore it on-demand using `userfaultfd` memory demand-paging. Container rootfs writes are host-backed: the overlay is assembled on the host (read-only OCI image lower plus a per-actor writable upper) and served to the guest over the single virtio-fs share, so they cost reclaimable host page cache rather than guest RAM, and `ROOTFS` and `MEMORY` snapshots ship the upper as its own tar (a `ROOTFS` restore puts it back and cold-boots on it). `DurableDir` volumes travel over that same share and are likewise shipped as a tar, so a `VOLUMES` snapshot can capture them without any guest memory. Each volume is a subdirectory of the share, so an actor can have several at no extra cost in devices, which is why the micro-VM class lifts the single-`DurableDir` limit that still applies to gVisor. + * **micro-VM** (`ateworker-microvm`): Runs the workload inside a [Kata Containers](https://katacontainers.io/) guest on the [Cloud Hypervisor](https://www.cloudhypervisor.org/) VMM. Suspend and resume capture a memory-only VM snapshot and restore it on-demand using `userfaultfd` memory demand-paging. Container rootfs writes are host-backed: the overlay is assembled on the host (read-only OCI image lower plus a per-actor writable upper) and served to the guest over the single virtio-fs share, so they cost reclaimable host page cache rather than guest RAM, and `ROOTFS` and `MEMORY` snapshots ship the upper as its own tar (a `ROOTFS` restore puts it back and cold-boots on it). `DurableDir` volumes travel over that same share and are likewise shipped as a tar, so a `VOLUMES` snapshot can capture them without any guest memory. Each volume is a subdirectory of the share, so an actor can have several at no extra cost in devices. ### Networking Stack (`atenet` + `atunnel`) @@ -409,7 +409,7 @@ sequenceDiagram A-->>Tunnel: response Tunnel-->>Gateway: response Gateway-->>Client: response - Note over API,Store: later: an explicit SuspendActor checkpoints back to storage and frees the worker + Note over API,Store: later: an explicit SuspendActor checkpoints the actor and frees the worker ``` An Actor's `state` moves through these states (UML state machine diagram): @@ -419,20 +419,29 @@ stateDiagram-v2 [*] --> SUSPENDED : CreateActor SUSPENDED --> RESUMING : ResumeActor RESUMING --> RUNNING : restore / boot complete - RUNNING --> SUSPENDING : SuspendActor + RUNNING --> SUSPENDING : SuspendActor (sync) SUSPENDING --> SUSPENDED : checkpoint complete - RUNNING --> SUSPENDING : PauseActor - SUSPENDED --> SUSPENDING : SuspendActor (paused: uploads the node-local snapshot) + RUNNING --> SUSPENDING : SuspendActor (async, the default: keeps the snapshot on the node) SUSPENDED --> [*] : DeleteActor ``` A pause and a suspend both pass through `SUSPENDING`, and a paused actor and a suspended actor are both `SUSPENDED`. They differ only in the shape of -`status.snapshots`: after a pause the latest snapshot has a -`locality` (the node holding its files) and no `durableSnapshot`, and -`status.assignedNode` pins the actor to that node, so `ResumeActor` restores -it there from the local files. After a suspend the latest snapshot has a -`COMPLETED` `durableSnapshot` and the actor is pinned to no node. +`status.snapshots`: + + * After a pause (an asynchronous suspend), the latest snapshot has a + `locality` (the node holding its files) and a `PENDING` + `durableSnapshot`. `status.assignedNode` pins the actor to that node, so + `ResumeActor` restores it there from the local files. + * atelet uploads a paused actor's snapshot in the background + (`--snapshot-upload-delay`) and reports it, which marks the + `durableSnapshot` `COMPLETED`. `--snapshot-detach-delay` after the pause, + atelet deletes the local files and reports that too, which clears the + `locality` and `status.assignedNode`. + * After a synchronous suspend, the latest snapshot has a `COMPLETED` + `durableSnapshot` and the actor is pinned to no node. + +`SuspendActor` on a `SUSPENDED` actor returns it as it is, in either mode. ### Phase 1: Creation (`CreateActor`) @@ -472,13 +481,18 @@ Triggered by an explicit `SuspendActor` call. 1. **Checkpoint**: The `atelet` instructs `ateworker` to freeze the process and capture a memory+disk snapshot. - 2. **Persistence**: The `atelet` streams the snapshot from the pod to durable storage (e.g., GCS). + 2. **Persistence**: The `atelet` uploads the snapshot to durable storage + (e.g., GCS). With `DURABILITY_MODE_SYNCHRONOUS` the upload finishes before + `SuspendActor` returns. With the asynchronous default the snapshot stays + on the node and `atelet` uploads it in the background. 3. **Reclaim**: The physical worker is wiped and returned to the `WorkerPool`. - 4. **Release**: The external snapshot the Actor held before this suspend is - deleted from object storage. An Actor owns one snapshot at a time; one it - borrowed from a tag is left alone, since the tag owns it. + 4. **Release**: A synchronous suspend deletes the external snapshot the + Actor held before it from object storage. An asynchronous suspend keeps + that snapshot, since its own is not yet durable. It is deleted by the + first suspend after a newer snapshot is durable, or by `DeleteActor`. A + snapshot borrowed from a tag is left alone, since the tag owns it. 5. **State**: State transitions back to `ACTOR_STATE_SUSPENDED`, and the Actor's `status.snapshots` records the external snapshot it resumes from. diff --git a/docs/authorization.md b/docs/authorization.md index 6943ada4c5..323775e71a 100644 --- a/docs/authorization.md +++ b/docs/authorization.md @@ -137,8 +137,9 @@ one **atespace**. Substrate's components hold roles that access policies cannot grant, and only through their pod certificates. `ate-api-server`'s `--controller-spiffe-id`, -`--egress-gateway-spiffe-id` and `--ingress-router-spiffe-id` flags bind them, -defaulting to the install's; `atelet`'s comes from `--atelet-service-account`. +`--egress-gateway-spiffe-id`, `--ingress-router-spiffe-id` and +`--api-server-sidecar-spiffe-id` flags bind them, defaulting to the install's; +`atelet`'s comes from `--atelet-service-account`. | Role | Held by | Allows | |---|---|---| @@ -146,6 +147,7 @@ defaulting to the install's; `atelet`'s comes from `--atelet-service-account`. | `egress_gateway` | `atenet-egress` | Get actors and their egress policies; list actors across all atespaces; mint actor JWTs and certificates | | `ingress_router` | `atenet-router` | Resume actors; list actors across all atespaces | | `atelet` | `atelet` | Get actors' egress policies, for the worker that enforces them | +| `api_server_sidecar` | the `ate-api-server` pod identity, which its sidecars share, today the objectstore control plugin | Get actors, to derive the snapshot prefixes a per-actor storage credential may cover | Only `egress_gateway` can mint actor credentials: not even a global owner can, since a credential lets its holder act as the actor. @@ -186,10 +188,10 @@ The access policy RPCs are checked even with authorization off. | `ListActorTemplates` | `can_list_actor_templates`, the atespace, or global if none is given | Atespace viewer; global viewer for all atespaces | | `CreateActor` | 1. `can_create_actor`, the actor's atespace
2. `can_use_template`, the actor template
3. `can_use_tag`, the `sourceTag`, if one is given | 1. Atespace editor
2. Viewer of the template's atespace, which may be another one
3. Editor of the tag's atespace. Golden tags skip this check: the handler only accepts one built from the actor's own template. | | `UpdateActor` | 1. `can_update`, the actor
2. `can_use_template`, the template in the request | 1. Editor of its atespace
2. Viewer of the template's atespace, so an actor can't be switched to a template its editor can't use | -| `GetActor` | `can_get`, the actor | Viewer of its atespace; `atenet-egress` | +| `GetActor` | `can_get`, the actor | Viewer of its atespace; `atenet-egress`; the objectstore control plugin | | `DeleteActor` | `can_delete`, the actor | Editor of its atespace | | `ListActors` | `can_list_actors`, the atespace, or global if none is given | Atespace viewer; for all atespaces, global viewer, `atenet-router` or `atenet-egress` | -| `PauseActor`, `SuspendActor`, `RevertActor` | `can_pause` / `can_suspend` / `can_revert`, the actor | Editor of its atespace | +| `SuspendActor`, `RevertActor` | `can_suspend` / `can_revert`, the actor | Editor of its atespace | | `ResumeActor` | `can_resume`, the actor | Editor of its atespace; `atenet-router` | | `GetActorEgressPolicy` | `can_get_egress_policy`, the actor | Viewer of its atespace; `atenet-egress`; `atelet` | | `CreateActorEgressPolicy`, `UpdateActorEgressPolicy`, `DeleteActorEgressPolicy` | `can_create_egress_policy` / `can_update_egress_policy` / `can_delete_egress_policy`, the actor | Editor of its atespace | diff --git a/docs/csi-volumes.md b/docs/csi-volumes.md index 9997644c4e..388d7ce2f2 100644 --- a/docs/csi-volumes.md +++ b/docs/csi-volumes.md @@ -187,9 +187,6 @@ containers: httpGet: path: /readyz port: 8080 -sandboxConfig: - sandboxClass: gvisor - configName: gvisor-default resources: limits: - name: cpu diff --git a/docs/dev/microvm-local.md b/docs/dev/microvm-local.md index 0a1a1650a1..54d582b323 100644 --- a/docs/dev/microvm-local.md +++ b/docs/dev/microvm-local.md @@ -77,13 +77,13 @@ Expected: NAMESPACE NAME DESIRED READY AVAILABLE ate-demo-counter-microvm workerpool.ate.dev/counter-microvm 1 1 1 -ATESPACE NAME SANDBOX CLASS GOLDEN SNAPSHOT ERROR AGE -ate-demo-counter-microvm counter-microvm microvm b9f6bd93-3c5a-4b64-9d5e-2f8a1c7d0e42 1m +ATESPACE NAME GOLDEN TAG ERROR AGE +ate-demo-counter-microvm counter-microvm b9f6bd93-3c5a-4b64-9d5e-2f8a1c7d0e42 1m ``` -The template is ready once the GOLDEN SNAPSHOT column is non-empty (the +The template is ready once the GOLDEN TAG column is non-empty (the value is a UUID); ERROR means the golden build failed, and `-o yaml` shows -the full error message. An empty GOLDEN SNAPSHOT with no ERROR means the +the full error message. An empty GOLDEN TAG with no ERROR means the golden build — a full guest boot plus checkpoint — is still running. ## Option B: Apple Silicon macOS via Lima diff --git a/docs/dev/suspend-resume-phase-breakdown.md b/docs/dev/suspend-resume-phase-breakdown.md index d32d10f672..50cee957cb 100644 --- a/docs/dev/suspend-resume-phase-breakdown.md +++ b/docs/dev/suspend-resume-phase-breakdown.md @@ -87,7 +87,7 @@ The form fields are the flags documented in `benchmarking/locust/common/memload_config.py`; the boomer workers fetch the values from the master on each spawn. Let it run for about 90 seconds (≈10 cycles per user), then stop the test. For a single operation without locust, -`kubectl ate suspend actor -a benchmark-workloads` and +`kubectl ate suspend actor -a benchmark-workloads --mode=sync` and `kubectl ate resume actor -a benchmark-workloads` on any actor in the atespace produce one record pair each. diff --git a/docs/egress-traffic.md b/docs/egress-traffic.md index f25c7d8853..7fddd9ffa4 100644 --- a/docs/egress-traffic.md +++ b/docs/egress-traffic.md @@ -6,7 +6,7 @@ which opens a CONNECT tunnel to the egress gateway. Egress gateway applies polic DNS over UDP and TCP goes to the sandbox's DNS relay, which forwards a query to the worker pod's resolvers only when the queried hostname matches a host or host wildcard in the actor's egress policy. Queries from an actor without a -policy, or for a hostname not allowed by any rule, are answered `REFUSED`. +policy, or for a hostname not allowed by any rule, are answered `NXDOMAIN`. A policy created, updated, or deleted while the actor runs applies to its DNS within about 5 seconds. @@ -22,8 +22,8 @@ cannot reach the collector until the actor has an `http` egress rule for it. See | :---- | :---- | :---- | :---- | :---- | | any | HTTP(S) 1.1 / 2 | Supported with policy controls | atunnel -> egress gateway -> origin | `403 Forbidden` | | any | WebSocket | Blocked | n/a | `403 Forbidden` | -| any | Standard HTTP(S) CONNECT (forward-proxy tunnel) | Blocked | n/a | `403 Forbidden` | -| 53 | DNS to the sandbox's gateway | Allowed for hostnames matching the actor's egress policy | sandbox DNS relay -> worker pod's resolvers | `REFUSED` | +| any | Standard HTTP(S) CONNECT (forward-proxy tunnel) | Blocked | n/a | `403 Forbidden` (Envoy), `405 Method Not Allowed` (agentgateway) | +| 53 | DNS to the sandbox's gateway | Allowed for hostnames matching the actor's egress policy | sandbox DNS relay -> worker pod's resolvers | `NXDOMAIN` | | any | Any other TCP | Blocked | n/a | The connection is accepted and then closed with no bytes returned. There is no status code. atunnel logs the failure. | ## UDP @@ -31,7 +31,7 @@ cannot reach the collector until the actor has an `http` egress rule for it. See | Port | Traffic | Behavior | Path | What the actor sees when refused | | :---- | :---- | :---- | :---- | :---- | -| 53 | DNS to the sandbox's gateway | Allowed for hostnames matching the actor's egress policy | sandbox DNS relay -> worker pod's resolvers | `REFUSED` | +| 53 | DNS to the sandbox's gateway | Allowed for hostnames matching the actor's egress policy | sandbox DNS relay -> worker pod's resolvers | `NXDOMAIN` | | any other | Any other UDP | Blocked | n/a | Packets are dropped, not rejected: no ICMP port-unreachable is sent, so the client hangs until its own timeout. | ## Other protocols @@ -42,4 +42,4 @@ Everything that is neither TCP nor UDP is blocked. Packets are dropped, not reje If you would like Substrate to support egress traffic that is blocked above, please [file an issue](https://github.com/agent-substrate/substrate/issues/new) -describing your use case. \ No newline at end of file +describing your use case. diff --git a/docs/egress-trust-bundle.md b/docs/egress-trust-bundle.md index c426723e04..57ce065c1f 100644 --- a/docs/egress-trust-bundle.md +++ b/docs/egress-trust-bundle.md @@ -34,7 +34,7 @@ gateway. Instead, the sandbox's DNS relay forwards a query to the worker pod's resolvers, which are the host cluster's configured DNS servers, only when the queried hostname matches a host or host wildcard in the actor's egress policy: queries from an actor without a policy, or for a hostname not matched by any -rule, are answered `REFUSED`. +rule, are answered `NXDOMAIN`. ## Configuring a workload to trust public and man-in-the-middle CAs diff --git a/docs/glossary.md b/docs/glossary.md index 7b99b303b0..69e800fb48 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -83,14 +83,17 @@ for etcd. ## Lifecycle -- **Suspend**: hibernate a running or paused Actor into a durable snapshot in - external storage. A running Actor is checkpointed on its Worker (which is - then freed); a paused Actor's node-local snapshot is uploaded — narrowed to - the commit scope when the pause captured more — ending its node pinning. - -- **Pause**: a short-term checkpoint of a running Actor. Snapshot files remain - on the node VM, and the following Resume is prioritized onto the node VM - where the snapshots are persisted. +- **Suspend**: hibernate a running Actor into a snapshot. The Actor is + checkpointed on its Worker, which is then freed. A synchronous suspend + uploads the snapshot to external storage before it returns; an asynchronous + one is a Pause. Suspending an Actor that is already suspended or paused + returns it as it is. + +- **Pause**: a short-term checkpoint of a running Actor, taken by an + asynchronous suspend. Snapshot files remain on the node VM, and the following + Resume is prioritized onto the node VM where the snapshots are persisted. + atelet uploads the snapshot to external storage in the background, and ends + the node pinning once it deletes the local files. - **Resume**: activate a suspended/paused Actor by restoring it onto a Worker. The common path restores from a snapshot rather than cold-booting. @@ -104,11 +107,7 @@ for etcd. containers, potentially at different paths. This is the per-Actor application-data surface. - How many an `ActorTemplate` may declare depends on its `sandboxClass`: - a `microvm` template may declare several (they are subdirectories of one - virtio-fs share, so they cost nothing extra per volume), while a `gvisor` - template is limited to one until gVisor can accept more than a single - durable mount. + An `ActorTemplate` may declare several, on either sandbox class. ## Snapshots @@ -121,9 +120,7 @@ for etcd. afresh from the OCI image with the `DurableDir` contents restored. - **`ROOTFS`**: volumes plus the root filesystem changes made since boot. Process memory is discarded; on Resume the containers cold-boot from the - OCI image with their rootfs writes back in place. Served by the micro-VM - sandbox class only: gVisor keeps rootfs changes inside its memory - checkpoint, so a gVisor template that requests it is rejected. + OCI image with their rootfs writes back in place. - **`MEMORY`**: volumes, rootfs changes, and process memory. Used to capture everything needed to resume hot. diff --git a/docs/metrics/registry/metrics.yaml b/docs/metrics/registry/metrics.yaml index 850cb3d1e0..35117811f9 100644 --- a/docs/metrics/registry/metrics.yaml +++ b/docs/metrics/registry/metrics.yaml @@ -245,9 +245,8 @@ groups: - id: registry.ate.sandbox type: attribute_group brief: > - The sandbox that hosts an actor. The class comes from the template. Thus - it adds no series to the template labels. It lets a dashboard group the - data by class without a list of the template names. + The sandbox that hosts an actor. The class comes from the worker pool or + the snapshot runtime. It lets a dashboard group the data by class. attributes: - id: ate.sandbox.class stability: development @@ -269,8 +268,8 @@ groups: The source has no class, or a class that is not in this list. The default is not gvisor. Thus a bad record stays visible. ateapi reports it for a Worker that CreateWorker accepted with - no class, and for an ActorTemplate with no sandbox config. - atelet reports it for a snapshot manifest with no class. + no class. atelet reports it for a snapshot manifest with no + class. - id: ate.sandbox.oom_kills stability: development type: int @@ -303,11 +302,7 @@ groups: - id: latest stability: development value: latest - brief: The durable snapshot of the actor in object storage. - - id: golden - stability: development - value: golden - brief: The golden image of the ActorTemplate. + brief: A durable snapshot in object storage, including a template's golden snapshot. - id: boot stability: development value: boot @@ -378,10 +373,39 @@ groups: Checkpoint. atelet writes the snapshot to its destination. It uploads an external snapshot. It renames a local snapshot. The ate.snapshot.kind key tells you which. + - id: cleanup + stability: development + value: cleanup + brief: > + Checkpoint. atelet removes superseded local snapshots, unmounts + external volumes, and resets the actor directories on the node. - id: total stability: development value: total brief: The full operation. Use this value as the denominator. Do not add the phases together. + - id: ate.snapshot.upload_overdue_band + stability: development + brief: > + How long past the upload delay (--snapshot-upload-delay) an undurable + local snapshot has waited for a durable upload. + type: + members: + - id: under_1m + stability: development + value: under_1m + brief: Overdue by up to 1 minute. + - id: 1m_to_5m + stability: development + value: 1m_to_5m + brief: Overdue by more than 1 minute and up to 5 minutes. + - id: 5m_to_15m + stability: development + value: 5m_to_15m + brief: Overdue by more than 5 minutes and up to 15 minutes. + - id: over_15m + stability: development + value: over_15m + brief: Overdue by more than 15 minutes. - id: registry.ate.scheduler type: attribute_group @@ -704,7 +728,7 @@ groups: stability: stable type: int brief: The TCP port of the server that a gRPC client called. - examples: [8085] + examples: [21998] # --------------------------------------------------------------------------- # Metrics. ateapi @@ -831,9 +855,8 @@ groups: - ref: ate.sandbox.class requirement_level: conditionally_required: > - The operation is resume, suspend or revert, and ateapi found - the ActorTemplate. A revert continues when the ActorTemplate no - longer exists, and then omits this key. + The operation is resume, suspend or revert, and the actor has a + worker assignment or a snapshot with a sandbox runtime. - ref: ate.snapshot.kind requirement_level: conditionally_required: The operation is a resume and the snapshot source is known. Other operations do not restore, thus this key is absent. @@ -983,6 +1006,166 @@ groups: - ref: ate.template.name requirement_level: required + - id: metric.atelet.snapshot.upload.duration + type: metric + metric_name: atelet.snapshot.upload.duration + instrument: histogram + unit: s + stability: development + brief: The time of one background upload of a local snapshot to object storage on atelet. + note: > + atelet uploads deferred snapshots in the background once the upload delay + or the detach delay elapses. A probe that finds the manifest committed + already without uploading is not recorded. An upload that Terminate or an + atelet shutdown cancels records error.type Canceled. Synchronous uploads + from UPLOAD_MODE_IMMEDIATE checkpoints and PromoteSnapshot are recorded + on ate.actor.checkpoint.duration instead. + annotations: + substrate: + emitted_by: [atelet] + golden_signals: [latency, errors] + code_anchor: cmd/atelet/internal/snapshottracker/metrics.go + buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30, 60] + cuj: Are background snapshot uploads slow or failing on a node? + attributes: + - ref: ate.snapshot.fidelity + requirement_level: required + - ref: error.type + requirement_level: + conditionally_required: The upload failed. If this key is absent, the upload was correct. + + - id: metric.atelet.snapshot.detach.duration + type: metric + metric_name: atelet.snapshot.detach.duration + instrument: histogram + unit: s + stability: development + brief: The time of deleting the local snapshots of an actor during a background detach on atelet, excluding any snapshot upload. + note: > + atelet detaches an actor once it stays suspended on the node for the + detach delay. If object storage does not hold the latest local snapshot + yet, the detach uploads it first and records that upload on + atelet.snapshot.upload.duration; a detach whose upload fails stops there + and is not recorded here. Once the snapshot is durable, the detach deletes + the local snapshots of the actor, which is what this histogram times. A + detach that Terminate or an atelet shutdown cancels records error.type + Canceled. A failed detach keeps the local snapshots, and atelet tries it + again later. A detach that a DetachActor call asks for is not recorded. + annotations: + substrate: + emitted_by: [atelet] + golden_signals: [latency, errors] + code_anchor: cmd/atelet/internal/snapshottracker/metrics.go + buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30, 60] + cuj: Local snapshots stay on a node after the detach delay. Do the detaches fail? + attributes: + - ref: ate.snapshot.fidelity + requirement_level: required + - ref: error.type + requirement_level: + conditionally_required: The detach failed. If this key is absent, the detach was correct. + + - id: metric.atelet.snapshot.upload.overdue.duration + type: metric + metric_name: atelet.snapshot.upload.overdue.duration + instrument: histogram + unit: s + stability: development + brief: The time from when a background snapshot upload on atelet is due to when a worker starts it. + note: > + This instrument has no labels. An upload is due the upload delay + (--snapshot-upload-delay) after the state of the actor on the node + stopped being durable, or when the retry delay ends after a failure. A + fixed number of upload workers (--snapshot-upload-workers) runs uploads + on each node, separate from the detach workers. The value stays near zero + while an upload worker is free. It grows when every upload worker is + busy, while a detach or a PromoteSnapshot of the actor runs, and after an + atelet restart for the uploads that came due while atelet was down. + annotations: + substrate: + emitted_by: [atelet] + golden_signals: [saturation] + code_anchor: cmd/atelet/internal/snapshottracker/metrics.go + buckets: [0.1, 1, 5, 10, 30, 60, 120, 300, 600, 1200, 1800, 3600] + cuj: Background uploads fall behind on a node. Are the upload workers saturated? + attributes: [] + + - id: metric.atelet.snapshot.detach.overdue.duration + type: metric + metric_name: atelet.snapshot.detach.overdue.duration + instrument: histogram + unit: s + stability: development + brief: The time from when a background detach on atelet is due to when a worker starts it. + note: > + This instrument has no labels. A detach is due the detach delay + (--snapshot-detach-delay) after the latest checkpoint of the actor, or + when the retry delay ends after a failed detach. A fixed number of detach + workers (--snapshot-detach-workers) runs detaches on each node, separate + from the upload workers. The value stays near zero while a detach worker + is free. It grows when every detach worker is busy, while an upload, Run, + Restore, Checkpoint, or PromoteSnapshot of the actor runs, and after an + atelet restart for the detaches that came due while atelet was down. + annotations: + substrate: + emitted_by: [atelet] + golden_signals: [saturation] + code_anchor: cmd/atelet/internal/snapshottracker/metrics.go + buckets: [0.1, 1, 5, 10, 30, 60, 120, 300, 600, 1200, 1800, 3600] + cuj: Local snapshots stay on a node after the detach delay. Are the detach workers saturated? + attributes: [] + + - id: metric.atelet.snapshot.upload.overdue.actors + type: metric + metric_name: atelet.snapshot.upload.overdue.actors + instrument: updowncounter + unit: "{actor}" + stability: development + brief: The number of actors on the node whose local snapshot is past the upload delay and not durable yet, by overdue band. + note: > + atelet reads this value with a callback. It counts each actor with a + snapshot state file on the node, and not only the actors in memory. + Overdue time counts from when the state of the actor stopped being + durable plus the upload delay (--snapshot-upload-delay). An actor leaves + the count only when its upload completes or the tracker drops it, so a + stuck or retrying upload keeps moving into higher bands. An undurable + snapshot with no object storage location to upload to does not enter this + count. The sum across bands is the number of actors on the node with an + overdue upload. atelet sets all four band values, also when they are 0. + annotations: + substrate: + emitted_by: [atelet] + golden_signals: [saturation, errors] + code_anchor: cmd/atelet/internal/snapshottracker/metrics.go + cuj: How many actors on a node have overdue snapshot uploads, and how far behind are they? + attributes: + - ref: ate.snapshot.upload_overdue_band + requirement_level: required + + - id: metric.atelet.snapshot.undurable.age.max + type: metric + metric_name: atelet.snapshot.undurable.age.max + instrument: gauge + unit: s + stability: development + brief: The age of the oldest local snapshot on the node that object storage does not hold yet. + note: > + atelet reads this value with a callback. This instrument has no labels. + The age counts from when the state of the actor on the node stopped being + durable. The upload delay (--snapshot-upload-delay) counts from the same + time. The value is 0 if object storage holds each local snapshot. On a + correct node the value stays below the upload delay plus the time of one + upload. A larger value means that the uploads fail, wait for a worker, or + hang, or that a local snapshot has no object storage location to upload + it to. + annotations: + substrate: + emitted_by: [atelet] + golden_signals: [saturation, errors] + code_anchor: cmd/atelet/internal/snapshottracker/metrics.go + cuj: Do background uploads keep up on a node, or would a loss of the node lose old actor state? + attributes: [] + - id: metric.ate.actor.stats.cpu.time type: metric metric_name: ate.actor.stats.cpu.time diff --git a/docs/network-egress.md b/docs/network-egress.md index 842e046c4f..94344e9072 100644 --- a/docs/network-egress.md +++ b/docs/network-egress.md @@ -8,7 +8,7 @@ Agent Substrate enforces that all outbound traffic from an actor, except traffic ## Transport -At present, actor TCP traffic exits the sandbox through [`atunnel`](./glossary.md#components) and is forwarded on to the PEP. The exception is DNS: TCP and UDP traffic to port 53 on the sandbox's gateway bypasses the PEP and is answered by the sandbox's DNS relay, which forwards a query to the worker pod's resolvers only when the queried hostname matches a host or host wildcard in the actor's egress policy; queries from an actor without a policy, or for a hostname not matched by any rule, are refused. All other protocols are blocked. `atunnel` is configured with the PEP's network address and port at startup and uses that address to open an mTLS HTTP/1.1 CONNECT tunnel to the PEP. The CONNECT request target and `Host` header are both set to the original destination IP and port that `atunnel` derived from the actor's TCP connection. Actor networking is currently IPv4-only. +At present, actor TCP traffic exits the sandbox through [`atunnel`](./glossary.md#components) and is forwarded on to the PEP. The exception is DNS: TCP and UDP traffic to port 53 on the sandbox's gateway bypasses the PEP and is answered by the sandbox's DNS relay, which forwards a query to the worker pod's resolvers only when the queried hostname matches a host or host wildcard in the actor's egress policy; queries from an actor without a policy, or for a hostname not matched by any rule, are answered `NXDOMAIN`. All other protocols are blocked. `atunnel` is configured with the PEP's network address and port at startup and uses that address to open an mTLS HTTP/1.1 CONNECT tunnel to the PEP. The CONNECT request target and `Host` header are both set to the original destination IP and port that `atunnel` derived from the actor's TCP connection. Actor networking is currently IPv4-only. Before traffic reaches the egress PEP, ateworker rejects packets destined for the IPv4 link-local range `169.254.0.0/16` or the IPv6 link-local range `fe80::/10`. The internal actor network `169.254.17.0/30` is exempt because it carries traffic for the actor gateway, DNS relay, ingress, and wakeup probes. Localhost traffic remains inside the actor sandbox and does not traverse the PEP. Access to non-link-local destinations remains controlled by substrate egress policy and, at the `WorkerPool` boundary, Kubernetes NetworkPolicy. diff --git a/docs/observability.md b/docs/observability.md index 2b93ffbdae..1cc0362e45 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -173,7 +173,7 @@ ateapi's `Actor state changed` is written once per committed actor state transit `ate.actor.state` takes the `ateapipb.ActorState` values lowercased, so the log vocabulary and the state machine cannot fork. The last record for an actor's uid is the state it is in now, and its timestamp is when that state began. Query it per actor, not in aggregate: neither key is a metric label, because both only ever appear beside actor identity, which [the cardinality rules](#the-metric-registry) keep off metrics entirely. -`ate.actor.operation.name` says which operation drove the transition, which the state alone does not: an actor reaches `suspended` both when it is created and when it is suspended. PauseActor and SuspendActor are one operation and are both labeled `suspend`. +`ate.actor.operation.name` says which operation drove the transition, which the state alone does not: an actor reaches `suspended` both when it is created and when it is suspended. SuspendActor in either durability mode is one operation, labeled `suspend`. The record goes out after the store commit, never before, and the state is read straight off the committed record rather than named by the caller. Every state commit has a version check too, so if two writers race, the one that lost writes nothing. You will never see a state here that the store did not actually hold. @@ -289,6 +289,12 @@ Agent Substrate emits foundational OpenTelemetry system and server metrics to mo | `ate.scheduler.assignment.duration` | ateapi | histogram | time of one attempt to assign a worker to an actor, with one record for each resume (ateapi retries an attempt that loses a race for a worker and records only the last attempt), with the outcome (`assigned` / `no_capacity` / `error`), the assigned pool (`ate.workerpool.namespace` + `ate.workerpool.name`) and sandbox class to catch scheduling latency and capacity starvation problems | | `ate.actor.restore.duration` | atelet | histogram | how long each phase of a restore takes on the worker node, which is where cold-start latency actually goes once ateapi hands off (labels `ate.snapshot.phase`, `ate.snapshot.kind`, `ate.snapshot.fidelity`, `ate.template.atespace`, `ate.template.name`, `ate.sandbox.class`) | | `ate.actor.checkpoint.duration` | atelet | histogram | the same phase breakdown for writing a snapshot, so a slow suspend can be attributed to ateworker or to the upload (same labels as the restore histogram) | +| `atelet.snapshot.upload.duration` | atelet | histogram | how long a background upload of a local snapshot to object storage takes and whether it failed (`error.type` present = failure, absent = success; label `ate.snapshot.fidelity`) | +| `atelet.snapshot.detach.duration` | atelet | histogram | how long a background detach takes to delete an actor's local snapshots, excluding any snapshot upload, and whether it failed (`error.type` present = failure, absent = success; label `ate.snapshot.fidelity`) | +| `atelet.snapshot.upload.overdue.duration` | atelet | histogram | how long after a background snapshot upload came due a worker started it: near zero while an upload worker is free, growing when the node's upload workers fall behind (no labels) | +| `atelet.snapshot.detach.overdue.duration` | atelet | histogram | how long after a background detach came due a worker started it: near zero while a detach worker is free, growing when the node's detach workers fall behind (no labels) | +| `atelet.snapshot.upload.overdue.actors` | atelet | up/down counter | actors on the node with an undurable local snapshot past `--snapshot-upload-delay` waiting for an upload to finish, bucketed by time overdue (label `ate.snapshot.upload_overdue_band`: `under_1m`, `1m_to_5m`, `5m_to_15m`, `over_15m`) | +| `atelet.snapshot.undurable.age.max` | atelet | gauge | age of the oldest local snapshot on the node that is not durable yet, counted from when the actor's state stopped being durable, or 0 when every local snapshot is durable; a healthy node stays below `--snapshot-upload-delay` plus one upload | | `ate.imagecache.requests` | atelet | counter | image lookups in the node-local image cache, by outcome (`ate.imagecache.outcome`), with `error.type` on the `error` outcome. A miss pays for the pull and the unpack, so the hit ratio per node is a leading indicator of resume latency | The table lists the OpenTelemetry instrument names. How a name appears in a query depends on the backend (Cloud Monitoring (GMP) / Kind collector). @@ -311,13 +317,19 @@ For `ate.imagecache.requests`: **No keys** means the operation has no pool. The actor-centric instruments omit the pair, so a crash before the actor reached a worker, or the `no_capacity` outcome, names no pool. No instrument sends the pair with both keys empty. The three snapshot labels are orthogonal and mean the same thing on every histogram that carries them: -* `ate.snapshot.kind`: which snapshot the operation reads or writes. `local` (node-local, written by a pause), `latest` (the actor's own durable snapshot), `golden` (the template's image), or `boot` (from scratch, so it never appears on the atelet histograms). +* `ate.snapshot.kind`: which snapshot the operation reads or writes. `local` (node-local, written by a pause), `latest` (a durable snapshot, including a template's golden snapshot), or `boot` (from scratch, so it never appears on the atelet histograms). * `ate.snapshot.fidelity`: what state it holds. `volumes`, `rootfs`, or `memory`. -* `ate.snapshot.phase`: which step was timed. `volume_mount`, `manifest_fetch`, `sandbox_assets`, `download`, `oci_unpack`, `worker_restore` on restore; `sandbox_assets`, `worker_checkpoint`, `persist` on checkpoint; `total` on both. +* `ate.snapshot.phase`: which step was timed. `volume_mount`, `manifest_fetch`, `sandbox_assets`, `download`, `oci_unpack`, `worker_restore` on restore; `sandbox_assets`, `worker_checkpoint`, `persist`, `cleanup` on checkpoint; `total` on both. + +Golden snapshots use `latest` on both control-plane and atelet histograms, so +the resulting series also includes other durable restores and does not isolate +golden first restores. To investigate an individual activation, correlate the +Actor's recorded snapshots with its logs and traces instead of adding Actor +identity to metric labels. **Phases overlap and do not sum to `total`.** The download runs concurrently with the asset fetch and OCI unpack, so each is an independent observation; use `total` as the denominator. A phase that never started is absent rather than zero. -The `ate.*` control-plane metric labels are either fixed value sets (operation, outcome, state, class, kind, scope, phase) or scoped to the deployment catalog (template and pool names are operator-created, never derived from request payloads), and the label set varies per operation: resume carries the most dimensions, delete only the operation and error type. `ate.sandbox.class` is derived from the template (each template has exactly one class), so it adds no extra series next to the template labels; it exists so dashboards can aggregate by class without enumerating template names. High-cardinality actor identity (name/uid/atespace) stays off metrics entirely and lives on logs and traces instead — for resource usage, on the [per-actor usage events](#per-actor-usage-events). +The `ate.*` control-plane metric labels are either fixed value sets (operation, outcome, state, class, kind, scope, phase) or scoped to the deployment catalog (template and pool names are operator-created, never derived from request payloads), and the label set varies per operation: resume carries the most dimensions, delete only the operation and error type. `ate.sandbox.class` is derived from the runtime on the actor's worker assignment or snapshot (each worker pool has exactly one class), so it adds no extra series next to the pool label; it exists so dashboards can aggregate by class without enumerating pool names. High-cardinality actor identity (name/uid/atespace) stays off metrics entirely and lives on logs and traces instead — for resource usage, on the [per-actor usage events](#per-actor-usage-events). ### The metric registry @@ -370,7 +382,7 @@ A queue that has never processed an item bridges as an exponential histogram wit ateapi, atelet, atenet-router and the credential provider also serve every instrument on their Prometheus `/metrics` endpoint. A cluster that scrapes those endpoints sets `OTEL_METRICS_EXPORTER=prometheus` on the components, so each series reaches the backend once. With `prometheus` the components install no OTLP metric reader and keep the Prometheus one; traces and logs are unaffected. `none` stops the push too. For now `/metrics` stays on for every value, including `otlp` and `none`. This is temporary until the probes move off `:9090`; `otlp` and `none` will then turn it off ([#2067](https://github.com/agent-substrate/substrate/issues/2067)). The variable takes a comma-separated list, with the same rules as `OTEL_LOGS_EXPORTER`. Unset is `otlp,prometheus`, or `otlp` alone on ateworker. `otlp` keeps the push, and `prometheus` or `none` stops it. ateworker serves no `/metrics`, so there `prometheus` is skipped as unknown with a warning, and the push stays on unless the variable is `none`. -atecontroller's `:8080` listener follows the variable. When the variable is unset or `otlp,prometheus`, atecontroller pushes every metric over OTLP, and `:8080` serves only the controller-runtime families ([#2402](https://github.com/agent-substrate/substrate/issues/2402)). With `otlp`, atecontroller pushes every metric and closes `:8080`. With `prometheus`, atecontroller does not push, and `:8080` serves the controller-runtime families and the `ate.workerpool.*` instruments. With `none`, atecontroller does not push and closes `:8080`. +atecontroller's `:8080` listener follows the variable. When the variable is unset or `otlp,prometheus`, atecontroller pushes every metric over OTLP, and `:8080` serves the controller-runtime families and the `ate.workerpool.*` instruments. With `otlp`, atecontroller pushes every metric and closes `:8080`. With `prometheus`, atecontroller does not push, and `:8080` serves the controller-runtime families and the `ate.workerpool.*` instruments. With `none`, atecontroller does not push and closes `:8080`. ### Local Metrics with Prometheus (Kind Cluster) @@ -470,7 +482,7 @@ Verbatim forwarding is restricted to known ateworker sources and refuses anythin ### Actor telemetry needs an egress policy -An actor's own OpenTelemetry SDK does not use the relay. It dials the collector over the network like any other outbound connection, and that connection is subject to the actor egress lockdown described in [Network Egress](network-egress.md): every actor TCP connection is redirected into `atunnel` and carried to the egress gateway, which refuses anything the actor's `EgressPolicy` does not allow. An actor with no policy, or a policy with no matching rule, cannot reach the collector at all. With no policy, or a policy whose rules do not include the collector's hostname, the actor cannot even resolve the collector's name: the sandbox's DNS relay answers its queries `REFUSED`, so each export fails before it connects. If a rule allows the hostname on another protocol or port (for example, `*` on port `443`), DNS resolves the name, the SDK sees `Unavailable: connection reset by peer` on each export, and the gateway logs why it denied the connection. +An actor's own OpenTelemetry SDK does not use the relay. It dials the collector over the network like any other outbound connection, and that connection is subject to the actor egress lockdown described in [Network Egress](network-egress.md): every actor TCP connection is redirected into `atunnel` and carried to the egress gateway, which refuses anything the actor's `EgressPolicy` does not allow. An actor with no policy, or a policy with no matching rule, cannot reach the collector at all. With no policy, or a policy whose rules do not include the collector's hostname, the actor cannot even resolve the collector's name: the sandbox's DNS relay answers its queries `NXDOMAIN`, so each export fails before it connects. If a rule allows the hostname on another protocol or port (for example, `*` on port `443`), DNS resolves the name, the SDK sees `Unavailable: connection reset by peer` on each export, and the gateway logs why it denied the connection. To let an actor export, give it an `http` rule naming the collector's Service and the OTLP ports. It must be an `http` rule, not `https`: the OTLP/gRPC exporter is gRPC over cleartext HTTP/2 (h2c) on `4317`, which an `http` rule admits, and the OTLP/HTTP exporter is cleartext HTTP/1.1 on `4318`. On Kind the collector is `opentelemetry-collector.otel-system.svc`; on GKE with the managed collector it is `opentelemetry-collector.gke-managed-otel.svc.cluster.local`. Whatever the name, it has to match the host the actor's `OTEL_EXPORTER_OTLP_ENDPOINT` names, because the rule is matched on the request authority. diff --git a/docs/resource-model.md b/docs/resource-model.md index b4a90adf0b..6aba72da9d 100644 --- a/docs/resource-model.md +++ b/docs/resource-model.md @@ -104,7 +104,7 @@ classDiagram Tag "*" --> "1" Actor : source_actor ActorAssignment "0..1" --> "1" Actor : actor ActorTemplate ..> WorkerPool : worker_selector - ActorTemplate ..> SandboxConfig : sandbox_config + WorkerPool ..> SandboxConfig : sandboxClasses WorkerPool "1" *-- "*" WorkerPod Worker "1" --> "1" WorkerPod ``` diff --git a/docs/upgrade.md b/docs/upgrade.md index 7f5e73d0aa..ed65431c8d 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -12,14 +12,16 @@ This runbook upgrades a running Agent Substrate install to a newer release. If an upgrade goes wrong, you can roll back to the release you upgraded from, and no further. - +These rules and the step order below come from the [version skew policy](version-skew-policy.md). + + | Step | What changes | What running actors see | How long | |---|---|---|---| | 1 | podcertificate-controller | Nothing | A minute | | 2 | atelet, one node at a time | Nothing | Seconds to about 6 minutes per node | | 3 | workers | `SIGTERM`, then a 30-minute window to be suspended before `SIGKILL` | Seconds to hours per pool | -| 4 | ate-api-server, the CRDs and ate-controller, atenet, SandboxConfig, then the rest of the system bundle | API calls, long requests and outbound connections can be cut once; workers may get `SIGTERM` again. See step 4 | A few minutes, or as long as step 3 if workers roll again | +| 4 | ate-api-server, the CRDs and ate-controller, atenet, SandboxConfig, then the rest of the system bundle | API calls, long requests and outbound connections can be cut once. See step 4 | A few minutes | Rollback is the same list in reverse, from the old release. An actor that crashes along the way goes back to its last snapshot with `kubectl ate revert actor -a `. @@ -192,14 +194,11 @@ kubectl ate get actors -A ```bash go run ./cmd/ate-setup deploy ate-controller --config ~/ate-upgrade/new.yaml -kubectl get deploy -A -l ate.dev/worker-pool ``` -This applies the new CRDs and ate-controller's RBAC, then deploys the new `ate-controller`. Rerunning is safe. If this release changes the worker pod template, the new ate-controller rolls every pool again, with another round of `SIGTERM`: wait as in step 3. - - +This applies the new CRDs and ate-controller's RBAC, then deploys the new `ate-controller`. Rerunning is safe. Within a major the worker pod template stays the same, so no pool rolls. -- **Done:** the command exits 0, `kubectl -n ate-system get pods -l app=ate-controller` shows one pod, Running, with RESTARTS 0, and every pool shows READY `n/n` with UP-TO-DATE and AVAILABLE `n`. Check the pools a minute after the new pod starts: a second roll begins only once it reconciles them. +- **Done:** the command exits 0, `kubectl -n ate-system get pods -l app=ate-controller` shows one pod, Running, with RESTARTS 0, and a minute later `kubectl get pods -A -l ate.dev/worker-pool` shows no worker pod younger than this step. - **Stuck:** the new ate-controller pod keeps restarting. [Roll back step 4](#roll-back-step-4). #### 4c. atenet @@ -278,7 +277,7 @@ go run ./cmd/ate-setup deploy apiserver --config ~/ate-upgrade/old.yaml go install ./cmd/kubectl-ate ``` -`deploy ate-controller` restores the old CRDs, RBAC and `ate-controller`. If the release changed the worker pod template, every pool rolls again: wait as in step 3. Fields the new release added to WorkerPools, SandboxConfigs or CSIDriverConfigs are dropped from those objects. Remove them from your YAML files too, or `kubectl apply` rejects the files. +`deploy ate-controller` restores the old CRDs, RBAC and `ate-controller`. No pool rolls. Fields the new release added to WorkerPools, SandboxConfigs or CSIDriverConfigs are dropped from those objects. Remove them from your YAML files too, or `kubectl apply` rejects the files. The database schema stays new; the old API server drops fields it does not know when it rewrites a record. Step 4e is not undone either: the NetworkPolicies and the bundled PostgreSQL stay as the new release left them. diff --git a/docs/version-skew-policy.md b/docs/version-skew-policy.md new file mode 100644 index 0000000000..68663bcf1a --- /dev/null +++ b/docs/version-skew-policy.md @@ -0,0 +1,143 @@ +# Version skew policy + +The [compatibility policy](https://docs.google.com/document/d/1obSIvfcafLNniLYTQCcT2eCgxHqa2AQ3Ga7YTsju49s/edit?usp=sharing&resourcekey=0-5VqJ13Vj1QctB7SnnfmtkQ) says what may change between two releases. + + + +This document says which releases of the substrate components may run together, and the order in which they are upgraded so that only supported combinations occur. The [upgrade runbook](upgrade.md) implements it and may add restrictions of its own. + +## Scope + +Two releases are compatible when every change between them is compatible under the compatibility policy. This document covers moving an installation between two compatible releases, and the mixture of components that exists while that happens. Between incompatible releases, reinstall. + +This policy applies from 1.0.0. From 1.0.0, the patch and minor releases of one major are compatible. + +Below, N and N+1 are two consecutive compatible releases. All components of a release are built from one commit and carry one version string. + +Patch releases add no API fields, public or internal, and no schema migrations. + +## Components + +* `ate-api-server`: serves the public API, calls `atelet`, talks to the database. +* `ate-controller`: renders WorkerPools into Deployments, registers workers with `ate-api-server`. +* `atelet`: one pod per node (DaemonSet), drives the `ateworker` processes on its node. +* Workers: the pods of a WorkerPool, running `ateworker` and the actors. +* `atenet`: router and egress gateway. +* `podcertificate-controller`: signs the certificates components present to each other. +* Clients: `kubectl-ate` and anything else using the public API. + +See more in [architecture.md](architecture.md). + +## Supported version skew + +The components are ordered: `podcertificate-controller`, `atelet`, workers, `ate-api-server`, `ate-controller`, `atenet`, clients. + +* A component may be one release newer than any component after it in this order. +* A component must not be newer than any component before it. +* While a component rolls, its own replicas, nodes or pods are at N and N+1. + * The API schema is not shared. Until every `ate-api-server` replica is at N+1, a request can reach an N replica, which rejects fields new in N+1. So clients must not use fields new in N+1 until `ate-api-server` finishes rolling. + * The database schema is shared. The first N+1 `ate-api-server` replica migrates it, so `ate-api-server` replicas at N and N+1 use the N+1 database schema while `ate-api-server` rolls. + +So during an upgrade from N to N+1, a new `atelet` drives old `ateworker` processes on its node, a new `atelet` calls an old `ate-api-server`, an old `ate-controller` renders Deployments for new workers, and old `ate-controller` and clients talk to a new `ate-api-server`. + +The reverse pairs are not supported: in particular, a worker must not be newer than the `atelet` on its node, and `ate-api-server` must not be newer than any worker. `ate-api-server` reaches the workers through `atelet`, and a new `ate-api-server` may rely on new worker behavior behind existing fields, so the workers go first. + +## Requirement of a release + +The skew rules above hold only if every release keeps the following. + +### Worker pod template + +`ate-controller` upgrades after the workers. So the workers start the `ateworker` N+1 image with the Deployment `ate-controller` N renders. + +Within a major, `ate-controller` renders the same worker pod template for the same WorkerPool. New `ateworker` behavior comes from the image or from a new WorkerPool field, not from the template. So `ateworker` accepts the flags and environment the template passes for the whole major, and no WorkerPool rolls at `ate-controller`'s step, in an upgrade or a rollback. + +### API fields + +These rules say what a component does with a protobuf field it does not know. + +Servers: + +* `ate-api-server` rejects a request that carries a field it does not know, with `INVALID_ARGUMENT`. +* `ate-api-server` drops the fields it does not know when it reads a stored object, and serves the reduced object. Such a field was written by a newer replica, during `ate-api-server`'s roll or before a rollback, and this release cannot validate it. Rejecting it would make the object unreadable. Keeping it would serve a field this release cannot validate. +* `atelet` rejects fields it does not know. Its callers, `ate-api-server` and `ateworker`, are never newer than it during an upgrade or a rollback in the supported order, so an unknown field means the upgrade order was not followed. +* `ateworker` rejects fields it does not know. Its caller, `atelet`, can be one release newer, so `atelet` sends a new field only on releases after `ateworker` learns it (two-step releases). + +Clients: + +* A client that reads an object, changes it and writes it back keeps the unknown fields. An older client must not strip fields that a newer server still understands. +* `kubectl-ate` rejects a manifest file that has fields it does not know, as the server does for a create. + +Wherever a caller can be newer than its server, the caller sends a new field only one release after the server learns it (two-step releases). Today that is `atelet` calling `ate-api-server` and `ateworker`. + +### CRDs + +The N+1 CRDs are applied with `ate-controller` N+1. Components that upgrade before it read the N CRDs, so users must not set new CRD fields until the upgrade completes. + +Between compatible releases, a CRD change only adds optional fields. The CRDs serve one version with no conversion. So a rollback from N+1 that applies the N CRDs again removes every field N does not know from every object. + +### SandboxConfig + +Each release ships its SandboxConfig under its own name and keeps the earlier ones. The upgrade adds the N+1 one, and WorkerPools without a `configRef` switch to it. That does not change the pod template, so workers do not roll. + +An actor resolves its pool's SandboxConfig each time it starts. If the runtime its snapshot needs is gone, it cold boots, keeping its files and losing its memory. Rollback deletes the N+1 SandboxConfig. + + + +### Database schema + +The schema is updated with `ate-api-server`, and migrations follow [postgresql-schema-evolution.md](dev/postgresql-schema-evolution.md): every migration keeps working for the previous `ate-api-server` binary. Migrations are never rolled back. + +### Data on nodes and in snapshots + +This covers Substrate's own data only. The sandbox runtime's checkpoint files follow the runtime's own rules: a restore across runtime versions keeps only the file systems. + +This section is about data one release writes and another release reads, whether that is a different component or the same component after an upgrade or a rollback. Substrate's data comes in two kinds, by how long it lives. + +* Data on the node: the directory layout and the files `atelet` keeps there. + * What N writes, N+1 can read. What N+1 writes, N can read. + * So a layout change takes two releases: N+1 reads the old and the new layout but still writes the old one, and N+2 writes the new one. This keeps a rollback possible. + * `atelet` passes the per-actor directories to `ateworker` in the RPC, so only `atelet` knows the layout. The sockets `atelet` and `ateworker` use to reach each other do not change within a major. +* Data in snapshots: `manifest.json`, which `atelet` writes and reads. + * Within a major, a release reads the snapshot data any earlier release of that major wrote. + * A new field must be safe to ignore. A release ignores manifest fields it does not know, so a snapshot taken at N+1 still restores after a rollback to N. + +### System bundle + +The upgrade applies the rest of the system bundle last, and a rollback does not undo it. So a component at N+1 must work with these resources at N, and a component at N must work with them at N+1. + +Changes that only take something away, such as a NetworkPolicy that denies traffic, can go in this step. Anything a new component needs to run, such as a new Secret or a rule that allows its traffic, ships in that component's own step. + +## Supported upgrade order + +To move from N to N+1, in this order, waiting for each step to finish: + +1. `podcertificate-controller`. +2. `atelet`, node by node. +3. Every WorkerPool's `spec.workerImage`, which rolls its workers. +4. `ate-api-server`. The first N+1 replica migrates the schema. +5. The CRDs, then `ate-controller`. +6. `atenet`. +7. The SandboxConfig admission policy and the default SandboxConfig. +8. The rest of the substrate system bundle: what no step above deploys. +9. Clients. + +Rollback from N+1 to N is the same list in reverse, with release N, except that the schema migration and step 8 are not undone. Clients stop sending N+1 fields before `ate-api-server` goes back, and `atelet` goes back only after every worker is back at N. Rolling back further than N is not supported. + +### Skipping releases + +Patch releases can be skipped: `x.y.1` can upgrade straight to `x.y.5`. + +Minor releases cannot: to go from `x.1` to `x.3`, upgrade to `x.2` first. The two-release steps and the database schema evolution both assume each upgrade moves one minor release. + +## Outside this policy + +The Kubernetes version, the database engine, the node OS, the CSI drivers and the actor images are not changed by the Agent Substrate upgrade. + +## Testing + +Before a release is cut, an installation at the previous compatible release is upgraded to it with actors running and rolled back, in the order above. + +## Exceptions + +As in the compatibility policy. diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 2d0c3c4635..f4823e1da4 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -178,7 +178,7 @@ usage() { echo " else the cluster's OIDC discovery document" echo " ACTOR_JWT_ALGORITHM Signing algorithm of a newly created actor JWT pool: ES256 (default) | RS256." echo " Use RS256 (a 4096-bit RSA key) for relying parties that don't support ES256. Has no effect once the pool exists." - echo " ACTOR_JWT_ISSUER iss claim of actor JWTs, verbatim. Relying parties fetch /.well-known/openid-configuration." + echo " ACTOR_JWT_ISSUER iss claim of actor JWTs, as https:// with no path. Relying parties fetch /.well-known/openid-configuration." echo " Default: https://idp..svc" echo "" echo "Authorization configuration:" diff --git a/hack/install-microvm-deps.sh b/hack/install-microvm-deps.sh index e37fd313ad..6c3f774e58 100755 --- a/hack/install-microvm-deps.sh +++ b/hack/install-microvm-deps.sh @@ -27,10 +27,6 @@ # `microvm` SandboxConfig referencing those assets. Every asset sha256 is # pinned in the manifest, so the apply only substitutes the bucket name. # -# ActorTemplates must reference the SandboxConfig explicitly via -# sandboxConfig.configName: microvm. This avoids a dirty teardown silently -# binding new templates to a stale config. -# # On --delete: removes the SandboxConfig from the cluster. The delete is best # effort. A SandboxConfig that a WorkerPool uses carries a protection # finalizer, which atecontroller releases only once no pool's status names @@ -225,4 +221,4 @@ sed -e "s|\${BUCKET_SCHEME}|${BUCKET_SCHEME}|g" \ "${MANIFEST_TEMPLATE}" \ | run_kubectl apply -f - -log "Done. ActorTemplates must reference this SandboxConfig by name (sandboxConfig.configName: microvm)." +log "Done. Installed cluster-wide microvm SandboxConfig." diff --git a/hack/render-manifests.sh b/hack/render-manifests.sh index 316445397b..8f207d612b 100755 --- a/hack/render-manifests.sh +++ b/hack/render-manifests.sh @@ -30,6 +30,7 @@ CHART_DIR="${ROOT}/charts/substrate" CHECK_MODE="false" PRESERVED_FILES=( ate-api-server.yaml + ate-idp-server.yaml ate-controller.yaml ate-otel-config.yaml ate-system-namespace.yaml @@ -147,6 +148,7 @@ fi mkdir -p "${OUT_DIR}" find "${OUT_DIR}" -maxdepth 1 -type f -name '*.yaml' \ ! -name 'ate-api-server.yaml' \ + ! -name 'ate-idp-server.yaml' \ ! -name 'ate-controller.yaml' \ ! -name 'ate-otel-config.yaml' \ ! -name 'ate-system-namespace.yaml' \ diff --git a/hack/run-e2e.sh b/hack/run-e2e.sh index 3c02ebe776..0667ab8e18 100755 --- a/hack/run-e2e.sh +++ b/hack/run-e2e.sh @@ -152,7 +152,7 @@ if [[ "${has_timeout_flag}" == "false" ]]; then fi # Assembled once so the two execution paths below cannot drift apart. -test_argv=(-v "$target_path") +test_argv=("$target_path") test_argv+=(${default_go_test_args[@]+"${default_go_test_args[@]}"}) test_argv+=(${go_test_args[@]+"${go_test_args[@]}"}) test_argv+=(-args --e2e) @@ -175,6 +175,8 @@ fi # report consumers, the JSON event stream for failure analysis. Unset, this is a # plain go test and gotestsum is never built, so a local run needs no toolchain # beyond go itself. +# NOTE: -v is only for plain go test, with -json it replaces the +# -test.v=test2json that marks t.Error output in the JSON if [[ -n "${E2E_JUNIT_FILE:-}" ]]; then mkdir -p "$(dirname "${E2E_JUNIT_FILE}")" # Claim the path before running, so a run killed part-way still owes a file @@ -188,5 +190,5 @@ if [[ -n "${E2E_JUNIT_FILE:-}" ]]; then -- "${test_argv[@]}" fi -exec go test "${test_argv[@]}" +exec go test -v "${test_argv[@]}" diff --git a/internal/ateattr/ateattr.go b/internal/ateattr/ateattr.go index 54e31b26e5..5f37b59d3a 100644 --- a/internal/ateattr/ateattr.go +++ b/internal/ateattr/ateattr.go @@ -310,12 +310,11 @@ const ( // Values for SnapshotKindKey, set by ateapi from its own resume branching, so // the label is bounded at the producer: Local restores an in-node snapshot, -// Latest pulls the actor's durable snapshot from object storage, Golden pulls the -// template's golden image, Boot is a from-scratch start (not a restore). +// Latest pulls a durable snapshot from object storage (including a template's +// golden snapshot), Boot is a from-scratch start (not a restore). // atelet derives the same values for its own histograms, where the kind is the // snapshot a restore reads or a checkpoint writes; Boot never appears there. const ( - SnapshotKindGolden = "golden" SnapshotKindLatest = "latest" SnapshotKindLocal = "local" SnapshotKindBoot = "boot" @@ -329,6 +328,18 @@ const ( SnapshotFidelityUnknown = "unknown" ) +// SnapshotUploadOverdueBandKey buckets how long past the upload delay an +// actor's undurable local snapshot has been waiting on atelet. +const SnapshotUploadOverdueBandKey = attribute.Key("ate.snapshot.upload_overdue_band") + +// Values for SnapshotUploadOverdueBandKey. +const ( + SnapshotUploadOverdueBandUnder1m = "under_1m" + SnapshotUploadOverdueBand1mTo5m = "1m_to_5m" + SnapshotUploadOverdueBand5mTo15m = "5m_to_15m" + SnapshotUploadOverdueBandOver15m = "over_15m" +) + // SnapshotFidelityValue maps the wire enum onto its label value, shared so // ateapi (which sets the fidelity) and atelet (which receives it) cannot // drift. An unrecognized fidelity reports as unknown rather than stringified, @@ -360,6 +371,7 @@ const ( // Persist is one step with two destinations (upload for external, rename // for local); SnapshotKindKey already says which. SnapshotPhasePersist = "persist" + SnapshotPhaseCleanup = "cleanup" SnapshotPhaseTotal = "total" ) @@ -382,7 +394,8 @@ func NormalizeSandboxClass(class string) string { // helper and not through SandboxClassKey. // // To omit the attribute while the class is unknown is a different choice, and -// two emitters make it: recordSchedulerAssignment and snapshotOp.attrs. +// three emitters make it: lifecycleOpAttrs, recordSchedulerAssignment and +// snapshotOp.attrs. func SandboxClassAttribute(class string) attribute.KeyValue { return SandboxClassKey.String(NormalizeSandboxClass(class)) } diff --git a/internal/ateattr/ateattr_test.go b/internal/ateattr/ateattr_test.go index 64773d9f1f..6810a486ed 100644 --- a/internal/ateattr/ateattr_test.go +++ b/internal/ateattr/ateattr_test.go @@ -391,7 +391,6 @@ func TestMetricLabelValues(t *testing.T) { {SchedulerOutcomeNoCapacity, "no_capacity"}, {SchedulerOutcomeError, "error"}, - {SnapshotKindGolden, "golden"}, {SnapshotKindLatest, "latest"}, {SnapshotKindLocal, "local"}, {SnapshotKindBoot, "boot"}, @@ -409,6 +408,7 @@ func TestMetricLabelValues(t *testing.T) { {SnapshotPhaseWorkerRestore, "worker_restore"}, {SnapshotPhaseWorkerCheckpoint, "worker_checkpoint"}, {SnapshotPhasePersist, "persist"}, + {SnapshotPhaseCleanup, "cleanup"}, {SnapshotPhaseTotal, "total"}, {SandboxClassUnknown, "unknown"}, diff --git a/internal/ateinterceptors/rejectunknown.go b/internal/ateinterceptors/rejectunknown.go index 4a6fd94789..a2f5ecfdb6 100644 --- a/internal/ateinterceptors/rejectunknown.go +++ b/internal/ateinterceptors/rejectunknown.go @@ -18,7 +18,7 @@ import ( "context" "strconv" - "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/resources" "google.golang.org/grpc" "google.golang.org/protobuf/encoding/protowire" "google.golang.org/protobuf/proto" @@ -33,7 +33,7 @@ import ( func RejectUnknownFieldsUnaryInterceptor(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { if msg, ok := req.(proto.Message); ok { if errs := findUnknownFields(msg); len(errs) > 0 { - return nil, apierror.InvalidArgument("%v", errs.ToAggregate()) + return nil, resources.ToAPIError(errs) } } return handler(ctx, req) diff --git a/internal/atelet/port.go b/internal/atelet/port.go index d89ad3adf4..e2a1a2c13e 100644 --- a/internal/atelet/port.go +++ b/internal/atelet/port.go @@ -16,5 +16,7 @@ package atelet const ( // DefaultPort is the port atelet's gRPC server listens on. - DefaultPort = 8085 + // It is also atelet's hostPort, so it is chosen to avoid common node ports, + // the NodePort range, and the ephemeral range. + DefaultPort = 21998 ) diff --git a/cmd/ateworker-microvm/sandboxnet.go b/internal/ateworker/actornet/etcfiles.go similarity index 56% rename from cmd/ateworker-microvm/sandboxnet.go rename to internal/ateworker/actornet/etcfiles.go index 298220f3d7..ba361bcad3 100644 --- a/cmd/ateworker-microvm/sandboxnet.go +++ b/internal/ateworker/actornet/etcfiles.go @@ -14,20 +14,26 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package actornet import ( "fmt" "os" - - "go.ate.dev/substrate/internal/ateworker/actornet" ) -// writeActorResolvConf points the guest resolver at its fixed gateway address. -func writeActorResolvConf(rootfs string) error { +// WriteRootfsEtcFiles writes the sandbox's /etc/resolv.conf and /etc/hosts +// into rootfs, pointing the resolver at the actor's gateway. +// +// rootfs must be the base layer under the actor's restored rootfs writes, so +// rewriting it on every activation keeps the actor's own edits. Writing these +// after restore instead would clobber them. +func WriteRootfsEtcFiles(rootfs, hostname string) error { pod, err := os.ReadFile("/etc/resolv.conf") if err != nil { return fmt.Errorf("reading the worker pod resolv.conf: %w", err) } - return actornet.WriteRootfsResolvConf(rootfs, actornet.SandboxResolvConf(actornet.ActorVethGateway, pod)) + if err := WriteRootfsResolvConf(rootfs, SandboxResolvConf(ActorVethGateway, pod)); err != nil { + return err + } + return WriteRootfsHosts(rootfs, SandboxHosts(hostname, ActorVethIP)) } diff --git a/internal/ateworker/actornet/resolvconf.go b/internal/ateworker/actornet/resolvconf.go index 32959d3171..10ee4023e8 100644 --- a/internal/ateworker/actornet/resolvconf.go +++ b/internal/ateworker/actornet/resolvconf.go @@ -41,12 +41,31 @@ func SandboxResolvConf(nameserver string, podResolvConf []byte) []byte { return []byte(out.String()) } +// SandboxHosts maps localhost to loopback and hostname to ip, the way a +// container runtime fills in /etc/hosts. +func SandboxHosts(hostname, ip string) []byte { + return []byte("# Agent Substrate-managed hosts file.\n" + + "127.0.0.1\tlocalhost\n" + + "::1\tlocalhost ip6-localhost ip6-loopback\n" + + ip + "\t" + hostname + "\n") +} + // WriteRootfsResolvConf installs content at /etc/resolv.conf inside rootfs. +func WriteRootfsResolvConf(rootfs string, content []byte) error { + return writeRootfsEtcFile(rootfs, "resolv.conf", content) +} + +// WriteRootfsHosts installs content at /etc/hosts inside rootfs. +func WriteRootfsHosts(rootfs string, content []byte) error { + return writeRootfsEtcFile(rootfs, "hosts", content) +} + +// writeRootfsEtcFile installs content at /etc/name inside rootfs. // // os.Root confines path traversal; unlinking prevents writes through existing links. -func WriteRootfsResolvConf(rootfs string, content []byte) error { +func writeRootfsEtcFile(rootfs, name string, content []byte) error { if len(content) == 0 { - return fmt.Errorf("refusing to write an empty resolv.conf") + return fmt.Errorf("refusing to write an empty %s", name) } root, err := os.OpenRoot(rootfs) if err != nil { @@ -56,19 +75,20 @@ func WriteRootfsResolvConf(rootfs string, content []byte) error { if err := root.Mkdir("etc", 0o755); err != nil && !errors.Is(err, fs.ErrExist) { return fmt.Errorf("creating %q: %w", filepath.Join(rootfs, "etc"), err) } - if err := root.Remove("etc/resolv.conf"); err != nil && !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("removing existing resolv.conf: %w", err) + path := "etc/" + name + if err := root.Remove(path); err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("removing existing %s: %w", name, err) } - f, err := root.OpenFile("etc/resolv.conf", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + f, err := root.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) if err != nil { - return fmt.Errorf("creating resolv.conf: %w", err) + return fmt.Errorf("creating %s: %w", name, err) } _, err = f.Write(content) if closeErr := f.Close(); err == nil { err = closeErr } if err != nil { - return fmt.Errorf("writing resolv.conf: %w", err) + return fmt.Errorf("writing %s: %w", name, err) } return nil } diff --git a/internal/ateworker/actornet/resolvconf_test.go b/internal/ateworker/actornet/resolvconf_test.go index 6d664e41ca..04a2eefa1d 100644 --- a/internal/ateworker/actornet/resolvconf_test.go +++ b/internal/ateworker/actornet/resolvconf_test.go @@ -108,3 +108,25 @@ func TestWriteRootfsResolvConfDoesNotFollowAPlantedSymlink(t *testing.T) { t.Errorf("the symlink was followed and the file outside the rootfs was overwritten with %q", victim) } } + +func TestWriteRootfsHosts(t *testing.T) { + rootfs := t.TempDir() + // Images built by docker ship an empty /etc/hosts. + if err := os.Mkdir(filepath.Join(rootfs, "etc"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(rootfs, "etc", "hosts"), nil, 0o644); err != nil { + t.Fatal(err) + } + if err := WriteRootfsHosts(rootfs, SandboxHosts("actor", "169.254.17.2")); err != nil { + t.Fatalf("WriteRootfsHosts: %v", err) + } + got, err := os.ReadFile(filepath.Join(rootfs, "etc", "hosts")) + if err != nil { + t.Fatalf("reading what was written: %v", err) + } + want := "# Agent Substrate-managed hosts file.\n127.0.0.1\tlocalhost\n::1\tlocalhost ip6-localhost ip6-loopback\n169.254.17.2\tactor\n" + if string(got) != want { + t.Errorf("wrote %q, want %q", got, want) + } +} diff --git a/internal/ateworker/register/register_test.go b/internal/ateworker/register/register_test.go index ab63b289cb..d04166998e 100644 --- a/internal/ateworker/register/register_test.go +++ b/internal/ateworker/register/register_test.go @@ -80,9 +80,9 @@ func TestFromFilesMissing(t *testing.T) { func TestBuildRuntime(t *testing.T) { got := buildRuntime("gvisor", sandboxruntimes.Attributes{ - hardware.AttrSandboxRuntimeHash: "sha256:abc", - hardware.AttrCPUFeatures: "3f", - hardware.AttrArchitecture: runtime.GOARCH, + hardware.AttrSandboxBundleHash: "sha256:abc", + hardware.AttrCPUFeatures: "3f", + hardware.AttrArchitecture: runtime.GOARCH, }) want := &ateletpb.SandboxRuntime{ SandboxClass: "gvisor", @@ -91,7 +91,7 @@ func TestBuildRuntime(t *testing.T) { Attributes: []*ateletpb.AttributeEntry{ {Key: hardware.AttrArchitecture, Value: runtime.GOARCH}, {Key: hardware.AttrCPUFeatures, Value: "3f"}, - {Key: hardware.AttrSandboxRuntimeHash, Value: "sha256:abc"}, + {Key: hardware.AttrSandboxBundleHash, Value: "sha256:abc"}, }, }, } @@ -104,16 +104,16 @@ func TestReconcilerSync(t *testing.T) { capacity := &ateletpb.WorkerResources{Actors: testActors} v1 := &sandboxruntimes.File{ DefaultRuntime: sandboxruntimes.Attributes{ - hardware.AttrArchitecture: runtime.GOARCH, - hardware.AttrCPUFeatures: "3", - hardware.AttrSandboxRuntimeHash: "sha256:v1", + hardware.AttrArchitecture: runtime.GOARCH, + hardware.AttrCPUFeatures: "3", + hardware.AttrSandboxBundleHash: "sha256:v1", }, } v2 := &sandboxruntimes.File{ DefaultRuntime: sandboxruntimes.Attributes{ - hardware.AttrArchitecture: runtime.GOARCH, - hardware.AttrCPUFeatures: "7", - hardware.AttrSandboxRuntimeHash: "sha256:v2", + hardware.AttrArchitecture: runtime.GOARCH, + hardware.AttrCPUFeatures: "7", + hardware.AttrSandboxBundleHash: "sha256:v2", }, RestorableRuntimes: []sandboxruntimes.Attributes{v1.DefaultRuntime}, } @@ -242,7 +242,7 @@ func TestReconcilerRunBackoff(t *testing.T) { defer cancel() file := &sandboxruntimes.File{ - DefaultRuntime: sandboxruntimes.Attributes{hardware.AttrSandboxRuntimeHash: "sha256:v1"}, + DefaultRuntime: sandboxruntimes.Attributes{hardware.AttrSandboxBundleHash: "sha256:v1"}, } changes := make(chan struct{}, 1) @@ -269,7 +269,7 @@ func TestReconcilerRunBackoff(t *testing.T) { case 4: // Succeed on attempt 4, then trigger a file update on changes. file = &sandboxruntimes.File{ - DefaultRuntime: sandboxruntimes.Attributes{hardware.AttrSandboxRuntimeHash: "sha256:v2"}, + DefaultRuntime: sandboxruntimes.Attributes{hardware.AttrSandboxBundleHash: "sha256:v2"}, } changes <- struct{}{} return nil @@ -321,7 +321,7 @@ func TestReconcilerIgnoresChangesDuringBackoff(t *testing.T) { defer cancel() file := &sandboxruntimes.File{ - DefaultRuntime: sandboxruntimes.Attributes{hardware.AttrSandboxRuntimeHash: "sha256:v1"}, + DefaultRuntime: sandboxruntimes.Attributes{hardware.AttrSandboxBundleHash: "sha256:v1"}, } changes := make(chan struct{}, 1) // started reports each registration attempt as it begins; release lets @@ -409,7 +409,7 @@ func TestReconcilerRunResync(t *testing.T) { cancel() } return &sandboxruntimes.File{ - DefaultRuntime: sandboxruntimes.Attributes{hardware.AttrSandboxRuntimeHash: "sha256:v1"}, + DefaultRuntime: sandboxruntimes.Attributes{hardware.AttrSandboxBundleHash: "sha256:v1"}, }, nil }, func(context.Context, *ateletpb.RegisterWorkerRequest) error { return nil }, @@ -548,9 +548,9 @@ func TestReconcilerRunWatchesFile(t *testing.T) { // before atelet has reconciled the pod. v1 := &sandboxruntimes.File{ DefaultRuntime: sandboxruntimes.Attributes{ - hardware.AttrArchitecture: runtime.GOARCH, - hardware.AttrCPUFeatures: "3", - hardware.AttrSandboxRuntimeHash: "sha256:v1", + hardware.AttrArchitecture: runtime.GOARCH, + hardware.AttrCPUFeatures: "3", + hardware.AttrSandboxBundleHash: "sha256:v1", }, } if _, err := sandboxruntimes.Write(path, v1); err != nil { @@ -569,9 +569,9 @@ func TestReconcilerRunWatchesFile(t *testing.T) { // second registration. v2 := &sandboxruntimes.File{ DefaultRuntime: sandboxruntimes.Attributes{ - hardware.AttrArchitecture: runtime.GOARCH, - hardware.AttrCPUFeatures: "7", - hardware.AttrSandboxRuntimeHash: "sha256:v2", + hardware.AttrArchitecture: runtime.GOARCH, + hardware.AttrCPUFeatures: "7", + hardware.AttrSandboxBundleHash: "sha256:v2", }, RestorableRuntimes: []sandboxruntimes.Attributes{v1.DefaultRuntime}, } diff --git a/internal/ateworkervalidation/doc.go b/internal/ateworkervalidation/doc.go new file mode 100644 index 0000000000..7291ba468e --- /dev/null +++ b/internal/ateworkervalidation/doc.go @@ -0,0 +1,28 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ateworkervalidation holds the declarative validation generated for +// the Worker RPC surface. It is a separate package, unlike controlapi where +// the generated code lives with the handlers, because both worker binaries +// (ateworker-gvisor and ateworker-microvm) implement the service in package +// main and share one set of rules. +// +// Kubernetes codegen tools required this to be in doc.go, no other name will +// work. +// +// +k8s:validation-gen=TypesWithSuffix=Request +// +k8s:validation-gen-input=go.ate.dev/substrate/internal/proto/ateworkerpb +// +k8s:validation-gen-scheme-registry=nil +// +k8s:validation-gen-deep-equal-func=ateDeepEqual +package ateworkervalidation diff --git a/internal/ateworkervalidation/validate.go b/internal/ateworkervalidation/validate.go new file mode 100644 index 0000000000..a90526a270 --- /dev/null +++ b/internal/ateworkervalidation/validate.go @@ -0,0 +1,154 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ateworkervalidation + +import ( + "context" + + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/resources" + "k8s.io/apimachinery/pkg/api/operation" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +// Each wrapper validates its request at the RPC edge. A non-nil return is the +// InvalidArgument error the handler responds with, built by +// resources.ToAPIError. Requests are validated as creates: no Worker RPC +// updates prior state, so there is nothing to ratchet against. + +// ValidateRunWorkloadRequest validates req at the RPC edge. +func ValidateRunWorkloadRequest(ctx context.Context, req *ateworkerpb.RunWorkloadRequest) error { + if errs := Validate_RunWorkloadRequest(ctx, createOp(), nil, req, nil); len(errs) > 0 { + return resources.ToAPIError(errs) + } + return nil +} + +// ValidateCheckpointWorkloadRequest validates req at the RPC edge. +func ValidateCheckpointWorkloadRequest(ctx context.Context, req *ateworkerpb.CheckpointWorkloadRequest) error { + if errs := Validate_CheckpointWorkloadRequest(ctx, createOp(), nil, req, nil); len(errs) > 0 { + return resources.ToAPIError(errs) + } + return nil +} + +// ValidateRestoreWorkloadRequest validates req at the RPC edge. +func ValidateRestoreWorkloadRequest(ctx context.Context, req *ateworkerpb.RestoreWorkloadRequest) error { + if errs := Validate_RestoreWorkloadRequest(ctx, createOp(), nil, req, nil); len(errs) > 0 { + return resources.ToAPIError(errs) + } + return nil +} + +// ValidateTerminateWorkloadRequest validates req at the RPC edge. +func ValidateTerminateWorkloadRequest(ctx context.Context, req *ateworkerpb.TerminateWorkloadRequest) error { + if errs := Validate_TerminateWorkloadRequest(ctx, createOp(), nil, req, nil); len(errs) > 0 { + return resources.ToAPIError(errs) + } + return nil +} + +// ValidateGetWorkloadStatsRequest validates req at the RPC edge. +func ValidateGetWorkloadStatsRequest(ctx context.Context, req *ateworkerpb.GetWorkloadStatsRequest) error { + if errs := Validate_GetWorkloadStatsRequest(ctx, createOp(), nil, req, nil); len(errs) > 0 { + return resources.ToAPIError(errs) + } + return nil +} + +// GetActiveWorkloadStatsRequest has no fields, so validation-gen emits +// nothing for it and there is no wrapper. + +// ValidateCustom_EgressGateway_Address holds the gateway address to the +// host:port shape the tunnel dials. The rule is atelet's, shared through +// internal/resources, so the two ends of the RPC agree on what is dialable. +func ValidateCustom_EgressGateway_Address(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { + return resources.ValidateHostPort(fldPath, *value) +} + +// ValidateCustom_HttpGetAction_Path holds a wakeup probe's path to the RFC +// 3986 path shape atelet already enforces, shared through internal/resources. +func ValidateCustom_HttpGetAction_Path(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { + return resources.ValidateHTTPGetPath(fldPath, *value) +} + +// ValidateCustom_VolumeMount_MountPath holds a mount path to the clean +// absolute Unix path shape atelet already enforces, shared through +// internal/resources. +func ValidateCustom_VolumeMount_MountPath(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { + return resources.ValidateMountPath(fldPath, *value) +} + +// ValidateCustom_Container holds a container's mount paths unique and +// non-nesting across its four mount lists, as atelet does for its single +// list: volumes cannot mount onto other volumes. A repeat within one list is +// the list key's finding and is not reported again here. +func ValidateCustom_Container(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *ateworkerpb.Container) field.ErrorList { + type mount struct { + path, list string + fld *field.Path + } + var mounts []mount + add := func(list string, i int, path string) { + mounts = append(mounts, mount{path: path, list: list, fld: fldPath.Child(list).Index(i).Child("mount_path")}) + } + for i, m := range value.GetDurableDirVolumeMounts() { + add("durable_dir_volume_mounts", i, m.GetMountPath()) + } + for i, m := range value.GetExternalVolumeMounts() { + add("external_volume_mounts", i, m.GetMountPath()) + } + for i, m := range value.GetSystemInfoVolumeMounts() { + add("system_info_volume_mounts", i, m.GetMountPath()) + } + for i, m := range value.GetImageVolumeMounts() { + add("image_volume_mounts", i, m.GetMountPath()) + } + + var errs field.ErrorList + firstList := make(map[string]string, len(mounts)) + paths := make([]string, len(mounts)) + for i, m := range mounts { + paths[i] = m.path + if m.path == "" { + continue + } + if list, seen := firstList[m.path]; seen { + if list != m.list { + errs = append(errs, field.Duplicate(m.fld, m.path)) + } + continue + } + firstList[m.path] = m.list + } + return append(errs, resources.ValidateNestedMountPathsAt(paths, func(i int) *field.Path { return mounts[i].fld })...) +} + +// ValidateCustom_ActorDirs holds every directory to an absolute, clean path. +// Presence and length are the tags' job. The rule is the one both worker +// binaries applied by hand before, shared through internal/resources. +func ValidateCustom_ActorDirs(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *ateworkerpb.ActorDirs) field.ErrorList { + return resources.ValidateActorDirs(value, fldPath) +} + +func createOp() operation.Operation { + return operation.Operation{Type: operation.Create} +} + +// ateDeepEqual is the deep-equal function declarative validation's generated +// code calls by name; it delegates to resources.DeepEqual. +func ateDeepEqual[T any](a, b T) bool { + return resources.DeepEqual(a, b) +} diff --git a/internal/ateworkervalidation/validation_test.go b/internal/ateworkervalidation/validation_test.go new file mode 100644 index 0000000000..4144941827 --- /dev/null +++ b/internal/ateworkervalidation/validation_test.go @@ -0,0 +1,832 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ateworkervalidation + +import ( + "context" + "fmt" + "strings" + "testing" + + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "google.golang.org/grpc/codes" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +// Every request is validated down to its leaves: identity, paths, the spec's +// containers, their probes and mounts, and the egress gateway. + +// maxContainers is the +k8s:maxItems bound on WorkloadSpec.containers. +const maxContainers = 10 + +func fullSpec() *ateworkerpb.WorkloadSpec { + return &ateworkerpb.WorkloadSpec{Containers: []*ateworkerpb.Container{{ + Name: "main", + WakeupProbe: &ateworkerpb.WakeupProbe{HttpGet: &ateworkerpb.HttpGetAction{Path: "/healthz", Port: 8080}, TimeoutSeconds: 30}, + DurableDirVolumeMounts: []*ateworkerpb.VolumeMount{{Name: "data", MountPath: "/data"}}, + ExternalVolumeMounts: []*ateworkerpb.VolumeMount{{Name: "ext", MountPath: "/ext"}}, + SystemInfoVolumeMounts: []*ateworkerpb.VolumeMount{{Name: "sysinfo", MountPath: "/etc/sysinfo"}}, + ImageVolumeMounts: []*ateworkerpb.VolumeMount{{Name: "img", MountPath: "/img"}}, + }}} +} + +func fullActorDirs() *ateworkerpb.ActorDirs { + return &ateworkerpb.ActorDirs{ + RootDir: "/actors/uid", + OciBundleDir: "/actors/uid/bundles", + CheckpointDir: "/actors/uid/checkpoint", + RestoreDir: "/actors/uid/restore", + DurableDirVolumeMountsDir: "/actors/uid/durable", + SystemInfoVolumeRootsDir: "/actors/uid/sysinfo", + ExternalVolumeMountsDir: "/actors/uid/volumes", + SandboxAssetsDir: "/actors/uid/sandbox-assets", + } +} + +func validRunWorkloadRequest(mutate ...func(*ateworkerpb.RunWorkloadRequest)) *ateworkerpb.RunWorkloadRequest { + r := &ateworkerpb.RunWorkloadRequest{ + Atespace: "team-a", + ActorName: "actor-1", + ActorUid: "01234567-89ab-cdef-0123-456789abcdef", + ActorTemplateAtespace: "team-a", + ActorTemplateName: "template-1", + RunscPath: "/opt/runsc", + Spec: fullSpec(), + RuntimeAssetPaths: map[string]string{"kata-kernel": "/opt/kernel", "virtiofsd": "/opt/virtiofsd"}, + EgressGateway: &ateworkerpb.EgressGateway{Address: "gateway:443"}, + CpuMilli: 1500, + MemoryBytes: 1 << 30, + ActorDirs: fullActorDirs(), + } + for _, m := range mutate { + m(r) + } + return r +} + +func TestValidateRunWorkloadRequest(t *testing.T) { + valid := validRunWorkloadRequest + assets := field.NewPath("runtime_asset_paths") + probe := field.NewPath("spec", "containers").Index(0).Child("wakeup_probe") + + tests := []struct { + name string + obj *ateworkerpb.RunWorkloadRequest + want field.ErrorList + }{{ + name: "valid", + obj: valid(), + }, { + name: "valid: gVisor shape, no assets, no egress, unset size", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { + r.RuntimeAssetPaths, r.EgressGateway, r.CpuMilli, r.MemoryBytes = nil, nil, 0, 0 + }), + }, { + name: "missing atespace", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { r.Atespace = "" }), + want: field.ErrorList{field.Required(field.NewPath("atespace"), "")}, + }, { + name: "invalid atespace: uppercase", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { r.Atespace = "Team-A" }), + want: field.ErrorList{field.Invalid(field.NewPath("atespace"), nil, "").WithOrigin("format=k8s-short-name")}, + }, { + name: "missing actor_name", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { r.ActorName = "" }), + want: field.ErrorList{field.Required(field.NewPath("actor_name"), "")}, + }, { + name: "missing actor_uid", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { r.ActorUid = "" }), + want: field.ErrorList{field.Required(field.NewPath("actor_uid"), "")}, + }, { + name: "invalid actor_uid: not a uuid", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { r.ActorUid = "uid-a" }), + want: field.ErrorList{field.Invalid(field.NewPath("actor_uid"), nil, "").WithOrigin("format=k8s-uuid")}, + }, { + name: "invalid actor_template_name: underscore", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { r.ActorTemplateName = "template_1" }), + want: field.ErrorList{field.Invalid(field.NewPath("actor_template_name"), nil, "").WithOrigin("format=k8s-short-name")}, + }, { + name: "runsc_path too long", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { r.RunscPath = "/" + strings.Repeat("x", 4096) }), + want: field.ErrorList{field.TooLong(field.NewPath("runsc_path"), nil, 4096).WithOrigin("maxLength")}, + }, { + name: "missing spec", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { r.Spec = nil }), + want: field.ErrorList{field.Required(field.NewPath("spec"), "")}, + }, { + name: "too many runtime_asset_paths", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { + r.RuntimeAssetPaths = map[string]string{} + for i := range 17 { + r.RuntimeAssetPaths[fmt.Sprintf("asset-%d", i)] = "/opt/asset" + } + }), + want: field.ErrorList{field.TooMany(assets, 17, 16).WithOrigin("maxProperties")}, + }, { + name: "runtime asset name too long", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { + r.RuntimeAssetPaths = map[string]string{strings.Repeat("k", 65): "/opt/asset"} + }), + want: field.ErrorList{field.TooLong(assets, nil, 64).WithOrigin("maxLength")}, // keys are reported at the map + }, { + name: "runtime asset path too long", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { + r.RuntimeAssetPaths = map[string]string{"kata-kernel": "/" + strings.Repeat("x", 4096)} + }), + want: field.ErrorList{field.TooLong(assets.Key("kata-kernel"), nil, 4096).WithOrigin("maxLength")}, + }, { + name: "negative cpu_milli", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { r.CpuMilli = -1 }), + want: field.ErrorList{field.Invalid(field.NewPath("cpu_milli"), nil, "").WithOrigin("minimum")}, + }, { + name: "cpu_milli past atelet's bound", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { r.CpuMilli = 1000000 }), + want: field.ErrorList{field.Invalid(field.NewPath("cpu_milli"), nil, "").WithOrigin("maximum")}, + }, { + name: "negative memory_bytes", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { r.MemoryBytes = -1 }), + want: field.ErrorList{field.Invalid(field.NewPath("memory_bytes"), nil, "").WithOrigin("minimum")}, + }, { + name: "empty egress_gateway", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { r.EgressGateway = &ateworkerpb.EgressGateway{} }), + want: field.ErrorList{field.Required(field.NewPath("egress_gateway", "address"), "")}, + }, { + name: "egress_gateway address without port", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { r.EgressGateway.Address = "gateway" }), + want: field.ErrorList{field.Invalid(field.NewPath("egress_gateway", "address"), nil, "")}, + }, { + name: "egress_gateway address too long", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { r.EgressGateway.Address = strings.Repeat("h", 258) + ":443" }), + want: field.ErrorList{ + field.TooLong(field.NewPath("egress_gateway", "address"), nil, 261).WithOrigin("maxLength"), + field.Invalid(field.NewPath("egress_gateway", "address"), nil, ""), + }, + }, { + name: "missing actor_dirs", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { r.ActorDirs = nil }), + want: field.ErrorList{field.Required(field.NewPath("actor_dirs"), "")}, + }, { + name: "relative oci_bundle_dir", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { r.ActorDirs.OciBundleDir = "bundles" }), + want: field.ErrorList{field.Invalid(field.NewPath("actor_dirs", "oci_bundle_dir"), nil, "")}, + }, { + name: "spec without containers", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { r.Spec = &ateworkerpb.WorkloadSpec{} }), + want: field.ErrorList{field.Required(field.NewPath("spec", "containers"), "")}, + }, { + name: "bad container name", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { r.Spec.Containers[0].Name = "Main" }), + want: field.ErrorList{field.Invalid(field.NewPath("spec", "containers").Index(0).Child("name"), nil, "").WithOrigin("format=k8s-short-name")}, + }, { + name: "valid: no wakeup probe", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { r.Spec.Containers[0].WakeupProbe = nil }), + }, { + name: "empty wakeup probe", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { r.Spec.Containers[0].WakeupProbe = &ateworkerpb.WakeupProbe{} }), + want: field.ErrorList{ + field.Required(probe.Child("http_get"), ""), + field.Required(probe.Child("timeout_seconds"), ""), + }, + }, { + name: "wakeup probe timeout too long", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { r.Spec.Containers[0].WakeupProbe.TimeoutSeconds = 3601 }), + want: field.ErrorList{field.Invalid(probe.Child("timeout_seconds"), nil, "").WithOrigin("maximum")}, + }, { + name: "empty http_get", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { + r.Spec.Containers[0].WakeupProbe.HttpGet = &ateworkerpb.HttpGetAction{} + }), + want: field.ErrorList{ + field.Required(probe.Child("http_get", "path"), ""), + field.Required(probe.Child("http_get", "port"), ""), + }, + }, { + name: "http_get path without leading slash", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { r.Spec.Containers[0].WakeupProbe.HttpGet.Path = "healthz" }), + want: field.ErrorList{field.Invalid(probe.Child("http_get", "path"), nil, "")}, + }, { + name: "http_get path with query", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { + r.Spec.Containers[0].WakeupProbe.HttpGet.Path = "/healthz?x=1" + }), + want: field.ErrorList{field.Invalid(probe.Child("http_get", "path"), nil, "")}, + }, { + name: "http_get path too long", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { + r.Spec.Containers[0].WakeupProbe.HttpGet.Path = "/" + strings.Repeat("x", 1024) + }), + want: field.ErrorList{field.TooLong(probe.Child("http_get", "path"), nil, 1024).WithOrigin("maxLength")}, + }, { + name: "http_get port past the range", + obj: valid(func(r *ateworkerpb.RunWorkloadRequest) { r.Spec.Containers[0].WakeupProbe.HttpGet.Port = 65536 }), + want: field.ErrorList{field.Invalid(probe.Child("http_get", "port"), nil, "").WithOrigin("maximum")}, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + matcher := field.ErrorMatcher{}.ByType().ByField().ByOrigin() + matcher.Test(t, tt.want, Validate_RunWorkloadRequest(context.Background(), createOp(), nil, tt.obj, nil)) + }) + } +} + +// TestValidateRunWorkloadRequestEdge covers the handler-facing wrapper: valid +// passes, invalid comes back as InvalidArgument. +func TestValidateRunWorkloadRequestEdge(t *testing.T) { + if err := ValidateRunWorkloadRequest(context.Background(), validRunWorkloadRequest()); err != nil { + t.Fatalf("valid request rejected: %v", err) + } + err := ValidateRunWorkloadRequest(context.Background(), &ateworkerpb.RunWorkloadRequest{}) + if apierror.Code(err) != codes.InvalidArgument { + t.Fatalf("empty request error = %v, want InvalidArgument", err) + } +} + +func validCheckpointWorkloadRequest(mutate ...func(*ateworkerpb.CheckpointWorkloadRequest)) *ateworkerpb.CheckpointWorkloadRequest { + r := &ateworkerpb.CheckpointWorkloadRequest{ + Atespace: "team-a", + ActorName: "actor-1", + ActorUid: "01234567-89ab-cdef-0123-456789abcdef", + ActorTemplateAtespace: "team-a", + ActorTemplateName: "template-1", + RunscPath: "/opt/runsc", + Spec: fullSpec(), + Fidelity: ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + ActorDirs: fullActorDirs(), + } + for _, m := range mutate { + m(r) + } + return r +} + +// Checkpoint shares its identity fields with Run, which the Run table +// covers; this table holds the shared gates at one case each and covers what +// only Checkpoint has: fidelity and actor_dirs. +func TestValidateCheckpointWorkloadRequest(t *testing.T) { + valid := validCheckpointWorkloadRequest + + tests := []struct { + name string + obj *ateworkerpb.CheckpointWorkloadRequest + want field.ErrorList + }{{ + name: "valid", + obj: valid(), + }, { + name: "valid: volumes fidelity", + obj: valid(func(r *ateworkerpb.CheckpointWorkloadRequest) { + r.Fidelity = ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES + }), + }, { + name: "valid: rootfs fidelity, which runtimes judge for themselves", + obj: valid(func(r *ateworkerpb.CheckpointWorkloadRequest) { + r.Fidelity = ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS + }), + }, { + name: "missing actor_uid", + obj: valid(func(r *ateworkerpb.CheckpointWorkloadRequest) { r.ActorUid = "" }), + want: field.ErrorList{field.Required(field.NewPath("actor_uid"), "")}, + }, { + name: "invalid actor_name: trailing dash", + obj: valid(func(r *ateworkerpb.CheckpointWorkloadRequest) { r.ActorName = "actor-" }), + want: field.ErrorList{field.Invalid(field.NewPath("actor_name"), nil, "").WithOrigin("format=k8s-short-name")}, + }, { + name: "runsc_path too long", + obj: valid(func(r *ateworkerpb.CheckpointWorkloadRequest) { r.RunscPath = "/" + strings.Repeat("x", 4096) }), + want: field.ErrorList{field.TooLong(field.NewPath("runsc_path"), nil, 4096).WithOrigin("maxLength")}, + }, { + name: "missing spec", + obj: valid(func(r *ateworkerpb.CheckpointWorkloadRequest) { r.Spec = nil }), + want: field.ErrorList{field.Required(field.NewPath("spec"), "")}, + }, { + name: "missing fidelity", + obj: valid(func(r *ateworkerpb.CheckpointWorkloadRequest) { + r.Fidelity = ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED + }), + want: field.ErrorList{field.Required(field.NewPath("fidelity"), "")}, + }, { + name: "fidelity past the enum", + obj: valid(func(r *ateworkerpb.CheckpointWorkloadRequest) { r.Fidelity = ateworkerpb.SnapshotFidelity(4) }), + want: field.ErrorList{field.Invalid(field.NewPath("fidelity"), nil, "").WithOrigin("maximum")}, + }, { + name: "missing actor_dirs", + obj: valid(func(r *ateworkerpb.CheckpointWorkloadRequest) { r.ActorDirs = nil }), + want: field.ErrorList{field.Required(field.NewPath("actor_dirs"), "")}, + }, { + name: "relative checkpoint_dir", + obj: valid(func(r *ateworkerpb.CheckpointWorkloadRequest) { r.ActorDirs.CheckpointDir = "checkpoint" }), + want: field.ErrorList{field.Invalid(field.NewPath("actor_dirs", "checkpoint_dir"), nil, "")}, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + matcher := field.ErrorMatcher{}.ByType().ByField().ByOrigin() + matcher.Test(t, tt.want, Validate_CheckpointWorkloadRequest(context.Background(), createOp(), nil, tt.obj, nil)) + }) + } +} + +// TestValidateCheckpointWorkloadRequestEdge covers the handler-facing +// wrapper: valid passes, invalid comes back as InvalidArgument. +func TestValidateCheckpointWorkloadRequestEdge(t *testing.T) { + if err := ValidateCheckpointWorkloadRequest(context.Background(), validCheckpointWorkloadRequest()); err != nil { + t.Fatalf("valid request rejected: %v", err) + } + err := ValidateCheckpointWorkloadRequest(context.Background(), &ateworkerpb.CheckpointWorkloadRequest{}) + if apierror.Code(err) != codes.InvalidArgument { + t.Fatalf("empty request error = %v, want InvalidArgument", err) + } +} + +// WorkloadSpec is reached from all four lifecycle requests; the Run table +// carries one case to show the path prefix, and this table holds the rules. +func TestValidateWorkloadSpec(t *testing.T) { + container := func(name string) *ateworkerpb.Container { return &ateworkerpb.Container{Name: name} } + tests := []struct { + name string + obj *ateworkerpb.WorkloadSpec + want field.ErrorList + }{{ + name: "valid", + obj: fullSpec(), + }, { + name: "valid: at the containers bound", + obj: func() *ateworkerpb.WorkloadSpec { + spec := &ateworkerpb.WorkloadSpec{} + for i := range maxContainers { + spec.Containers = append(spec.Containers, container(fmt.Sprintf("c-%d", i))) + } + return spec + }(), + }, { + name: "no containers: Terminate's fallback spec for a template-less actor", + obj: &ateworkerpb.WorkloadSpec{}, + }, { + name: "over the containers bound", + obj: func() *ateworkerpb.WorkloadSpec { + spec := &ateworkerpb.WorkloadSpec{} + for i := range maxContainers + 1 { + spec.Containers = append(spec.Containers, container(fmt.Sprintf("c-%d", i))) + } + return spec + }(), + want: field.ErrorList{field.TooMany(field.NewPath("containers"), maxContainers+1, maxContainers).WithOrigin("maxItems")}, + }, { + name: "duplicate container name", + obj: &ateworkerpb.WorkloadSpec{Containers: []*ateworkerpb.Container{container("app"), container("app")}}, + want: field.ErrorList{field.Duplicate(field.NewPath("containers").Index(1), nil)}, + }, { + name: "nil container", + obj: &ateworkerpb.WorkloadSpec{Containers: []*ateworkerpb.Container{nil}}, + want: field.ErrorList{field.Required(field.NewPath("containers").Index(0), "")}, + }, { + name: "bad name inside the list", + obj: &ateworkerpb.WorkloadSpec{Containers: []*ateworkerpb.Container{container("app"), container("Sidecar")}}, + want: field.ErrorList{field.Invalid(field.NewPath("containers").Index(1).Child("name"), nil, "").WithOrigin("format=k8s-short-name")}, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + matcher := field.ErrorMatcher{}.ByType().ByField().ByOrigin() + matcher.Test(t, tt.want, Validate_WorkloadSpec(context.Background(), createOp(), nil, tt.obj, nil)) + }) + } +} + +// The list rules (no nil entries, at most 32, unique mount paths within a +// list) come from the tags; the items' own rules and the cross-list nesting +// and duplicate checks are covered below. The probe is covered from the Run +// table. +func TestValidateContainer(t *testing.T) { + valid := func(mutate ...func(*ateworkerpb.Container)) *ateworkerpb.Container { + c := fullSpec().Containers[0] + for _, m := range mutate { + m(c) + } + return c + } + tests := []struct { + name string + obj *ateworkerpb.Container + want field.ErrorList + }{{ + name: "valid", + obj: valid(), + }, { + name: "valid: name only", + obj: &ateworkerpb.Container{Name: "app"}, + }, { + name: "missing name", + obj: valid(func(c *ateworkerpb.Container) { c.Name = "" }), + want: field.ErrorList{field.Required(field.NewPath("name"), "")}, + }, { + name: "invalid name: the sandbox's own container", + obj: valid(func(c *ateworkerpb.Container) { c.Name = "_pause" }), + want: field.ErrorList{field.Invalid(field.NewPath("name"), nil, "").WithOrigin("format=k8s-short-name")}, + }, { + name: "too many external mounts", + obj: valid(func(c *ateworkerpb.Container) { + c.ExternalVolumeMounts = nil + for i := range 33 { + c.ExternalVolumeMounts = append(c.ExternalVolumeMounts, &ateworkerpb.VolumeMount{Name: "v", MountPath: fmt.Sprintf("/m/%d", i)}) + } + }), + want: field.ErrorList{field.TooMany(field.NewPath("external_volume_mounts"), 33, 32).WithOrigin("maxItems")}, + }, { + name: "duplicate durable-dir mount path", + obj: valid(func(c *ateworkerpb.Container) { + c.DurableDirVolumeMounts = []*ateworkerpb.VolumeMount{{Name: "a", MountPath: "/data"}, {Name: "b", MountPath: "/data"}} + }), + want: field.ErrorList{field.Duplicate(field.NewPath("durable_dir_volume_mounts").Index(1), nil)}, + }, { + name: "nil image mount", + obj: valid(func(c *ateworkerpb.Container) { c.ImageVolumeMounts = []*ateworkerpb.VolumeMount{nil} }), + want: field.ErrorList{field.Required(field.NewPath("image_volume_mounts").Index(0), "")}, + }, { + name: "valid: no mounts", + obj: valid(func(c *ateworkerpb.Container) { + c.DurableDirVolumeMounts, c.ExternalVolumeMounts, c.SystemInfoVolumeMounts, c.ImageVolumeMounts = nil, nil, nil, nil + }), + }, { + name: "empty system-info mount", + obj: valid(func(c *ateworkerpb.Container) { c.SystemInfoVolumeMounts = []*ateworkerpb.VolumeMount{{}} }), + want: field.ErrorList{ + field.Required(field.NewPath("system_info_volume_mounts").Index(0).Child("name"), ""), + field.Required(field.NewPath("system_info_volume_mounts").Index(0).Child("mount_path"), ""), + }, + }, { + name: "invalid external mount name: uppercase", + obj: valid(func(c *ateworkerpb.Container) { c.ExternalVolumeMounts[0].Name = "EXT" }), + want: field.ErrorList{field.Invalid(field.NewPath("external_volume_mounts").Index(0).Child("name"), nil, "").WithOrigin("format=k8s-short-name")}, + }, { + name: "relative durable-dir mount_path", + obj: valid(func(c *ateworkerpb.Container) { c.DurableDirVolumeMounts[0].MountPath = "data" }), + want: field.ErrorList{field.Invalid(field.NewPath("durable_dir_volume_mounts").Index(0).Child("mount_path"), nil, "")}, + }, { + name: "image mount_path with dot-dot", + obj: valid(func(c *ateworkerpb.Container) { c.ImageVolumeMounts[0].MountPath = "/img/../etc" }), + want: field.ErrorList{field.Invalid(field.NewPath("image_volume_mounts").Index(0).Child("mount_path"), nil, "")}, + }, { + name: "image mount_path too long", + obj: valid(func(c *ateworkerpb.Container) { c.ImageVolumeMounts[0].MountPath = "/" + strings.Repeat("x", 4096) }), + want: field.ErrorList{field.TooLong(field.NewPath("image_volume_mounts").Index(0).Child("mount_path"), nil, 4096).WithOrigin("maxLength")}, + }, { + name: "external mount nested under another in its list", + obj: valid(func(c *ateworkerpb.Container) { + c.ExternalVolumeMounts = append(c.ExternalVolumeMounts, &ateworkerpb.VolumeMount{Name: "ext-2", MountPath: "/ext/inner"}) + }), + want: field.ErrorList{field.Invalid(field.NewPath("external_volume_mounts").Index(1).Child("mount_path"), nil, "")}, + }, { + name: "system-info mount nested under an external mount", + obj: valid(func(c *ateworkerpb.Container) { + c.SystemInfoVolumeMounts = append(c.SystemInfoVolumeMounts, &ateworkerpb.VolumeMount{Name: "sysinfo-2", MountPath: "/ext/inner"}) + }), + want: field.ErrorList{field.Invalid(field.NewPath("system_info_volume_mounts").Index(1).Child("mount_path"), nil, "")}, + }, { + name: "image mount nested over a system-info mount", + obj: valid(func(c *ateworkerpb.Container) { + c.ImageVolumeMounts = append(c.ImageVolumeMounts, &ateworkerpb.VolumeMount{Name: "etc", MountPath: "/etc"}) + }), + want: field.ErrorList{field.Invalid(field.NewPath("image_volume_mounts").Index(1).Child("mount_path"), nil, "")}, + }, { + name: "same mount path in two lists", + obj: valid(func(c *ateworkerpb.Container) { + c.ImageVolumeMounts = append(c.ImageVolumeMounts, &ateworkerpb.VolumeMount{Name: "data-again", MountPath: "/data"}) + }), + want: field.ErrorList{field.Duplicate(field.NewPath("image_volume_mounts").Index(1).Child("mount_path"), nil)}, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + matcher := field.ErrorMatcher{}.ByType().ByField().ByOrigin() + matcher.Test(t, tt.want, Validate_Container(context.Background(), createOp(), nil, tt.obj, nil)) + }) + } +} + +// ActorDirs is reached from all four lifecycle requests; each request table +// carries one case to show the path prefix, and this table holds the rules. +func TestValidateActorDirs(t *testing.T) { + valid := func(mutate ...func(*ateworkerpb.ActorDirs)) *ateworkerpb.ActorDirs { + d := fullActorDirs() + for _, m := range mutate { + m(d) + } + return d + } + tests := []struct { + name string + obj *ateworkerpb.ActorDirs + want field.ErrorList + }{{ + name: "valid", + obj: valid(), + }, { + name: "missing root_dir", + obj: valid(func(d *ateworkerpb.ActorDirs) { d.RootDir = "" }), + want: field.ErrorList{field.Required(field.NewPath("root_dir"), "")}, + }, { + name: "missing external_volume_mounts_dir", + obj: valid(func(d *ateworkerpb.ActorDirs) { d.ExternalVolumeMountsDir = "" }), + want: field.ErrorList{field.Required(field.NewPath("external_volume_mounts_dir"), "")}, + }, { + name: "missing sandbox_assets_dir", + obj: valid(func(d *ateworkerpb.ActorDirs) { d.SandboxAssetsDir = "" }), + want: field.ErrorList{field.Required(field.NewPath("sandbox_assets_dir"), "")}, + }, { + name: "relative sandbox_assets_dir", + obj: valid(func(d *ateworkerpb.ActorDirs) { d.SandboxAssetsDir = "sandbox-assets" }), + want: field.ErrorList{field.Invalid(field.NewPath("sandbox_assets_dir"), nil, "")}, + }, { + name: "relative oci_bundle_dir", + obj: valid(func(d *ateworkerpb.ActorDirs) { d.OciBundleDir = "bundles" }), + want: field.ErrorList{field.Invalid(field.NewPath("oci_bundle_dir"), nil, "")}, + }, { + name: "unclean restore_dir", + obj: valid(func(d *ateworkerpb.ActorDirs) { d.RestoreDir = "/actors/uid/../other/restore" }), + want: field.ErrorList{field.Invalid(field.NewPath("restore_dir"), nil, "")}, + }, { + name: "trailing slash on durable_dir_volume_mounts_dir", + obj: valid(func(d *ateworkerpb.ActorDirs) { d.DurableDirVolumeMountsDir = "/actors/uid/durable/" }), + want: field.ErrorList{field.Invalid(field.NewPath("durable_dir_volume_mounts_dir"), nil, "")}, + }, { + name: "system_info_volume_roots_dir too long", + obj: valid(func(d *ateworkerpb.ActorDirs) { d.SystemInfoVolumeRootsDir = "/" + strings.Repeat("x", 4096) }), + want: field.ErrorList{field.TooLong(field.NewPath("system_info_volume_roots_dir"), nil, 4096).WithOrigin("maxLength")}, + }, { + name: "two bad dirs are both reported", + obj: valid(func(d *ateworkerpb.ActorDirs) { d.RootDir, d.CheckpointDir = "", "checkpoint" }), + want: field.ErrorList{ + field.Required(field.NewPath("root_dir"), ""), + field.Invalid(field.NewPath("checkpoint_dir"), nil, ""), + }, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + matcher := field.ErrorMatcher{}.ByType().ByField().ByOrigin() + matcher.Test(t, tt.want, Validate_ActorDirs(context.Background(), createOp(), nil, tt.obj, nil)) + }) + } +} + +func validRestoreWorkloadRequest(mutate ...func(*ateworkerpb.RestoreWorkloadRequest)) *ateworkerpb.RestoreWorkloadRequest { + r := &ateworkerpb.RestoreWorkloadRequest{ + Atespace: "team-a", + ActorName: "actor-1", + ActorUid: "01234567-89ab-cdef-0123-456789abcdef", + ActorTemplateAtespace: "team-a", + ActorTemplateName: "template-1", + RunscPath: "/opt/runsc", + Spec: fullSpec(), + RuntimeAssetPaths: map[string]string{"kata-kernel": "/opt/kernel"}, + Fidelity: ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, + EgressGateway: &ateworkerpb.EgressGateway{Address: "gateway:443"}, + CpuMilli: 1500, + MemoryBytes: 1 << 30, + ActorDirs: fullActorDirs(), + } + for _, m := range mutate { + m(r) + } + return r +} + +// Restore shares its identity, size, and asset fields with Run, which the +// Run table covers; this table holds the shared gates at one case each and +// covers what only Restore has. +func TestValidateRestoreWorkloadRequest(t *testing.T) { + valid := validRestoreWorkloadRequest + + tests := []struct { + name string + obj *ateworkerpb.RestoreWorkloadRequest + want field.ErrorList + }{{ + name: "valid", + obj: valid(), + }, { + name: "valid: memory fidelity", + obj: valid(func(r *ateworkerpb.RestoreWorkloadRequest) { + r.Fidelity = ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY + }), + }, { + name: "valid: rootfs fidelity, which runtimes judge for themselves", + obj: valid(func(r *ateworkerpb.RestoreWorkloadRequest) { + r.Fidelity = ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS + }), + }, { + name: "missing actor_uid", + obj: valid(func(r *ateworkerpb.RestoreWorkloadRequest) { r.ActorUid = "" }), + want: field.ErrorList{field.Required(field.NewPath("actor_uid"), "")}, + }, { + name: "invalid atespace: uppercase", + obj: valid(func(r *ateworkerpb.RestoreWorkloadRequest) { r.Atespace = "Team-A" }), + want: field.ErrorList{field.Invalid(field.NewPath("atespace"), nil, "").WithOrigin("format=k8s-short-name")}, + }, { + name: "missing spec", + obj: valid(func(r *ateworkerpb.RestoreWorkloadRequest) { r.Spec = nil }), + want: field.ErrorList{field.Required(field.NewPath("spec"), "")}, + }, { + name: "spec without containers", + obj: valid(func(r *ateworkerpb.RestoreWorkloadRequest) { r.Spec = &ateworkerpb.WorkloadSpec{} }), + want: field.ErrorList{field.Required(field.NewPath("spec", "containers"), "")}, + }, { + name: "missing actor_dirs", + obj: valid(func(r *ateworkerpb.RestoreWorkloadRequest) { r.ActorDirs = nil }), + want: field.ErrorList{field.Required(field.NewPath("actor_dirs"), "")}, + }, { + name: "negative cpu_milli", + obj: valid(func(r *ateworkerpb.RestoreWorkloadRequest) { r.CpuMilli = -1 }), + want: field.ErrorList{field.Invalid(field.NewPath("cpu_milli"), nil, "").WithOrigin("minimum")}, + }, { + name: "cpu_milli past atelet's bound", + obj: valid(func(r *ateworkerpb.RestoreWorkloadRequest) { r.CpuMilli = 1000000 }), + want: field.ErrorList{field.Invalid(field.NewPath("cpu_milli"), nil, "").WithOrigin("maximum")}, + }, { + name: "runtime asset path too long", + obj: valid(func(r *ateworkerpb.RestoreWorkloadRequest) { + r.RuntimeAssetPaths = map[string]string{"kata-kernel": "/" + strings.Repeat("x", 4096)} + }), + want: field.ErrorList{field.TooLong(field.NewPath("runtime_asset_paths").Key("kata-kernel"), nil, 4096).WithOrigin("maxLength")}, + }, { + name: "egress_gateway address without port", + obj: valid(func(r *ateworkerpb.RestoreWorkloadRequest) { r.EgressGateway.Address = "gateway" }), + want: field.ErrorList{field.Invalid(field.NewPath("egress_gateway", "address"), nil, "")}, + }, { + name: "missing fidelity", + obj: valid(func(r *ateworkerpb.RestoreWorkloadRequest) { + r.Fidelity = ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED + }), + want: field.ErrorList{field.Required(field.NewPath("fidelity"), "")}, + }, { + name: "fidelity past the enum", + obj: valid(func(r *ateworkerpb.RestoreWorkloadRequest) { r.Fidelity = ateworkerpb.SnapshotFidelity(4) }), + want: field.ErrorList{field.Invalid(field.NewPath("fidelity"), nil, "").WithOrigin("maximum")}, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + matcher := field.ErrorMatcher{}.ByType().ByField().ByOrigin() + matcher.Test(t, tt.want, Validate_RestoreWorkloadRequest(context.Background(), createOp(), nil, tt.obj, nil)) + }) + } +} + +// TestValidateRestoreWorkloadRequestEdge covers the handler-facing wrapper: +// valid passes, invalid comes back as InvalidArgument. +func TestValidateRestoreWorkloadRequestEdge(t *testing.T) { + if err := ValidateRestoreWorkloadRequest(context.Background(), validRestoreWorkloadRequest()); err != nil { + t.Fatalf("valid request rejected: %v", err) + } + err := ValidateRestoreWorkloadRequest(context.Background(), &ateworkerpb.RestoreWorkloadRequest{}) + if apierror.Code(err) != codes.InvalidArgument { + t.Fatalf("empty request error = %v, want InvalidArgument", err) + } +} + +func validTerminateWorkloadRequest(mutate ...func(*ateworkerpb.TerminateWorkloadRequest)) *ateworkerpb.TerminateWorkloadRequest { + r := &ateworkerpb.TerminateWorkloadRequest{ + Atespace: "team-a", + ActorName: "actor-1", + ActorUid: "01234567-89ab-cdef-0123-456789abcdef", + ActorTemplateAtespace: "team-a", + ActorTemplateName: "template-1", + RunscPath: "/opt/runsc", + Spec: fullSpec(), + ActorDirs: fullActorDirs(), + } + for _, m := range mutate { + m(r) + } + return r +} + +func TestValidateTerminateWorkloadRequest(t *testing.T) { + valid := validTerminateWorkloadRequest + + tests := []struct { + name string + obj *ateworkerpb.TerminateWorkloadRequest + want field.ErrorList + }{{ + name: "valid", + obj: valid(), + }, { + name: "valid: no template ref, no runsc_path", + obj: valid(func(r *ateworkerpb.TerminateWorkloadRequest) { + r.ActorTemplateAtespace, r.ActorTemplateName, r.RunscPath = "", "", "" + }), + }, { + name: "missing atespace", + obj: valid(func(r *ateworkerpb.TerminateWorkloadRequest) { r.Atespace = "" }), + want: field.ErrorList{field.Required(field.NewPath("atespace"), "")}, + }, { + name: "invalid atespace: uppercase", + obj: valid(func(r *ateworkerpb.TerminateWorkloadRequest) { r.Atespace = "Team-A" }), + want: field.ErrorList{field.Invalid(field.NewPath("atespace"), nil, "").WithOrigin("format=k8s-short-name")}, + }, { + name: "missing actor_name", + obj: valid(func(r *ateworkerpb.TerminateWorkloadRequest) { r.ActorName = "" }), + want: field.ErrorList{field.Required(field.NewPath("actor_name"), "")}, + }, { + name: "invalid actor_name: trailing dash", + obj: valid(func(r *ateworkerpb.TerminateWorkloadRequest) { r.ActorName = "actor-" }), + want: field.ErrorList{field.Invalid(field.NewPath("actor_name"), nil, "").WithOrigin("format=k8s-short-name")}, + }, { + name: "missing actor_uid", + obj: valid(func(r *ateworkerpb.TerminateWorkloadRequest) { r.ActorUid = "" }), + want: field.ErrorList{field.Required(field.NewPath("actor_uid"), "")}, + }, { + name: "invalid actor_uid: not a uuid", + obj: valid(func(r *ateworkerpb.TerminateWorkloadRequest) { r.ActorUid = "uid-a" }), + want: field.ErrorList{field.Invalid(field.NewPath("actor_uid"), nil, "").WithOrigin("format=k8s-uuid")}, + }, { + name: "invalid actor_template_atespace: uppercase", + obj: valid(func(r *ateworkerpb.TerminateWorkloadRequest) { r.ActorTemplateAtespace = "Team-A" }), + want: field.ErrorList{field.Invalid(field.NewPath("actor_template_atespace"), nil, "").WithOrigin("format=k8s-short-name")}, + }, { + name: "invalid actor_template_name: underscore", + obj: valid(func(r *ateworkerpb.TerminateWorkloadRequest) { r.ActorTemplateName = "template_1" }), + want: field.ErrorList{field.Invalid(field.NewPath("actor_template_name"), nil, "").WithOrigin("format=k8s-short-name")}, + }, { + name: "runsc_path too long", + obj: valid(func(r *ateworkerpb.TerminateWorkloadRequest) { r.RunscPath = "/" + strings.Repeat("x", 4096) }), + want: field.ErrorList{field.TooLong(field.NewPath("runsc_path"), nil, 4096).WithOrigin("maxLength")}, + }, { + name: "missing spec", + obj: valid(func(r *ateworkerpb.TerminateWorkloadRequest) { r.Spec = nil }), + want: field.ErrorList{field.Required(field.NewPath("spec"), "")}, + }, { + name: "valid: spec without containers, the fallback for a template-less actor", + obj: valid(func(r *ateworkerpb.TerminateWorkloadRequest) { r.Spec = &ateworkerpb.WorkloadSpec{} }), + }, { + name: "missing actor_dirs", + obj: valid(func(r *ateworkerpb.TerminateWorkloadRequest) { r.ActorDirs = nil }), + want: field.ErrorList{field.Required(field.NewPath("actor_dirs"), "")}, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + matcher := field.ErrorMatcher{}.ByType().ByField().ByOrigin() + matcher.Test(t, tt.want, Validate_TerminateWorkloadRequest(context.Background(), createOp(), nil, tt.obj, nil)) + }) + } +} + +// TestValidateTerminateWorkloadRequestEdge covers the handler-facing wrapper: +// valid passes, invalid comes back as InvalidArgument. +func TestValidateTerminateWorkloadRequestEdge(t *testing.T) { + if err := ValidateTerminateWorkloadRequest(context.Background(), validTerminateWorkloadRequest()); err != nil { + t.Fatalf("valid request rejected: %v", err) + } + err := ValidateTerminateWorkloadRequest(context.Background(), &ateworkerpb.TerminateWorkloadRequest{}) + if apierror.Code(err) != codes.InvalidArgument { + t.Fatalf("empty request error = %v, want InvalidArgument", err) + } +} + +func TestValidateGetWorkloadStatsRequest(t *testing.T) { + tests := []struct { + name string + obj *ateworkerpb.GetWorkloadStatsRequest + want field.ErrorList + }{{ + name: "valid", + obj: &ateworkerpb.GetWorkloadStatsRequest{ActorUid: "01234567-89ab-cdef-0123-456789abcdef"}, + }, { + name: "missing actor_uid", + obj: &ateworkerpb.GetWorkloadStatsRequest{}, + want: field.ErrorList{field.Required(field.NewPath("actor_uid"), "")}, + }, { + name: "invalid actor_uid: not a uuid", + obj: &ateworkerpb.GetWorkloadStatsRequest{ActorUid: "uid-a"}, + want: field.ErrorList{field.Invalid(field.NewPath("actor_uid"), nil, "").WithOrigin("format=k8s-uuid")}, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + matcher := field.ErrorMatcher{}.ByType().ByField().ByOrigin() + matcher.Test(t, tt.want, Validate_GetWorkloadStatsRequest(context.Background(), createOp(), nil, tt.obj, nil)) + }) + } +} + +// TestValidateGetWorkloadStatsRequestEdge covers the handler-facing wrapper: +// valid passes, invalid comes back as InvalidArgument. +func TestValidateGetWorkloadStatsRequestEdge(t *testing.T) { + valid := &ateworkerpb.GetWorkloadStatsRequest{ActorUid: "01234567-89ab-cdef-0123-456789abcdef"} + if err := ValidateGetWorkloadStatsRequest(context.Background(), valid); err != nil { + t.Fatalf("valid request rejected: %v", err) + } + err := ValidateGetWorkloadStatsRequest(context.Background(), &ateworkerpb.GetWorkloadStatsRequest{}) + if apierror.Code(err) != codes.InvalidArgument { + t.Fatalf("empty request error = %v, want InvalidArgument", err) + } +} diff --git a/internal/ateworkervalidation/zz_generated.validation.go b/internal/ateworkervalidation/zz_generated.validation.go new file mode 100644 index 0000000000..acf75a2af9 --- /dev/null +++ b/internal/ateworkervalidation/zz_generated.validation.go @@ -0,0 +1,2421 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by validation-gen. DO NOT EDIT. + +package ateworkervalidation + +import ( + context "context" + + ateworkerpb "go.ate.dev/substrate/internal/proto/ateworkerpb" + operation "k8s.io/apimachinery/pkg/api/operation" + safe "k8s.io/apimachinery/pkg/api/safe" + validate "k8s.io/apimachinery/pkg/api/validate" + field "k8s.io/apimachinery/pkg/util/validation/field" +) + +// Validate_ActorDirs validates an instance of ActorDirs according +// to declarative validation rules in the API schema. +func Validate_ActorDirs( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateworkerpb.ActorDirs) (errs field.ErrorList) { + + // custom validation + if e := ValidateCustom_ActorDirs(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + + { // field ateworkerpb.ActorDirs.RootDir + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 4096); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.ActorDirs) *string { + return &oldObj.RootDir + }) + errs = append(errs, fn(fldPath.Child("root_dir"), &obj.RootDir, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.ActorDirs.OciBundleDir + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 4096); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.ActorDirs) *string { + return &oldObj.OciBundleDir + }) + errs = append(errs, fn(fldPath.Child("oci_bundle_dir"), &obj.OciBundleDir, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.ActorDirs.CheckpointDir + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 4096); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.ActorDirs) *string { + return &oldObj.CheckpointDir + }) + errs = append(errs, fn(fldPath.Child("checkpoint_dir"), &obj.CheckpointDir, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.ActorDirs.RestoreDir + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 4096); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.ActorDirs) *string { + return &oldObj.RestoreDir + }) + errs = append(errs, fn(fldPath.Child("restore_dir"), &obj.RestoreDir, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.ActorDirs.DurableDirVolumeMountsDir + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 4096); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.ActorDirs) *string { + return &oldObj.DurableDirVolumeMountsDir + }) + errs = append(errs, fn(fldPath.Child("durable_dir_volume_mounts_dir"), &obj.DurableDirVolumeMountsDir, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.ActorDirs.SystemInfoVolumeRootsDir + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 4096); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.ActorDirs) *string { + return &oldObj.SystemInfoVolumeRootsDir + }) + errs = append(errs, fn(fldPath.Child("system_info_volume_roots_dir"), &obj.SystemInfoVolumeRootsDir, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.ActorDirs.ExternalVolumeMountsDir + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 4096); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.ActorDirs) *string { + return &oldObj.ExternalVolumeMountsDir + }) + errs = append(errs, fn(fldPath.Child("external_volume_mounts_dir"), &obj.ExternalVolumeMountsDir, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.ActorDirs.SandboxAssetsDir + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 4096); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.ActorDirs) *string { + return &oldObj.SandboxAssetsDir + }) + errs = append(errs, fn(fldPath.Child("sandbox_assets_dir"), &obj.SandboxAssetsDir, oldVal, oldObj != nil)...) + } + + return errs +} + +// Validate_CheckpointWorkloadRequest validates an instance of CheckpointWorkloadRequest according +// to declarative validation rules in the API schema. +func Validate_CheckpointWorkloadRequest( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateworkerpb.CheckpointWorkloadRequest) (errs field.ErrorList) { + + { // field ateworkerpb.CheckpointWorkloadRequest.Atespace + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.CheckpointWorkloadRequest) *string { + return &oldObj.Atespace + }) + errs = append(errs, fn(fldPath.Child("atespace"), &obj.Atespace, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.CheckpointWorkloadRequest.ActorName + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.CheckpointWorkloadRequest) *string { + return &oldObj.ActorName + }) + errs = append(errs, fn(fldPath.Child("actor_name"), &obj.ActorName, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.CheckpointWorkloadRequest.ActorUid + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.UUID(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.CheckpointWorkloadRequest) *string { + return &oldObj.ActorUid + }) + errs = append(errs, fn(fldPath.Child("actor_uid"), &obj.ActorUid, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.CheckpointWorkloadRequest.ActorTemplateAtespace + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.CheckpointWorkloadRequest) *string { + return &oldObj.ActorTemplateAtespace + }) + errs = append(errs, fn(fldPath.Child("actor_template_atespace"), &obj.ActorTemplateAtespace, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.CheckpointWorkloadRequest.ActorTemplateName + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.CheckpointWorkloadRequest) *string { + return &oldObj.ActorTemplateName + }) + errs = append(errs, fn(fldPath.Child("actor_template_name"), &obj.ActorTemplateName, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.CheckpointWorkloadRequest.RunscPath + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 4096); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.CheckpointWorkloadRequest) *string { + return &oldObj.RunscPath + }) + errs = append(errs, fn(fldPath.Child("runsc_path"), &obj.RunscPath, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.CheckpointWorkloadRequest.Spec + fn := func( + fldPath *field.Path, + obj, oldObj *ateworkerpb.WorkloadSpec, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_WorkloadSpec(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.CheckpointWorkloadRequest) *ateworkerpb.WorkloadSpec { + return oldObj.Spec + }) + errs = append(errs, fn(fldPath.Child("spec"), obj.Spec, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.CheckpointWorkloadRequest.Fidelity + fn := func( + fldPath *field.Path, + obj, oldObj *ateworkerpb.SnapshotFidelity, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 3); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.CheckpointWorkloadRequest) *ateworkerpb.SnapshotFidelity { + return &oldObj.Fidelity + }) + errs = append(errs, fn(fldPath.Child("fidelity"), &obj.Fidelity, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.CheckpointWorkloadRequest.ActorDirs + fn := func( + fldPath *field.Path, + obj, oldObj *ateworkerpb.ActorDirs, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_ActorDirs(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.CheckpointWorkloadRequest) *ateworkerpb.ActorDirs { + return oldObj.ActorDirs + }) + errs = append(errs, fn(fldPath.Child("actor_dirs"), obj.ActorDirs, oldVal, oldObj != nil)...) + } + + return errs +} + +// Validate_Container validates an instance of Container according +// to declarative validation rules in the API schema. +func Validate_Container( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateworkerpb.Container) (errs field.ErrorList) { + + // custom validation + if e := ValidateCustom_Container(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + + { // field ateworkerpb.Container.Name + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.Container) *string { + return &oldObj.Name + }) + errs = append(errs, fn(fldPath.Child("name"), &obj.Name, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.Container.WakeupProbe + fn := func( + fldPath *field.Path, + obj, oldObj *ateworkerpb.WakeupProbe, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_WakeupProbe(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.Container) *ateworkerpb.WakeupProbe { + return oldObj.WakeupProbe + }) + errs = append(errs, fn(fldPath.Child("wakeup_probe"), obj.WakeupProbe, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.Container.DurableDirVolumeMounts + fn := func( + fldPath *field.Path, + obj, oldObj []*ateworkerpb.VolumeMount, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.PtrSliceNoNils[ateworkerpb.VolumeMount](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.OptionalSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // lists with map semantics require unique keys + if e := validate.PtrSliceUnique(ctx, op, fldPath, obj, oldObj, + func(a *ateworkerpb.VolumeMount, b *ateworkerpb.VolumeMount) bool { return a.MountPath == b.MountPath }); len(e) != 0 { + errs = append(errs, e...) + } + // iterate the list and call the type's validation function + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, + func(a *ateworkerpb.VolumeMount, b *ateworkerpb.VolumeMount) bool { return a.MountPath == b.MountPath }, ateDeepEqual, Validate_VolumeMount); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.Container) []*ateworkerpb.VolumeMount { + return oldObj.DurableDirVolumeMounts + }) + errs = append(errs, fn(fldPath.Child("durable_dir_volume_mounts"), obj.DurableDirVolumeMounts, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.Container.ExternalVolumeMounts + fn := func( + fldPath *field.Path, + obj, oldObj []*ateworkerpb.VolumeMount, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.PtrSliceNoNils[ateworkerpb.VolumeMount](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.OptionalSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // lists with map semantics require unique keys + if e := validate.PtrSliceUnique(ctx, op, fldPath, obj, oldObj, + func(a *ateworkerpb.VolumeMount, b *ateworkerpb.VolumeMount) bool { return a.MountPath == b.MountPath }); len(e) != 0 { + errs = append(errs, e...) + } + // iterate the list and call the type's validation function + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, + func(a *ateworkerpb.VolumeMount, b *ateworkerpb.VolumeMount) bool { return a.MountPath == b.MountPath }, ateDeepEqual, Validate_VolumeMount); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.Container) []*ateworkerpb.VolumeMount { + return oldObj.ExternalVolumeMounts + }) + errs = append(errs, fn(fldPath.Child("external_volume_mounts"), obj.ExternalVolumeMounts, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.Container.SystemInfoVolumeMounts + fn := func( + fldPath *field.Path, + obj, oldObj []*ateworkerpb.VolumeMount, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.PtrSliceNoNils[ateworkerpb.VolumeMount](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.OptionalSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // lists with map semantics require unique keys + if e := validate.PtrSliceUnique(ctx, op, fldPath, obj, oldObj, + func(a *ateworkerpb.VolumeMount, b *ateworkerpb.VolumeMount) bool { return a.MountPath == b.MountPath }); len(e) != 0 { + errs = append(errs, e...) + } + // iterate the list and call the type's validation function + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, + func(a *ateworkerpb.VolumeMount, b *ateworkerpb.VolumeMount) bool { return a.MountPath == b.MountPath }, ateDeepEqual, Validate_VolumeMount); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.Container) []*ateworkerpb.VolumeMount { + return oldObj.SystemInfoVolumeMounts + }) + errs = append(errs, fn(fldPath.Child("system_info_volume_mounts"), obj.SystemInfoVolumeMounts, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.Container.ImageVolumeMounts + fn := func( + fldPath *field.Path, + obj, oldObj []*ateworkerpb.VolumeMount, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.PtrSliceNoNils[ateworkerpb.VolumeMount](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.OptionalSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // lists with map semantics require unique keys + if e := validate.PtrSliceUnique(ctx, op, fldPath, obj, oldObj, + func(a *ateworkerpb.VolumeMount, b *ateworkerpb.VolumeMount) bool { return a.MountPath == b.MountPath }); len(e) != 0 { + errs = append(errs, e...) + } + // iterate the list and call the type's validation function + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, + func(a *ateworkerpb.VolumeMount, b *ateworkerpb.VolumeMount) bool { return a.MountPath == b.MountPath }, ateDeepEqual, Validate_VolumeMount); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.Container) []*ateworkerpb.VolumeMount { + return oldObj.ImageVolumeMounts + }) + errs = append(errs, fn(fldPath.Child("image_volume_mounts"), obj.ImageVolumeMounts, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.Container.ContainerSpec + fn := func( + fldPath *field.Path, + obj, oldObj *ateworkerpb.ContainerSpec, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.Container) *ateworkerpb.ContainerSpec { + return oldObj.ContainerSpec + }) + errs = append(errs, fn(fldPath.Child("container_spec"), obj.ContainerSpec, oldVal, oldObj != nil)...) + } + + return errs +} + +// Validate_EgressGateway validates an instance of EgressGateway according +// to declarative validation rules in the API schema. +func Validate_EgressGateway( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateworkerpb.EgressGateway) (errs field.ErrorList) { + + { // field ateworkerpb.EgressGateway.Address + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // custom validation + if e := ValidateCustom_EgressGateway_Address(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 261); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.EgressGateway) *string { + return &oldObj.Address + }) + errs = append(errs, fn(fldPath.Child("address"), &obj.Address, oldVal, oldObj != nil)...) + } + + return errs +} + +// Validate_GetWorkloadStatsRequest validates an instance of GetWorkloadStatsRequest according +// to declarative validation rules in the API schema. +func Validate_GetWorkloadStatsRequest( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateworkerpb.GetWorkloadStatsRequest) (errs field.ErrorList) { + + { // field ateworkerpb.GetWorkloadStatsRequest.ActorUid + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.UUID(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.GetWorkloadStatsRequest) *string { + return &oldObj.ActorUid + }) + errs = append(errs, fn(fldPath.Child("actor_uid"), &obj.ActorUid, oldVal, oldObj != nil)...) + } + + return errs +} + +// Validate_HttpGetAction validates an instance of HttpGetAction according +// to declarative validation rules in the API schema. +func Validate_HttpGetAction( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateworkerpb.HttpGetAction) (errs field.ErrorList) { + + { // field ateworkerpb.HttpGetAction.Path + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // custom validation + if e := ValidateCustom_HttpGetAction_Path(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 1024); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.HttpGetAction) *string { + return &oldObj.Path + }) + errs = append(errs, fn(fldPath.Child("path"), &obj.Path, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.HttpGetAction.Port + fn := func( + fldPath *field.Path, + obj, oldObj *int32, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 65535); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.HttpGetAction) *int32 { + return &oldObj.Port + }) + errs = append(errs, fn(fldPath.Child("port"), &obj.Port, oldVal, oldObj != nil)...) + } + + return errs +} + +// Validate_RestoreWorkloadRequest validates an instance of RestoreWorkloadRequest according +// to declarative validation rules in the API schema. +func Validate_RestoreWorkloadRequest( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateworkerpb.RestoreWorkloadRequest) (errs field.ErrorList) { + + { // field ateworkerpb.RestoreWorkloadRequest.Atespace + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.RestoreWorkloadRequest) *string { + return &oldObj.Atespace + }) + errs = append(errs, fn(fldPath.Child("atespace"), &obj.Atespace, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.RestoreWorkloadRequest.ActorName + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.RestoreWorkloadRequest) *string { + return &oldObj.ActorName + }) + errs = append(errs, fn(fldPath.Child("actor_name"), &obj.ActorName, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.RestoreWorkloadRequest.ActorUid + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.UUID(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.RestoreWorkloadRequest) *string { + return &oldObj.ActorUid + }) + errs = append(errs, fn(fldPath.Child("actor_uid"), &obj.ActorUid, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.RestoreWorkloadRequest.ActorTemplateAtespace + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.RestoreWorkloadRequest) *string { + return &oldObj.ActorTemplateAtespace + }) + errs = append(errs, fn(fldPath.Child("actor_template_atespace"), &obj.ActorTemplateAtespace, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.RestoreWorkloadRequest.ActorTemplateName + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.RestoreWorkloadRequest) *string { + return &oldObj.ActorTemplateName + }) + errs = append(errs, fn(fldPath.Child("actor_template_name"), &obj.ActorTemplateName, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.RestoreWorkloadRequest.RunscPath + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 4096); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.RestoreWorkloadRequest) *string { + return &oldObj.RunscPath + }) + errs = append(errs, fn(fldPath.Child("runsc_path"), &obj.RunscPath, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.RestoreWorkloadRequest.Spec + fn := func( + fldPath *field.Path, + obj, oldObj *ateworkerpb.WorkloadSpec, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + func() { // cohort = "containers" + earlyReturn := false + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "containers", + func(o *ateworkerpb.WorkloadSpec) []*ateworkerpb.Container { return o.Containers }, ateDeepEqual, validate.RequiredSlice).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "containers", + func(o *ateworkerpb.WorkloadSpec) []*ateworkerpb.Container { return o.Containers }, ateDeepEqual, + func(ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj []*ateworkerpb.Container) field.ErrorList { + return validate.MaxItems(ctx, op, fldPath, obj, oldObj, 10) + }).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "containers", + func(o *ateworkerpb.WorkloadSpec) []*ateworkerpb.Container { return o.Containers }, ateDeepEqual, validate.OptionalSlice).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + }() + // call the type's validation function + errs = append(errs, Validate_WorkloadSpec(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.RestoreWorkloadRequest) *ateworkerpb.WorkloadSpec { + return oldObj.Spec + }) + errs = append(errs, fn(fldPath.Child("spec"), obj.Spec, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.RestoreWorkloadRequest.RuntimeAssetPaths + fn := func( + fldPath *field.Path, + obj, oldObj map[string]string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.MaxProperties(ctx, op, fldPath, obj, oldObj, 16).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.OptionalMap(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.EachMapKey(ctx, op, fldPath, obj, oldObj, + func(ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *string) field.ErrorList { + return validate.MaxLength(ctx, op, fldPath, obj, oldObj, 64) + }); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.EachMapVal(ctx, op, fldPath, obj, oldObj, validate.DirectEqual, + func(ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *string) field.ErrorList { + return validate.MaxLength(ctx, op, fldPath, obj, oldObj, 4096) + }); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.RestoreWorkloadRequest) map[string]string { + return oldObj.RuntimeAssetPaths + }) + errs = append(errs, fn(fldPath.Child("runtime_asset_paths"), obj.RuntimeAssetPaths, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.RestoreWorkloadRequest.Fidelity + fn := func( + fldPath *field.Path, + obj, oldObj *ateworkerpb.SnapshotFidelity, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 3); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.RestoreWorkloadRequest) *ateworkerpb.SnapshotFidelity { + return &oldObj.Fidelity + }) + errs = append(errs, fn(fldPath.Child("fidelity"), &obj.Fidelity, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.RestoreWorkloadRequest.EgressGateway + fn := func( + fldPath *field.Path, + obj, oldObj *ateworkerpb.EgressGateway, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_EgressGateway(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.RestoreWorkloadRequest) *ateworkerpb.EgressGateway { + return oldObj.EgressGateway + }) + errs = append(errs, fn(fldPath.Child("egress_gateway"), obj.EgressGateway, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.RestoreWorkloadRequest.CpuMilli + fn := func( + fldPath *field.Path, + obj, oldObj *int64, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 999999); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.RestoreWorkloadRequest) *int64 { + return &oldObj.CpuMilli + }) + errs = append(errs, fn(fldPath.Child("cpu_milli"), &obj.CpuMilli, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.RestoreWorkloadRequest.MemoryBytes + fn := func( + fldPath *field.Path, + obj, oldObj *int64, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.RestoreWorkloadRequest) *int64 { + return &oldObj.MemoryBytes + }) + errs = append(errs, fn(fldPath.Child("memory_bytes"), &obj.MemoryBytes, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.RestoreWorkloadRequest.ActorDirs + fn := func( + fldPath *field.Path, + obj, oldObj *ateworkerpb.ActorDirs, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_ActorDirs(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.RestoreWorkloadRequest) *ateworkerpb.ActorDirs { + return oldObj.ActorDirs + }) + errs = append(errs, fn(fldPath.Child("actor_dirs"), obj.ActorDirs, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.RestoreWorkloadRequest.PreserveRestoreDir + fn := func( + fldPath *field.Path, + obj, oldObj *bool, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.RestoreWorkloadRequest) *bool { + return &oldObj.PreserveRestoreDir + }) + errs = append(errs, fn(fldPath.Child("preserve_restore_dir"), &obj.PreserveRestoreDir, oldVal, oldObj != nil)...) + } + + return errs +} + +// Validate_RunWorkloadRequest validates an instance of RunWorkloadRequest according +// to declarative validation rules in the API schema. +func Validate_RunWorkloadRequest( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateworkerpb.RunWorkloadRequest) (errs field.ErrorList) { + + { // field ateworkerpb.RunWorkloadRequest.Atespace + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.RunWorkloadRequest) *string { + return &oldObj.Atespace + }) + errs = append(errs, fn(fldPath.Child("atespace"), &obj.Atespace, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.RunWorkloadRequest.ActorName + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.RunWorkloadRequest) *string { + return &oldObj.ActorName + }) + errs = append(errs, fn(fldPath.Child("actor_name"), &obj.ActorName, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.RunWorkloadRequest.ActorUid + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.UUID(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.RunWorkloadRequest) *string { + return &oldObj.ActorUid + }) + errs = append(errs, fn(fldPath.Child("actor_uid"), &obj.ActorUid, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.RunWorkloadRequest.ActorTemplateAtespace + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.RunWorkloadRequest) *string { + return &oldObj.ActorTemplateAtespace + }) + errs = append(errs, fn(fldPath.Child("actor_template_atespace"), &obj.ActorTemplateAtespace, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.RunWorkloadRequest.ActorTemplateName + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.RunWorkloadRequest) *string { + return &oldObj.ActorTemplateName + }) + errs = append(errs, fn(fldPath.Child("actor_template_name"), &obj.ActorTemplateName, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.RunWorkloadRequest.RunscPath + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 4096); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.RunWorkloadRequest) *string { + return &oldObj.RunscPath + }) + errs = append(errs, fn(fldPath.Child("runsc_path"), &obj.RunscPath, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.RunWorkloadRequest.Spec + fn := func( + fldPath *field.Path, + obj, oldObj *ateworkerpb.WorkloadSpec, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + func() { // cohort = "containers" + earlyReturn := false + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "containers", + func(o *ateworkerpb.WorkloadSpec) []*ateworkerpb.Container { return o.Containers }, ateDeepEqual, validate.RequiredSlice).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "containers", + func(o *ateworkerpb.WorkloadSpec) []*ateworkerpb.Container { return o.Containers }, ateDeepEqual, + func(ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj []*ateworkerpb.Container) field.ErrorList { + return validate.MaxItems(ctx, op, fldPath, obj, oldObj, 10) + }).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "containers", + func(o *ateworkerpb.WorkloadSpec) []*ateworkerpb.Container { return o.Containers }, ateDeepEqual, validate.OptionalSlice).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + }() + // call the type's validation function + errs = append(errs, Validate_WorkloadSpec(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.RunWorkloadRequest) *ateworkerpb.WorkloadSpec { + return oldObj.Spec + }) + errs = append(errs, fn(fldPath.Child("spec"), obj.Spec, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.RunWorkloadRequest.RuntimeAssetPaths + fn := func( + fldPath *field.Path, + obj, oldObj map[string]string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.MaxProperties(ctx, op, fldPath, obj, oldObj, 16).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.OptionalMap(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.EachMapKey(ctx, op, fldPath, obj, oldObj, + func(ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *string) field.ErrorList { + return validate.MaxLength(ctx, op, fldPath, obj, oldObj, 64) + }); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.EachMapVal(ctx, op, fldPath, obj, oldObj, validate.DirectEqual, + func(ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *string) field.ErrorList { + return validate.MaxLength(ctx, op, fldPath, obj, oldObj, 4096) + }); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.RunWorkloadRequest) map[string]string { + return oldObj.RuntimeAssetPaths + }) + errs = append(errs, fn(fldPath.Child("runtime_asset_paths"), obj.RuntimeAssetPaths, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.RunWorkloadRequest.EgressGateway + fn := func( + fldPath *field.Path, + obj, oldObj *ateworkerpb.EgressGateway, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_EgressGateway(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.RunWorkloadRequest) *ateworkerpb.EgressGateway { + return oldObj.EgressGateway + }) + errs = append(errs, fn(fldPath.Child("egress_gateway"), obj.EgressGateway, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.RunWorkloadRequest.CpuMilli + fn := func( + fldPath *field.Path, + obj, oldObj *int64, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 999999); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.RunWorkloadRequest) *int64 { + return &oldObj.CpuMilli + }) + errs = append(errs, fn(fldPath.Child("cpu_milli"), &obj.CpuMilli, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.RunWorkloadRequest.MemoryBytes + fn := func( + fldPath *field.Path, + obj, oldObj *int64, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.RunWorkloadRequest) *int64 { + return &oldObj.MemoryBytes + }) + errs = append(errs, fn(fldPath.Child("memory_bytes"), &obj.MemoryBytes, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.RunWorkloadRequest.ActorDirs + fn := func( + fldPath *field.Path, + obj, oldObj *ateworkerpb.ActorDirs, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_ActorDirs(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.RunWorkloadRequest) *ateworkerpb.ActorDirs { + return oldObj.ActorDirs + }) + errs = append(errs, fn(fldPath.Child("actor_dirs"), obj.ActorDirs, oldVal, oldObj != nil)...) + } + + return errs +} + +// Validate_TerminateWorkloadRequest validates an instance of TerminateWorkloadRequest according +// to declarative validation rules in the API schema. +func Validate_TerminateWorkloadRequest( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateworkerpb.TerminateWorkloadRequest) (errs field.ErrorList) { + + { // field ateworkerpb.TerminateWorkloadRequest.Atespace + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.TerminateWorkloadRequest) *string { + return &oldObj.Atespace + }) + errs = append(errs, fn(fldPath.Child("atespace"), &obj.Atespace, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.TerminateWorkloadRequest.ActorName + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.TerminateWorkloadRequest) *string { + return &oldObj.ActorName + }) + errs = append(errs, fn(fldPath.Child("actor_name"), &obj.ActorName, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.TerminateWorkloadRequest.ActorUid + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.UUID(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.TerminateWorkloadRequest) *string { + return &oldObj.ActorUid + }) + errs = append(errs, fn(fldPath.Child("actor_uid"), &obj.ActorUid, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.TerminateWorkloadRequest.ActorTemplateAtespace + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.TerminateWorkloadRequest) *string { + return &oldObj.ActorTemplateAtespace + }) + errs = append(errs, fn(fldPath.Child("actor_template_atespace"), &obj.ActorTemplateAtespace, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.TerminateWorkloadRequest.ActorTemplateName + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.TerminateWorkloadRequest) *string { + return &oldObj.ActorTemplateName + }) + errs = append(errs, fn(fldPath.Child("actor_template_name"), &obj.ActorTemplateName, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.TerminateWorkloadRequest.RunscPath + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 4096); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.TerminateWorkloadRequest) *string { + return &oldObj.RunscPath + }) + errs = append(errs, fn(fldPath.Child("runsc_path"), &obj.RunscPath, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.TerminateWorkloadRequest.Spec + fn := func( + fldPath *field.Path, + obj, oldObj *ateworkerpb.WorkloadSpec, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_WorkloadSpec(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.TerminateWorkloadRequest) *ateworkerpb.WorkloadSpec { + return oldObj.Spec + }) + errs = append(errs, fn(fldPath.Child("spec"), obj.Spec, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.TerminateWorkloadRequest.ActorDirs + fn := func( + fldPath *field.Path, + obj, oldObj *ateworkerpb.ActorDirs, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_ActorDirs(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.TerminateWorkloadRequest) *ateworkerpb.ActorDirs { + return oldObj.ActorDirs + }) + errs = append(errs, fn(fldPath.Child("actor_dirs"), obj.ActorDirs, oldVal, oldObj != nil)...) + } + + return errs +} + +// Validate_VolumeMount validates an instance of VolumeMount according +// to declarative validation rules in the API schema. +func Validate_VolumeMount( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateworkerpb.VolumeMount) (errs field.ErrorList) { + + { // field ateworkerpb.VolumeMount.Name + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.VolumeMount) *string { + return &oldObj.Name + }) + errs = append(errs, fn(fldPath.Child("name"), &obj.Name, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.VolumeMount.MountPath + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // custom validation + if e := ValidateCustom_VolumeMount_MountPath(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 4096); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.VolumeMount) *string { + return &oldObj.MountPath + }) + errs = append(errs, fn(fldPath.Child("mount_path"), &obj.MountPath, oldVal, oldObj != nil)...) + } + + return errs +} + +// Validate_WakeupProbe validates an instance of WakeupProbe according +// to declarative validation rules in the API schema. +func Validate_WakeupProbe( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateworkerpb.WakeupProbe) (errs field.ErrorList) { + + { // field ateworkerpb.WakeupProbe.HttpGet + fn := func( + fldPath *field.Path, + obj, oldObj *ateworkerpb.HttpGetAction, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_HttpGetAction(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.WakeupProbe) *ateworkerpb.HttpGetAction { + return oldObj.HttpGet + }) + errs = append(errs, fn(fldPath.Child("http_get"), obj.HttpGet, oldVal, oldObj != nil)...) + } + + { // field ateworkerpb.WakeupProbe.TimeoutSeconds + fn := func( + fldPath *field.Path, + obj, oldObj *int32, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 3600); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.WakeupProbe) *int32 { + return &oldObj.TimeoutSeconds + }) + errs = append(errs, fn(fldPath.Child("timeout_seconds"), &obj.TimeoutSeconds, oldVal, oldObj != nil)...) + } + + return errs +} + +// Validate_WorkloadSpec validates an instance of WorkloadSpec according +// to declarative validation rules in the API schema. +func Validate_WorkloadSpec( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateworkerpb.WorkloadSpec) (errs field.ErrorList) { + + { // field ateworkerpb.WorkloadSpec.Containers + fn := func( + fldPath *field.Path, + obj, oldObj []*ateworkerpb.Container, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.PtrSliceNoNils[ateworkerpb.Container](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 10).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.OptionalSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // lists with map semantics require unique keys + if e := validate.PtrSliceUnique(ctx, op, fldPath, obj, oldObj, + func(a *ateworkerpb.Container, b *ateworkerpb.Container) bool { return a.Name == b.Name }); len(e) != 0 { + errs = append(errs, e...) + } + // iterate the list and call the type's validation function + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, + func(a *ateworkerpb.Container, b *ateworkerpb.Container) bool { return a.Name == b.Name }, ateDeepEqual, Validate_Container); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateworkerpb.WorkloadSpec) []*ateworkerpb.Container { + return oldObj.Containers + }) + errs = append(errs, fn(fldPath.Child("containers"), obj.Containers, oldVal, oldObj != nil)...) + } + + return errs +} diff --git a/internal/atunnel/dns/dnshandler.go b/internal/atunnel/dns/dnshandler.go index c0d6178b50..1ed65a27e8 100644 --- a/internal/atunnel/dns/dnshandler.go +++ b/internal/atunnel/dns/dnshandler.go @@ -117,10 +117,16 @@ func (h *dnsHandler) onRequestCommon(raw []byte) (dnsmessage.Header, dnsmessage. return hdr, dnsmessage.Question{}, errReplyAction(hdr, dnsmessage.RCodeNotImplemented, &q) } - // Check with query policy callout. + // Check with query policy callout. A disallowed name results in an NXDOMAIN. + // + // Note: NXDOMAIN seems to interact better with application stacks than other + // error codes such as REFUSED. cq := canonicalizeQuestion(q) if !h.allow(cq) { - return hdr, dnsmessage.Question{}, errReplyAction(hdr, dnsmessage.RCodeRefused, &q) + return hdr, dnsmessage.Question{}, action{ + kind: actionReply, + payload: protocol.NXDomainPacket(hdr, q), + } } return hdr, cq, action{ diff --git a/internal/atunnel/dns/dnshandler_test.go b/internal/atunnel/dns/dnshandler_test.go index 028d4010d6..54d724019b 100644 --- a/internal/atunnel/dns/dnshandler_test.go +++ b/internal/atunnel/dns/dnshandler_test.go @@ -44,6 +44,32 @@ func buildTestMessage(t *testing.T, hdr dnsmessage.Header, questions []dnsmessag return msg } +// checkUncacheableNegativeAnswer checks that the rest of the reply p is +// parsing carries an SOA record that forbids caching the negative answer. +func checkUncacheableNegativeAnswer(t *testing.T, parser *dnsmessage.Parser) { + t.Helper() + if err := parser.SkipAllQuestions(); err != nil { + t.Fatalf("skipping questions: %v", err) + } + if err := parser.SkipAllAnswers(); err != nil { + t.Fatalf("skipping answers: %v", err) + } + authorities, err := parser.AllAuthorities() + if err != nil { + t.Fatalf("parsing authorities: %v", err) + } + if len(authorities) != 1 { + t.Fatalf("authorities = %+v, want one SOA record", authorities) + } + soa, ok := authorities[0].Body.(*dnsmessage.SOAResource) + if !ok { + t.Fatalf("authority record = %T, want *dnsmessage.SOAResource", authorities[0].Body) + } + if authorities[0].Header.TTL != 0 || soa.MinTTL != 0 { + t.Errorf("SOA TTL = %d, MINIMUM = %d; want both 0 so the answer is not cached", authorities[0].Header.TTL, soa.MinTTL) + } +} + func TestDNSHandlerOnRequestDrop(t *testing.T) { lim := &limiter{ inFlight: &limiterSlots{max: 1}, @@ -157,7 +183,7 @@ func TestDNSHandlerOnRequestSynthesizedReplies(t *testing.T) { wantRCode: dnsmessage.RCodeNotImplemented, }, { - name: "disallowed domain -> REFUSED", + name: "disallowed domain -> NXDOMAIN", raw: buildTestMessage(t, dnsmessage.Header{ ID: 0x1007, RecursionDesired: true, @@ -166,10 +192,10 @@ func TestDNSHandlerOnRequestSynthesizedReplies(t *testing.T) { Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET, }}), - wantRCode: dnsmessage.RCodeRefused, + wantRCode: dnsmessage.RCodeNameError, }, { - name: "disallowed record type -> REFUSED", + name: "disallowed record type -> NXDOMAIN", raw: buildTestMessage(t, dnsmessage.Header{ ID: 0x1008, RecursionDesired: true, @@ -178,7 +204,7 @@ func TestDNSHandlerOnRequestSynthesizedReplies(t *testing.T) { Type: dnsmessage.TypeTXT, Class: dnsmessage.ClassINET, }}), - wantRCode: dnsmessage.RCodeRefused, + wantRCode: dnsmessage.RCodeNameError, }, } @@ -206,6 +232,9 @@ func TestDNSHandlerOnRequestSynthesizedReplies(t *testing.T) { if hdr.RCode != tc.wantRCode { t.Errorf("reply RCode = %v, want %v", hdr.RCode, tc.wantRCode) } + if hdr.RCode == dnsmessage.RCodeNameError { + checkUncacheableNegativeAnswer(t, &p) + } if got := lim.inFlight.occupied(); got != 0 { t.Errorf("inFlight slots = %d after synthesized reply, want 0", got) } diff --git a/internal/atunnel/dns/policy.go b/internal/atunnel/dns/policy.go index 6be8a9e643..3085f73998 100644 --- a/internal/atunnel/dns/policy.go +++ b/internal/atunnel/dns/policy.go @@ -47,7 +47,7 @@ func egressPolicyCheck(ctx context.Context, source PolicySource) queryPolicy { return func(q dnsmessage.Question) bool { policy, err := source.EgressPolicy(ctx) if err != nil { - slog.DebugContext(ctx, "dns relay refused a query; the actor's egress policy is unavailable", slog.Any("err", err)) + slog.DebugContext(ctx, "dns relay denied a query; the actor's egress policy is unavailable", slog.Any("err", err)) return false } mu.Lock() diff --git a/internal/atunnel/dns/policy_test.go b/internal/atunnel/dns/policy_test.go index 93164c2c13..cc01d7d979 100644 --- a/internal/atunnel/dns/policy_test.go +++ b/internal/atunnel/dns/policy_test.go @@ -199,10 +199,10 @@ var policySources = []struct { } // An actor without an egress policy can reach nothing through the egress -// gateway, so it may resolve nothing either. The relay refuses its queries -// without asking a resolver, and does the same when it cannot learn the -// policy. -func TestRelayRefusesUDPQueriesWithoutPolicy(t *testing.T) { +// gateway, so it may resolve nothing either. The relay answers its queries +// NXDOMAIN without asking a resolver, and does the same when it cannot learn +// the policy. +func TestRelayDeniesUDPQueriesWithoutPolicy(t *testing.T) { for _, tc := range policySources { t.Run(tc.name, func(t *testing.T) { var asked atomic.Int64 @@ -217,7 +217,7 @@ func TestRelayRefusesUDPQueriesWithoutPolicy(t *testing.T) { _, udpAddr, _ := serveLoopbackWithPolicy(t, relay, tc.policy) client := dialRelay(t, "udp", udpAddr) - checkAnswer(t, askUDP(t, client, dnsQuery(0x1111)), 0x1111, dnsmessage.RCodeRefused) + checkAnswer(t, askUDP(t, client, dnsQuery(0x1111)), 0x1111, dnsmessage.RCodeNameError) if got := asked.Load(); got != 0 { t.Errorf("the resolver was asked %d times, want 0", got) } @@ -225,7 +225,7 @@ func TestRelayRefusesUDPQueriesWithoutPolicy(t *testing.T) { } } -func TestRelayRefusesTCPQueriesWithoutPolicy(t *testing.T) { +func TestRelayDeniesTCPQueriesWithoutPolicy(t *testing.T) { for _, tc := range policySources { t.Run(tc.name, func(t *testing.T) { upstream, asked := newTCPResolver(t, func(query []byte) []byte { return dnsAnswer(query, 0) }) @@ -237,8 +237,8 @@ func TestRelayRefusesTCPQueriesWithoutPolicy(t *testing.T) { conn := dialRelay(t, "tcp", tcpAddr) // The connection stays open for the actor's next query. - checkAnswer(t, askTCP(t, conn, dnsQuery(0x1111)), 0x1111, dnsmessage.RCodeRefused) - checkAnswer(t, askTCP(t, conn, dnsQuery(0x2222)), 0x2222, dnsmessage.RCodeRefused) + checkAnswer(t, askTCP(t, conn, dnsQuery(0x1111)), 0x1111, dnsmessage.RCodeNameError) + checkAnswer(t, askTCP(t, conn, dnsQuery(0x2222)), 0x2222, dnsmessage.RCodeNameError) if got := asked.Load(); got != 0 { t.Errorf("the resolver was asked %d queries, want 0", got) } @@ -283,7 +283,7 @@ func TestRelayFollowsPolicyChanges(t *testing.T) { } conn := dialRelay(t, tc.network, addr) - checkAnswer(t, tc.ask(t, conn, dnsQuery(1)), 1, dnsmessage.RCodeRefused) + checkAnswer(t, tc.ask(t, conn, dnsQuery(1)), 1, dnsmessage.RCodeNameError) policy.policy.Store(&ateletpb.EgressPolicy{Rules: []*ateletpb.EgressRule{{ Http: &ateletpb.EgressDestinations{Hostnames: []string{"example.com", "*.wild.example"}}, }}}) @@ -291,9 +291,9 @@ func TestRelayFollowsPolicyChanges(t *testing.T) { wildQuery, wildQ := dnsQueryForName(t, 3, "api.wild.example.") checkAnswerForQuestion(t, tc.ask(t, conn, wildQuery), 3, wildQ, dnsmessage.RCodeSuccess) unlistedQuery, unlistedQ := dnsQueryForName(t, 4, "other.example.com.") - checkAnswerForQuestion(t, tc.ask(t, conn, unlistedQuery), 4, unlistedQ, dnsmessage.RCodeRefused) + checkAnswerForQuestion(t, tc.ask(t, conn, unlistedQuery), 4, unlistedQ, dnsmessage.RCodeNameError) policy.policy.Store(nil) - checkAnswer(t, tc.ask(t, conn, dnsQuery(5)), 5, dnsmessage.RCodeRefused) + checkAnswer(t, tc.ask(t, conn, dnsQuery(5)), 5, dnsmessage.RCodeNameError) }) } } diff --git a/internal/atunnel/dns/protocol/protocol.go b/internal/atunnel/dns/protocol/protocol.go index d21c712842..8d4e73f0c3 100644 --- a/internal/atunnel/dns/protocol/protocol.go +++ b/internal/atunnel/dns/protocol/protocol.go @@ -79,9 +79,10 @@ func QDCount(raw []byte) uint16 { return binary.BigEndian.Uint16(raw[4:6]) } -// ErrorPacket returns a serialized DNS packet signaling an error. -func ErrorPacket(hdr dnsmessage.Header, rcode dnsmessage.RCode, q *dnsmessage.Question) []byte { - respHdr := dnsmessage.Header{ +// responseHeader returns the header of a synthesized response to a request +// with header hdr. +func responseHeader(hdr dnsmessage.Header, rcode dnsmessage.RCode) dnsmessage.Header { + return dnsmessage.Header{ ID: hdr.ID, Response: true, OpCode: hdr.OpCode, @@ -89,8 +90,42 @@ func ErrorPacket(hdr dnsmessage.Header, rcode dnsmessage.RCode, q *dnsmessage.Qu RecursionAvailable: true, RCode: rcode, } +} + +// NXDomainPacket returns a serialized NXDOMAIN response to the question q. +// +// The authority section carries an SOA record for the root zone whose TTL and +// MINIMUM are both 0. Resolvers that cache negative answers take the cache +// lifetime from that record (RFC 2308 section 5), so they do not cache this +// one; without an SOA record some, such as dnspython, cache it indefinitely. +func NXDomainPacket(hdr dnsmessage.Header, q dnsmessage.Question) []byte { + root := dnsmessage.MustNewName(".") + b := dnsmessage.NewBuilder(nil, responseHeader(hdr, dnsmessage.RCodeNameError)) + if err := b.StartQuestions(); err != nil { + return ErrorPacket(hdr, dnsmessage.RCodeNameError, &q) + } + if err := b.Question(q); err != nil { + return ErrorPacket(hdr, dnsmessage.RCodeNameError, &q) + } + if err := b.StartAuthorities(); err != nil { + return ErrorPacket(hdr, dnsmessage.RCodeNameError, &q) + } + if err := b.SOAResource( + dnsmessage.ResourceHeader{Name: root, Class: dnsmessage.ClassINET, TTL: 0}, + dnsmessage.SOAResource{NS: root, MBox: root, MinTTL: 0}, + ); err != nil { + return ErrorPacket(hdr, dnsmessage.RCodeNameError, &q) + } + out, err := b.Finish() + if err != nil { + return ErrorPacket(hdr, dnsmessage.RCodeNameError, &q) + } + return out +} - b := dnsmessage.NewBuilder(nil, respHdr) +// ErrorPacket returns a serialized DNS packet signaling an error. +func ErrorPacket(hdr dnsmessage.Header, rcode dnsmessage.RCode, q *dnsmessage.Question) []byte { + b := dnsmessage.NewBuilder(nil, responseHeader(hdr, rcode)) // Try to tack on the question, if possible. if q != nil { if err := b.StartQuestions(); err == nil { diff --git a/internal/atunnel/dns/protocol/protocol_test.go b/internal/atunnel/dns/protocol/protocol_test.go index 52c953dace..5b3bd96677 100644 --- a/internal/atunnel/dns/protocol/protocol_test.go +++ b/internal/atunnel/dns/protocol/protocol_test.go @@ -285,3 +285,85 @@ func TestErrorPacket(t *testing.T) { } }) } + +func TestNXDomainPacket(t *testing.T) { + q := dnsmessage.Question{ + Name: dnsmessage.MustNewName("blocked.example.com."), + Type: dnsmessage.TypeA, + Class: dnsmessage.ClassINET, + } + reqHdr := dnsmessage.Header{ID: 0xabcd, RecursionDesired: true} + pkt := NXDomainPacket(reqHdr, q) + + var p dnsmessage.Parser + gotHdr, err := p.Start(pkt) + if err != nil { + t.Fatalf("p.Start() unexpected error: %v", err) + } + if gotHdr.ID != reqHdr.ID { + t.Errorf("ID = %#x, want %#x", gotHdr.ID, reqHdr.ID) + } + if !gotHdr.Response || !gotHdr.RecursionDesired || !gotHdr.RecursionAvailable { + t.Errorf("header = %+v, want QR, RD, and RA set", gotHdr) + } + if gotHdr.RCode != dnsmessage.RCodeNameError { + t.Errorf("RCode = %v, want %v", gotHdr.RCode, dnsmessage.RCodeNameError) + } + questions, err := p.AllQuestions() + if err != nil { + t.Fatalf("p.AllQuestions() unexpected error: %v", err) + } + if len(questions) != 1 || questions[0] != q { + t.Errorf("questions = %+v, want [%+v]", questions, q) + } + answers, err := p.AllAnswers() + if err != nil { + t.Fatalf("p.AllAnswers() unexpected error: %v", err) + } + if len(answers) != 0 { + t.Errorf("answers = %+v, want none", answers) + } + + // The SOA record's TTL and MINIMUM bound how long a resolver may cache the + // negative answer (RFC 2308 section 5); both must be 0. + authorities, err := p.AllAuthorities() + if err != nil { + t.Fatalf("p.AllAuthorities() unexpected error: %v", err) + } + if len(authorities) != 1 { + t.Fatalf("authorities = %+v, want one SOA record", authorities) + } + soaHdr := authorities[0].Header + soa, ok := authorities[0].Body.(*dnsmessage.SOAResource) + if !ok { + t.Fatalf("authority record = %T, want *dnsmessage.SOAResource", authorities[0].Body) + } + if soaHdr.Name.String() != "." || soaHdr.Class != dnsmessage.ClassINET { + t.Errorf("SOA owner/class = %v %v, want . ClassINET", soaHdr.Name, soaHdr.Class) + } + if soaHdr.TTL != 0 { + t.Errorf("SOA TTL = %d, want 0", soaHdr.TTL) + } + if soa.MinTTL != 0 { + t.Errorf("SOA MINIMUM = %d, want 0", soa.MinTTL) + } +} + +func TestNXDomainPacketUnencodableQuestion(t *testing.T) { + // A zero-value Name cannot be encoded; the reply is still NXDOMAIN. + badQ := dnsmessage.Question{Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET} + reqHdr := dnsmessage.Header{ID: 0x4321} + pkt := NXDomainPacket(reqHdr, badQ) + + var p dnsmessage.Parser + gotHdr, err := p.Start(pkt) + if err != nil { + t.Fatalf("p.Start() unexpected error: %v", err) + } + if gotHdr.ID != reqHdr.ID { + t.Errorf("ID = %#x, want %#x", gotHdr.ID, reqHdr.ID) + } + if gotHdr.RCode != dnsmessage.RCodeNameError { + t.Errorf("RCode = %v, want %v", gotHdr.RCode, dnsmessage.RCodeNameError) + } +} diff --git a/internal/atunnel/dns/readme.md b/internal/atunnel/dns/readme.md index 1b9e3aed75..2b5bda0010 100644 --- a/internal/atunnel/dns/readme.md +++ b/internal/atunnel/dns/readme.md @@ -39,7 +39,7 @@ Actor DNS relays DNS requests from the actor to the external world. - Two goroutines per accepted TCP connection sharing a per-connection downstream write mutex: - **Downstream reader**: loops reading length-prefixed frames from the actor connection and calls `onTCPRequest()`. - If forwarded: writes the unmodified frame directly to the dedicated upstream TCP connection. - - If rejected with a synthesized reply (`FORMERR`, `NOTIMP`, `REFUSED`): writes the framed error response directly to the actor connection under the write mutex without contacting upstream. + - If rejected with a synthesized reply (`FORMERR`, `NOTIMP`, `NXDOMAIN`): writes the framed error response directly to the actor connection under the write mutex without contacting upstream. - On client `EOF`: half-closes the upstream write side (`CloseWrite()`) so in-flight pipelined responses can finish draining. - **Upstream reader**: loops reading length-prefixed frames from the upstream TCP connection, validates each frame via `onTCPResponse()`, and writes valid responses back to the actor connection under the write mutex. @@ -54,7 +54,7 @@ UDP and TCP call up to the DNS protocol layer (`golang.org/x/net/dns/dnsmessage` - **Reply** (synthesized response without upstream round-trip): - `FORMERR` (`RCode = 1`): malformed header/question or `QDCOUNT != 1`. - `NOTIMP` (`RCode = 4`): `OpCode != 0` (non-standard query) or zone transfer `QTYPE` (`AXFR`, `IXFR`). - - `REFUSED` (`RCode = 5`): disallowed by the Actor's egress policy (see [Egress policy](#egress-policy)). + - `NXDOMAIN` (`RCode = 3`): disallowed by the Actor's egress policy (see [Egress policy](#egress-policy)). `NXDOMAIN` rather than `REFUSED` so stub resolvers continue to the next `resolv.conf` search suffix; glibc and musl abort the lookup on `REFUSED`. The authority section carries a root `SOA` record with `TTL` and `MINIMUM` of 0, so resolvers do not cache the denial past a policy change (RFC 2308 §5). - `onUDPRequest(raw []byte, clientAddr net.Addr, upstreams []string)` -> `action`: - Acquires an `inFlight` slot, runs `validateRequest()`, and on `actionForward` records the query in `pendingRequests` with a rewritten `upstreamID`. - `onUDPResponse(raw []byte, from net.Addr)` -> `action`: diff --git a/internal/atunnel/dns/tcphandler_test.go b/internal/atunnel/dns/tcphandler_test.go index 5041831219..2cdee60284 100644 --- a/internal/atunnel/dns/tcphandler_test.go +++ b/internal/atunnel/dns/tcphandler_test.go @@ -195,8 +195,8 @@ func TestTCPHandlerSynthesizedReplyAndHalfClose(t *testing.T) { binary.BigEndian.Uint16(r1[0:2]): dnsmessage.RCode(r1[3] & 0x0f), binary.BigEndian.Uint16(r2[0:2]): dnsmessage.RCode(r2[3] & 0x0f), } - if got[0xbad0] != dnsmessage.RCodeRefused { - t.Errorf("0xbad0 rcode = %v, want %v", got[0xbad0], dnsmessage.RCodeRefused) + if got[0xbad0] != dnsmessage.RCodeNameError { + t.Errorf("0xbad0 rcode = %v, want %v", got[0xbad0], dnsmessage.RCodeNameError) } if got[0x1234] != dnsmessage.RCodeSuccess { t.Errorf("0x1234 rcode = %v, want %v", got[0x1234], dnsmessage.RCodeSuccess) diff --git a/internal/atunnel/dns/udphandler_test.go b/internal/atunnel/dns/udphandler_test.go index ca7854a260..c7e1909652 100644 --- a/internal/atunnel/dns/udphandler_test.go +++ b/internal/atunnel/dns/udphandler_test.go @@ -115,8 +115,8 @@ func TestUDPHandlerSynthesizedReply(t *testing.T) { t.Fatal(err) } got := readWithin(t, client) - if rcode := dnsmessage.RCode(got[3] & 0x0f); rcode != dnsmessage.RCodeRefused { - t.Errorf("synthesized rcode = %v, want %v", rcode, dnsmessage.RCodeRefused) + if rcode := dnsmessage.RCode(got[3] & 0x0f); rcode != dnsmessage.RCodeNameError { + t.Errorf("synthesized rcode = %v, want %v", rcode, dnsmessage.RCodeNameError) } if upstreamCalls.Load() != 0 { t.Errorf("upstream was called %d times on synthesized reply, want 0", upstreamCalls.Load()) diff --git a/internal/benchmarking/boomer/agentsession/agentsession.go b/internal/benchmarking/boomer/agentsession/agentsession.go index e727a2a335..6abe8c64b5 100644 --- a/internal/benchmarking/boomer/agentsession/agentsession.go +++ b/internal/benchmarking/boomer/agentsession/agentsession.go @@ -36,6 +36,7 @@ import ( "log/slog" mathrand "math/rand/v2" "net/http" + "strconv" "strings" "sync" "time" @@ -96,10 +97,62 @@ func init() { Name: "agentsession", LocustFile: "agentsession.py", UserClass: agentSessionUserClass, + Config: knobsCodec, Init: initAgentSession, }) } +// knobs is the AgentSessionUser slice of the runtime config. The master +// populates the keys from the --agentsession-* locust flags +// (common/agentsession_config.py). +type knobs struct { + dynconfig.Lifecycle + // Script is the built-in script variant; "" falls back to the default. + Script string `json:"agentsession_script"` + // ScriptFile is a script YAML on the worker; wins over Script when set. + ScriptFile string `json:"agentsession_script_file"` + // ThinkScale multiplies the script's per-step think times; 0 reads as 1. + ThinkScale float64 `json:"agentsession_think_scale"` + // SnapshotSteps is a comma-separated list of 1-based step numbers + // after which the actor is hibernated; "" hibernates after every step. + SnapshotSteps string `json:"agentsession_snapshot_steps"` +} + +var knobsCodec = dynconfig.Typed[knobs]{ + Validate: func(k knobs) error { + if err := k.Lifecycle.Validate(); err != nil { + return err + } + if k.ThinkScale < 0 { + return fmt.Errorf("agentsession_think_scale cannot be negative: %f", k.ThinkScale) + } + if steps, err := parseSnapshotSteps(k.SnapshotSteps); err != nil { + return err + } else if len(steps) > 0 && k.LifecycleMode == dynconfig.LifecycleModeNone { + return fmt.Errorf("agentsession_snapshot_steps cannot be used with lifecycle_mode %q", dynconfig.LifecycleModeNone) + } + return nil + }, +} + +// parseSnapshotSteps parses a comma-separated list of 1-based step numbers +// (for example, "3,10,20,30,40,50,61") into a set. An empty string returns +// nil, nil (meaning hibernate after every step). +func parseSnapshotSteps(raw string) (map[int]bool, error) { + if strings.TrimSpace(raw) == "" { + return nil, nil + } + steps := make(map[int]bool) + for _, entry := range strings.Split(raw, ",") { + n, err := strconv.Atoi(strings.TrimSpace(entry)) + if err != nil || n <= 0 { + return nil, fmt.Errorf("invalid agentsession_snapshot_steps entry %q: must be a positive 1-based step number", entry) + } + steps[n] = true + } + return steps, nil +} + func initAgentSession(cfg *userclass.Config) (taskFn func(), shutdown func(context.Context)) { if cfg.Tracer == nil { cfg.Tracer = otel.Tracer("substrate-boomer/agentsession") @@ -177,12 +230,12 @@ type loadedScript struct { // scriptSource is what the knobs select: a file named by // --agentsession-script-file wins, else the built-in variant named by // --agentsession-script, else the default. -func scriptSource(dyn dynconfig.Config) (source string, fromFile bool) { - if dyn.AgentSessionScriptFile != "" { - return dyn.AgentSessionScriptFile, true +func scriptSource(dyn knobs) (source string, fromFile bool) { + if dyn.ScriptFile != "" { + return dyn.ScriptFile, true } - if dyn.AgentSessionScript != "" { - return dyn.AgentSessionScript, false + if dyn.Script != "" { + return dyn.Script, false } return DefaultScript, false } @@ -198,7 +251,7 @@ func scriptSource(dyn dynconfig.Config) (source string, fromFile bool) { func (r *runtime) loadScript() (*loadedScript, error) { r.scriptMu.Lock() defer r.scriptMu.Unlock() - source, fromFile := scriptSource(r.cfg.Dyn.Load()) + source, fromFile := scriptSource(dynconfig.Get[knobs](r.cfg.Dyn)) data, err := readScript(source, fromFile) if err != nil { return nil, err @@ -292,7 +345,7 @@ func memoryLimit(tmpl *ateapipb.ActorTemplate) (limit int64, found bool, err err // script's think time scaled by --agentsession-think-scale (0 reads as 1.0), // with ±20% jitter so a fleet of sessions doesn't move in lockstep. func (r *runtime) think(s Step) time.Duration { - scale := r.cfg.Dyn.Load().AgentSessionThinkScale + scale := dynconfig.Get[knobs](r.cfg.Dyn).ThinkScale if scale <= 0 { scale = 1.0 } @@ -355,6 +408,10 @@ func (r *runtime) startUser(ctx context.Context, loaded *loadedScript) (*session if err := r.checkTemplateMemory(ctx, loaded.Script); err != nil { return nil, err } + dyn := dynconfig.Get[knobs](r.cfg.Dyn) + if err := checkSnapshotSteps(dyn, loaded.Script); err != nil { + return nil, err + } u := &sessionUser{ cfg: r.cfg, actorName: "agent-" + uuid.NewString(), @@ -377,12 +434,33 @@ func (r *runtime) startUser(ctx context.Context, loaded *loadedScript) (*session u.suspendAndDelete(ctx) return nil, fmt.Errorf("waitServing: %w", err) } - // Park the fresh actor; step 01 wakes it like any other step. A failed - // park is re-driven at the top of the first step. - u.hibernate(ctx) + u.awake = true + // Park the fresh actor so step 01 wakes it like any other step, unless + // the session stays resident (--lifecycle-mode none or + // --agentsession-snapshot-steps). A failed park is re-driven at the top + // of the first step. + if u.shouldHibernateAtStart(dyn) { + u.hibernate(ctx) + } return u, nil } +// checkSnapshotSteps refuses to start a session when --agentsession-snapshot-steps +// names a 1-based step number beyond the loaded script's length, so a typo +// (or 61 against a shorter script) does not silently run resident. +func checkSnapshotSteps(dyn knobs, script *Script) error { + steps, err := parseSnapshotSteps(dyn.SnapshotSteps) + if err != nil { + return err + } + for step := range steps { + if step > len(script.Steps) { + return fmt.Errorf("agentsession_snapshot_steps entry %d exceeds script %q length (%d steps)", step, script.Name, len(script.Steps)) + } + } + return nil +} + // shutdownConcurrency bounds how many sessions suspendAndDelete at once. // Boomer gives the hook about a minute; at thousands of sessions a serial // sweep leaks most of the actors, while this many in flight clears them @@ -423,6 +501,9 @@ type sessionUser struct { steps []Step // ingestBuf is the runtime's shared read-only ingest payload. ingestBuf []byte + // awake is true when the actor is resident (RUNNING) rather than + // suspended or paused. Zero value (false) means parked. + awake bool // hibernatePending is set by a failed Pause/Suspend: the actor is // stranded RUNNING or SUSPENDING. Waking it from there would misreport // WakeFirstTouch, so runStep finishes the hibernate first. @@ -500,6 +581,30 @@ func (u *sessionUser) ref() *ateapipb.ObjectRef { return &ateapipb.ObjectRef{Atespace: u.cfg.Atespace, Name: u.actorName} } +// shouldHibernateAtStart reports whether a freshly started session should be +// parked before step 1. In resident mode (LifecycleModeNone) or when +// snapshot steps are configured, the actor stays awake after waitServing. +func (u *sessionUser) shouldHibernateAtStart(dyn knobs) bool { + if dyn.LifecycleMode == dynconfig.LifecycleModeNone { + return false + } + steps, err := parseSnapshotSteps(dyn.SnapshotSteps) + return err != nil || len(steps) == 0 +} + +// shouldHibernateAfterStep reports whether the actor should be parked after +// completing the current step (1-based index u.stepIndex+1). +func (u *sessionUser) shouldHibernateAfterStep(dyn knobs) bool { + if dyn.LifecycleMode == dynconfig.LifecycleModeNone { + return false + } + steps, err := parseSnapshotSteps(dyn.SnapshotSteps) + if err != nil || len(steps) == 0 { + return true + } + return steps[u.stepIndex+1] +} + // runStep wakes the actor, executes the step's ops in order, and suspends // it again, reporting whether everything succeeded. The wake is measured by // a dedicated ping sent ahead of the step's ops and recorded as @@ -510,14 +615,16 @@ func (u *sessionUser) runStep(ctx context.Context, step Step) bool { // A failed hibernate left the actor awake. In implicit mode the wake // ping would return in a few ms and be booked as a WakeFirstTouch // success; in explicit mode ResumeActor would fail FailedPrecondition. - // SuspendActor and PauseActor are re-entrant, so finish the hibernate + // SuspendActor is re-entrant in either mode, so finish the hibernate // and pick the step up next iteration. if u.hibernatePending { u.hibernate(ctx) return false } - if u.cfg.Dyn.Load().ResumeMode == dynconfig.ResumeModeExplicit { + dyn := dynconfig.Get[knobs](u.cfg.Dyn) + wasParked := !u.awake && dyn.LifecycleMode != dynconfig.LifecycleModeNone + if wasParked && dyn.ResumeMode == dynconfig.ResumeModeExplicit { if err := u.resume(ctx); err != nil { u.noteFailure(err) return false @@ -527,15 +634,22 @@ func (u *sessionUser) runStep(ctx context.Context, step Step) bool { wakeStart := time.Now() if err := u.execOp(ctx, ping()); err != nil { u.noteFailure(err) - bmetrics.RecordFailure("http", "WakeFirstTouch", agentSessionUserClass, time.Since(wakeStart), err.Error()) + if wasParked { + bmetrics.RecordFailure("http", "WakeFirstTouch", agentSessionUserClass, time.Since(wakeStart), err.Error()) + } slog.Warn("agent session wake failed", slog.String("actor", u.actorName), slog.String("step", step.Name), slog.String("err", err.Error())) - u.hibernate(ctx) + if wasParked || u.shouldHibernateAfterStep(dyn) { + u.hibernate(ctx) + } return false } - bmetrics.RecordSuccess("http", "WakeFirstTouch", agentSessionUserClass, time.Since(wakeStart), 0) + u.awake = true + if wasParked { + bmetrics.RecordSuccess("http", "WakeFirstTouch", agentSessionUserClass, time.Since(wakeStart), 0) + } metricName := "Step_" + step.Name stepStart := time.Now() @@ -548,14 +662,18 @@ func (u *sessionUser) runStep(ctx context.Context, step Step) bool { slog.String("step", step.Name), slog.Int("op", i), slog.String("err", err.Error())) - u.hibernate(ctx) + if wasParked || u.shouldHibernateAfterStep(dyn) { + u.hibernate(ctx) + } return false } } u.noteSuccess() bmetrics.RecordSuccess("http", metricName, agentSessionUserClass, time.Since(stepStart), 0) - u.hibernate(ctx) + if u.shouldHibernateAfterStep(dyn) { + u.hibernate(ctx) + } return true } @@ -747,26 +865,34 @@ func (u *sessionUser) resume(ctx context.Context) error { return err } -// hibernate suspends (or pauses, per LifecycleMode) the actor at the end of -// a step. A failure leaves hibernatePending set so the next step re-drives -// it instead of waking a stranded actor. +// hibernate suspends the actor at the end of a step, asynchronously in pause +// LifecycleMode. In LifecycleModeNone the actor stays resident and hibernate +// is a no-op. A failure leaves hibernatePending set so the next step +// re-drives it instead of waking a stranded actor. func (u *sessionUser) hibernate(ctx context.Context) { + mode := dynconfig.Get[knobs](u.cfg.Dyn).LifecycleMode + if mode == dynconfig.LifecycleModeNone { + u.hibernatePending = false + return + } var err error - if u.cfg.Dyn.Load().LifecycleMode == dynconfig.LifecycleModePause { - err = u.tracedCall(ctx, "PauseActor", func(callCtx context.Context, tr *metadata.MD) error { - _, err := u.cfg.APIStub.PauseActor(callCtx, &ateapipb.PauseActorRequest{Actor: u.ref()}, grpc.Trailer(tr)) + if mode == dynconfig.LifecycleModePause { + err = u.tracedCall(ctx, "SuspendActorAsync", func(callCtx context.Context, tr *metadata.MD) error { + _, err := u.cfg.APIStub.SuspendActor(callCtx, &ateapipb.SuspendActorRequest{Actor: u.ref(), Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}}, grpc.Trailer(tr)) return err }) } else { err = u.tracedCall(ctx, "SuspendActor", func(callCtx context.Context, tr *metadata.MD) error { - _, err := u.cfg.APIStub.SuspendActor(callCtx, &ateapipb.SuspendActorRequest{Actor: u.ref()}, grpc.Trailer(tr)) + _, err := u.cfg.APIStub.SuspendActor(callCtx, &ateapipb.SuspendActorRequest{Actor: u.ref(), Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}, grpc.Trailer(tr)) return err }) } u.hibernatePending = err != nil if err != nil { u.noteFailure(err) + return } + u.awake = false } // suspendAndDelete releases the actor and its worker. Suspend first: a diff --git a/internal/benchmarking/boomer/agentsession/agentsession_test.go b/internal/benchmarking/boomer/agentsession/agentsession_test.go index b6f9618d28..4f6c4c80b5 100644 --- a/internal/benchmarking/boomer/agentsession/agentsession_test.go +++ b/internal/benchmarking/boomer/agentsession/agentsession_test.go @@ -29,6 +29,7 @@ import ( "testing" "time" + "github.com/prometheus/client_golang/prometheus" "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" "go.ate.dev/substrate/internal/benchmarking/boomer/userclass" "go.ate.dev/substrate/internal/benchmarking/glutton/fake" @@ -37,6 +38,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" ) // TestEmbeddedScriptsAreValid keeps every built-in variant loadable: a file @@ -221,7 +223,7 @@ func TestExecOpAgainstFake(t *testing.T) { HTTPClient: http.DefaultClient, RouterURL: ts.URL, Atespace: "benchmark", - Dyn: dynconfig.NewHolder(dynconfig.Config{}), + Dyn: dynconfig.Static(knobs{}), }, actorName: "agent-test", } @@ -270,7 +272,7 @@ func TestExecOpAgainstFake(t *testing.T) { // error that leaves the previous script in place, and a changed knob loads // the new script for sessions that start afterwards. func TestLoadScriptFollowsTheKnob(t *testing.T) { - rt := &runtime{cfg: &userclass.Config{Dyn: dynconfig.NewHolder(dynconfig.Config{AgentSessionScript: "no-such-script"})}} + rt := &runtime{cfg: &userclass.Config{Dyn: dynconfig.Static(knobs{Script: "no-such-script"})}} if _, err := rt.loadScript(); err == nil { t.Fatal("loadScript accepted an unknown script name") } @@ -278,7 +280,7 @@ func TestLoadScriptFollowsTheKnob(t *testing.T) { t.Fatal("a failed load must not cache a script") } - rt.cfg.Dyn.Store(dynconfig.Config{}) + rt.cfg.Dyn = dynconfig.Static(knobs{}) s, err := rt.loadScript() if err != nil { t.Fatal(err) @@ -295,7 +297,7 @@ func TestLoadScriptFollowsTheKnob(t *testing.T) { // A knob that names something that does not load is an error, and the // previous script stays available to sessions that already run on it. - rt.cfg.Dyn.Store(dynconfig.Config{AgentSessionScript: "no-such-script"}) + rt.cfg.Dyn = dynconfig.Static(knobs{Script: "no-such-script"}) if _, err := rt.loadScript(); err == nil { t.Error("loadScript accepted an unknown name after a successful load") } @@ -308,7 +310,7 @@ func TestLoadScriptFollowsTheKnob(t *testing.T) { if err := os.WriteFile(path, []byte(validScript), 0o600); err != nil { t.Fatal(err) } - rt.cfg.Dyn.Store(dynconfig.Config{AgentSessionScriptFile: path}) + rt.cfg.Dyn = dynconfig.Static(knobs{ScriptFile: path}) next, err := rt.loadScript() if err != nil { t.Fatal(err) @@ -343,7 +345,7 @@ func TestLoadScriptFollowsTheKnob(t *testing.T) { func TestShutdownFansOut(t *testing.T) { const sessions = 40 ctl := &fakeControlClient{deleteDelay: 20 * time.Millisecond} - u := newTestUser(t, &fake.Server{}, ctl, dynconfig.Config{}) + u := newTestUser(t, &fake.Server{}, ctl, knobs{}) rt := &runtime{cfg: u.cfg} for i := range sessions { rt.users.Store(int64(i), &sessionUser{cfg: u.cfg, actorName: "agent-" + strconv.Itoa(i)}) @@ -403,9 +405,9 @@ func TestLoadScriptPrefersFile(t *testing.T) { if err := os.WriteFile(path, []byte(validScript), 0o600); err != nil { t.Fatal(err) } - rt := &runtime{cfg: &userclass.Config{Dyn: dynconfig.NewHolder(dynconfig.Config{ - AgentSessionScript: DefaultScript, - AgentSessionScriptFile: path, + rt := &runtime{cfg: &userclass.Config{Dyn: dynconfig.Static(knobs{ + Script: DefaultScript, + ScriptFile: path, })}} s, err := rt.loadScript() if err != nil { @@ -436,7 +438,7 @@ func TestStartUserChecksTemplateMemory(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { ctl := &fakeControlClient{templateMemory: tc.memory} - u := newTestUser(t, &fake.Server{}, ctl, dynconfig.Config{}) + u := newTestUser(t, &fake.Server{}, ctl, knobs{}) rt := &runtime{cfg: u.cfg} started, err := rt.startUser(context.Background(), &loadedScript{Script: script, ingestBuf: makeIngestBuf(script.Steps)}) @@ -469,7 +471,7 @@ func TestTemplateMemoryRefusalExpires(t *testing.T) { t.Fatal(err) } ctl := &fakeControlClient{templateMemory: "512Mi"} - u := newTestUser(t, &fake.Server{}, ctl, dynconfig.Config{}) + u := newTestUser(t, &fake.Server{}, ctl, knobs{}) rt := &runtime{cfg: u.cfg} if err := rt.checkTemplateMemory(context.Background(), script); err == nil { @@ -495,7 +497,7 @@ func TestTemplateMemoryRefusalExpires(t *testing.T) { // start afterwards, not one already walking its steps. func TestRunningSessionKeepsItsScript(t *testing.T) { ctl := &fakeControlClient{templateMemory: "1Gi"} - u := newTestUser(t, &fake.Server{}, ctl, dynconfig.Config{}) + u := newTestUser(t, &fake.Server{}, ctl, knobs{}) rt := &runtime{cfg: u.cfg} first, err := rt.loadScript() if err != nil { @@ -510,7 +512,7 @@ func TestRunningSessionKeepsItsScript(t *testing.T) { if err := os.WriteFile(path, []byte(validScript), 0o600); err != nil { t.Fatal(err) } - rt.cfg.Dyn.Store(dynconfig.Config{AgentSessionScriptFile: path}) + rt.cfg.Dyn = dynconfig.Static(knobs{ScriptFile: path}) second, err := rt.loadScript() if err != nil { t.Fatal(err) @@ -524,7 +526,7 @@ func TestRunningSessionKeepsItsScript(t *testing.T) { func TestThinkScaling(t *testing.T) { r := &runtime{ cfg: &userclass.Config{ - Dyn: dynconfig.NewHolder(dynconfig.Config{AgentSessionThinkScale: 0.5}), + Dyn: dynconfig.Static(knobs{ThinkScale: 0.5}), }, } s := Step{Think: 10 * time.Second} @@ -536,7 +538,7 @@ func TestThinkScaling(t *testing.T) { } // Zero scale reads as 1.0. - r.cfg.Dyn.Store(dynconfig.Config{}) + r.cfg.Dyn = dynconfig.Static(knobs{}) for i := 0; i < 100; i++ { got := r.think(s) if got < 8*time.Second || got > 12*time.Second { @@ -553,6 +555,8 @@ type fakeControlClient struct { calls []string resumeErrs []error suspendErrs []error + // suspendReqs holds every SuspendActor request, in arrival order. + suspendReqs []*ateapipb.SuspendActorRequest // sawDeadline is set when a call arrived with a context deadline. sawDeadline bool // templateMemory is the memory limit GetActorTemplate reports; "" means @@ -616,17 +620,13 @@ func (f *fakeControlClient) SuspendActor(ctx context.Context, in *ateapipb.Suspe f.record(ctx, "SuspendActor") f.mu.Lock() defer f.mu.Unlock() + f.suspendReqs = append(f.suspendReqs, in) if err := nextErr(&f.suspendErrs); err != nil { return nil, err } return &ateapipb.SuspendActorResponse{}, nil } -func (f *fakeControlClient) PauseActor(ctx context.Context, in *ateapipb.PauseActorRequest, opts ...grpc.CallOption) (*ateapipb.PauseActorResponse, error) { - f.record(ctx, "PauseActor") - return &ateapipb.PauseActorResponse{}, nil -} - func (f *fakeControlClient) DeleteActor(ctx context.Context, in *ateapipb.DeleteActorRequest, opts ...grpc.CallOption) (*ateapipb.Actor, error) { f.record(ctx, "DeleteActor") n := f.inFlight.Add(1) @@ -643,6 +643,12 @@ func (f *fakeControlClient) DeleteActor(ctx context.Context, in *ateapipb.Delete return &ateapipb.Actor{}, nil } +func (f *fakeControlClient) recordedSuspendReqs() []*ateapipb.SuspendActorRequest { + f.mu.Lock() + defer f.mu.Unlock() + return append([]*ateapipb.SuspendActorRequest(nil), f.suspendReqs...) +} + func (f *fakeControlClient) recordedCalls() []string { f.mu.Lock() defer f.mu.Unlock() @@ -659,7 +665,7 @@ func countCalls(calls []string, name string) int { return n } -func newTestUser(t *testing.T, srv *fake.Server, ctl *fakeControlClient, dyn dynconfig.Config) *sessionUser { +func newTestUser(t *testing.T, srv *fake.Server, ctl *fakeControlClient, dyn knobs) *sessionUser { t.Helper() ts := srv.Start(t) return &sessionUser{ @@ -668,7 +674,7 @@ func newTestUser(t *testing.T, srv *fake.Server, ctl *fakeControlClient, dyn dyn HTTPClient: ts.Client(), RouterURL: ts.URL, Atespace: "benchmark", - Dyn: dynconfig.NewHolder(dyn), + Dyn: dynconfig.Static(dyn), Tracer: otel.Tracer("test"), }, actorName: "agent-test", @@ -683,7 +689,7 @@ var pingStep = Step{Name: "01_test", Agent: "testing", Think: time.Second, Ops: func TestRunStep_RetriesStrandedHibernate(t *testing.T) { srv := &fake.Server{} ctl := &fakeControlClient{suspendErrs: []error{status.Error(codes.Unavailable, "ate-api-server restarting")}} - u := newTestUser(t, srv, ctl, dynconfig.Config{}) + u := newTestUser(t, srv, ctl, knobs{}) if !u.runStep(context.Background(), pingStep) { t.Fatal("runStep = false; the step's ops succeeded and must count") @@ -712,11 +718,53 @@ func TestRunStep_RetriesStrandedHibernate(t *testing.T) { } } +// A step ends with an async SuspendActor in pause LifecycleMode and a sync +// one in suspend LifecycleMode. +func TestRunStep_HibernateMode(t *testing.T) { + ref := &ateapipb.ObjectRef{Atespace: "benchmark", Name: "agent-test"} + for _, tt := range []struct { + name string + lifecycleMode string + want *ateapipb.SuspendActorRequest + }{ + { + name: "pause", + lifecycleMode: dynconfig.LifecycleModePause, + want: &ateapipb.SuspendActorRequest{ + Actor: ref, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, + }, + }, + { + name: "suspend", + lifecycleMode: dynconfig.LifecycleModeSuspend, + want: &ateapipb.SuspendActorRequest{ + Actor: ref, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + ctl := &fakeControlClient{} + u := newTestUser(t, &fake.Server{}, ctl, knobs{Lifecycle: dynconfig.Lifecycle{LifecycleMode: tt.lifecycleMode}}) + + if !u.runStep(context.Background(), pingStep) { + t.Fatal("runStep = false") + } + + got := ctl.recordedSuspendReqs() + if len(got) != 1 || !proto.Equal(got[0], tt.want) { + t.Errorf("SuspendActor requests = %v, want [%v]", got, tt.want) + } + }) + } +} + // ateapi reports a CRASHED actor on ResumeActor; it never recovers, so the // session must replace it on the first failure, not the third. func TestRunStep_ReplacesCrashedActorImmediately(t *testing.T) { ctl := &fakeControlClient{resumeErrs: []error{status.Error(codes.Aborted, "actor benchmark/agent-test crashed")}} - u := newTestUser(t, &fake.Server{}, ctl, dynconfig.Config{ResumeMode: dynconfig.ResumeModeExplicit}) + u := newTestUser(t, &fake.Server{}, ctl, knobs{Lifecycle: dynconfig.Lifecycle{ResumeMode: dynconfig.ResumeModeExplicit}}) if u.runStep(context.Background(), pingStep) { t.Fatal("runStep = true with a crashed actor") @@ -731,7 +779,7 @@ func TestRunStep_ReplacesCrashedActorImmediately(t *testing.T) { // a state nothing the driver can call moves the actor out of. func TestRunStep_ReplacesStuckActorAfterFailedWake(t *testing.T) { ctl := &fakeControlClient{suspendErrs: []error{status.Error(codes.FailedPrecondition, "MarkSuspending prerequisite not met (got: ACTOR_STATE_CRASHED)")}} - u := newTestUser(t, &fake.Server{Status: 503}, ctl, dynconfig.Config{}) + u := newTestUser(t, &fake.Server{Status: 503}, ctl, knobs{}) if u.runStep(context.Background(), pingStep) { t.Fatal("runStep = true with a failing router") @@ -750,7 +798,7 @@ func TestRunStep_KeepsActorThroughCapacityShortage(t *testing.T) { errs[i] = status.Error(codes.ResourceExhausted, "no worker has room for the actor") } ctl := &fakeControlClient{resumeErrs: errs} - u := newTestUser(t, &fake.Server{}, ctl, dynconfig.Config{ResumeMode: dynconfig.ResumeModeExplicit}) + u := newTestUser(t, &fake.Server{}, ctl, knobs{Lifecycle: dynconfig.Lifecycle{ResumeMode: dynconfig.ResumeModeExplicit}}) for range rounds { if u.runStep(context.Background(), pingStep) { @@ -769,7 +817,7 @@ func TestRunStep_KeepsActorThroughRouterCapacityErrors(t *testing.T) { for _, status := range []int{503, 504, 429} { t.Run(strconv.Itoa(status), func(t *testing.T) { ctl := &fakeControlClient{} - u := newTestUser(t, &fake.Server{Status: status}, ctl, dynconfig.Config{}) + u := newTestUser(t, &fake.Server{Status: status}, ctl, knobs{}) for range maxConsecutiveStepFailures + 2 { if u.runStep(context.Background(), pingStep) { t.Fatal("runStep = true with a failing router") @@ -786,7 +834,7 @@ func TestRunStep_KeepsActorThroughRouterCapacityErrors(t *testing.T) { // brings it back, so it is replaced on the first failure. func TestRunStep_ReplacesActorOnRouterNotFound(t *testing.T) { ctl := &fakeControlClient{} - u := newTestUser(t, &fake.Server{Status: 404}, ctl, dynconfig.Config{}) + u := newTestUser(t, &fake.Server{Status: 404}, ctl, knobs{}) u.runStep(context.Background(), pingStep) if !u.broken { t.Error("broken = false after a 404 wake; want immediate replacement") @@ -798,7 +846,7 @@ func TestRunStep_ReplacesActorOnRouterNotFound(t *testing.T) { // after maxConsecutiveStepFailures steps. func TestRunStep_ReplacesActorAfterRepeatedStepFailures(t *testing.T) { ctl := &fakeControlClient{} - u := newTestUser(t, &fake.Server{Status: 502}, ctl, dynconfig.Config{}) + u := newTestUser(t, &fake.Server{Status: 502}, ctl, knobs{}) for i := 1; i <= maxConsecutiveStepFailures; i++ { u.runStep(context.Background(), pingStep) @@ -810,7 +858,7 @@ func TestRunStep_ReplacesActorAfterRepeatedStepFailures(t *testing.T) { func TestControlRPCsCarryADeadline(t *testing.T) { ctl := &fakeControlClient{} - u := newTestUser(t, &fake.Server{}, ctl, dynconfig.Config{ResumeMode: dynconfig.ResumeModeExplicit}) + u := newTestUser(t, &fake.Server{}, ctl, knobs{Lifecycle: dynconfig.Lifecycle{ResumeMode: dynconfig.ResumeModeExplicit}}) u.runStep(context.Background(), pingStep) u.suspendAndDelete(context.Background()) @@ -845,7 +893,7 @@ func TestDwell(t *testing.T) { HTTPClient: http.DefaultClient, RouterURL: ts.URL, Atespace: "benchmark", - Dyn: dynconfig.NewHolder(dynconfig.Config{}), + Dyn: dynconfig.Static(knobs{}), }, actorName: "agent-test", } @@ -865,3 +913,150 @@ func TestDwell(t *testing.T) { t.Errorf("canceled dwell returned %v, want context.Canceled", err) } } + +// wakeFirstTouchCount returns the cumulative WakeFirstTouch count (success + +// failure) recorded in Prometheus so far. +func wakeFirstTouchCount(t *testing.T) float64 { + t.Helper() + mfs, err := prometheus.DefaultGatherer.Gather() + if err != nil { + t.Fatal(err) + } + var total float64 + for _, mf := range mfs { + if mf.GetName() != "locust_requests_total" { + continue + } + for _, m := range mf.GetMetric() { + var isWake bool + for _, lp := range m.GetLabel() { + if lp.GetName() == "name" && lp.GetValue() == "WakeFirstTouch" { + isWake = true + } + } + if isWake { + total += m.GetCounter().GetValue() + } + } + } + return total +} + +// TestLifecycleModeNone: in resident mode, startUser and runStep never park +// or explicitly resume the actor, never record WakeFirstTouch, and +// suspendAndDelete still suspends before deleting on teardown. +func TestLifecycleModeNone(t *testing.T) { + ctl := &fakeControlClient{templateMemory: "1Gi"} + u := newTestUser(t, &fake.Server{}, ctl, knobs{ + Lifecycle: dynconfig.Lifecycle{ + LifecycleMode: dynconfig.LifecycleModeNone, + ResumeMode: dynconfig.ResumeModeExplicit, + }, + }) + rt := &runtime{cfg: u.cfg} + loaded, err := rt.loadScript() + if err != nil { + t.Fatal(err) + } + session, err := rt.startUser(context.Background(), loaded) + if err != nil { + t.Fatal(err) + } + if !session.awake { + t.Error("session.awake = false after startUser in LifecycleModeNone") + } + wakesBefore := wakeFirstTouchCount(t) + for range 3 { + if !session.runStep(context.Background(), pingStep) { + t.Fatal("runStep = false in LifecycleModeNone") + } + } + if got := wakeFirstTouchCount(t) - wakesBefore; got != 0 { + t.Errorf("WakeFirstTouch recorded %v times in LifecycleModeNone, want 0", got) + } + for _, call := range []string{"SuspendActor", "ResumeActor"} { + if got := countCalls(ctl.recordedCalls(), call); got != 0 { + t.Errorf("%s called %d times in LifecycleModeNone, want 0; calls = %v", call, got, ctl.recordedCalls()) + } + } + + session.suspendAndDelete(context.Background()) + if got := countCalls(ctl.recordedCalls(), "SuspendActor"); got != 1 { + t.Errorf("SuspendActor on teardown = %d, want 1", got) + } + if got := countCalls(ctl.recordedCalls(), "DeleteActor"); got != 1 { + t.Errorf("DeleteActor on teardown = %d, want 1", got) + } +} + +// TestSnapshotSteps: when SnapshotSteps is set, the actor stays +// resident at start and between non-snapshot steps, parking only after the +// configured 1-based step numbers and recording WakeFirstTouch only on the +// steps that actually wake from a parked state. A step number past the +// script length is rejected in startUser before CreateActor. +func TestSnapshotSteps(t *testing.T) { + ctl := &fakeControlClient{templateMemory: "1Gi"} + u := newTestUser(t, &fake.Server{}, ctl, knobs{ + Lifecycle: dynconfig.Lifecycle{ + LifecycleMode: dynconfig.LifecycleModePause, + ResumeMode: dynconfig.ResumeModeExplicit, + }, + SnapshotSteps: "2,4", + }) + rt := &runtime{cfg: u.cfg} + loaded, err := rt.loadScript() + if err != nil { + t.Fatal(err) + } + session, err := rt.startUser(context.Background(), loaded) + if err != nil { + t.Fatal(err) + } + if got := ctl.recordedSuspendReqs(); len(got) != 0 { + t.Fatalf("SuspendActor requests after startUser = %v, want none", got) + } + + // Run 4 steps (1-based 1..4). Only steps 2 and 4 should park; only step 3 + // starts from a parked actor, so only step 3 should issue ResumeActor and + // record WakeFirstTouch. + wakesBefore := wakeFirstTouchCount(t) + for i := range 4 { + session.stepIndex = i + if !session.runStep(context.Background(), pingStep) { + t.Fatalf("step %d failed", i+1) + } + } + wantSuspend := &ateapipb.SuspendActorRequest{ + Actor: session.ref(), + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, + } + gotSuspends := ctl.recordedSuspendReqs() + if len(gotSuspends) != 2 { + t.Errorf("SuspendActor requests across 4 steps = %v, want 2 (steps 2 and 4)", gotSuspends) + } + for _, got := range gotSuspends { + if !proto.Equal(got, wantSuspend) { + t.Errorf("SuspendActor request = %v, want %v", got, wantSuspend) + } + } + if got := countCalls(ctl.recordedCalls(), "ResumeActor"); got != 1 { + t.Errorf("ResumeActor calls across 4 steps = %d, want 1 (at step 3 after step 2 parked); calls = %v", got, ctl.recordedCalls()) + } + if got := wakeFirstTouchCount(t) - wakesBefore; got != 1 { + t.Errorf("WakeFirstTouch recorded %v times across 4 steps, want 1 (at step 3 after step 2 parked)", got) + } + + // A step number beyond len(loaded.Steps) is rejected before CreateActor. + badCtl := &fakeControlClient{templateMemory: "1Gi"} + badUser := newTestUser(t, &fake.Server{}, badCtl, knobs{ + Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModePause}, + SnapshotSteps: "2,99", + }) + badRT := &runtime{cfg: badUser.cfg} + if _, err := badRT.startUser(context.Background(), loaded); err == nil || !strings.Contains(err.Error(), "exceeds script") { + t.Errorf("startUser with out-of-range snapshot step = %v, want error mentioning 'exceeds script'", err) + } + if got := countCalls(badCtl.recordedCalls(), "CreateActor"); got != 0 { + t.Errorf("CreateActor called %d times on out-of-range snapshot step, want 0", got) + } +} diff --git a/internal/benchmarking/boomer/agentsession/knobs_test.go b/internal/benchmarking/boomer/agentsession/knobs_test.go new file mode 100644 index 0000000000..b85bb83c4e --- /dev/null +++ b/internal/benchmarking/boomer/agentsession/knobs_test.go @@ -0,0 +1,64 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package agentsession + +import ( + "testing" + + "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" +) + +// The codec takes a payload naming every key the class reads and refuses +// the values it cannot run on. +func TestKnobsCodec(t *testing.T) { + h := dynconfig.NewHolder(knobsCodec) + if got := dynconfig.Get[knobs](h); got != (knobs{}) { + t.Fatalf("fresh holder = %+v, want the zero defaults", got) + } + if _, _, err := h.Apply([]byte(`{ + "resume_mode": "explicit", "lifecycle_mode": "pause", + "agentsession_script": "coding-session", + "agentsession_script_file": "/etc/agentsession/script.yaml", + "agentsession_think_scale": 0.5, + "agentsession_snapshot_steps": "3,10,20,30,40,50,61" + }`)); err != nil { + t.Fatalf("Apply(valid): %v", err) + } + want := knobs{ + Lifecycle: dynconfig.Lifecycle{ResumeMode: dynconfig.ResumeModeExplicit, LifecycleMode: dynconfig.LifecycleModePause}, + Script: "coding-session", + ScriptFile: "/etc/agentsession/script.yaml", + ThinkScale: 0.5, + SnapshotSteps: "3,10,20,30,40,50,61", + } + if got := dynconfig.Get[knobs](h); got != want { + t.Errorf("decoded =\n %+v, want\n %+v", got, want) + } + for name, payload := range map[string]string{ + "negative think scale": `{"agentsession_think_scale": -1}`, + "invalid resume mode": `{"resume_mode": "invalid_mode"}`, + "invalid snapshot steps entry": `{"agentsession_snapshot_steps": "3,zero,10"}`, + "non-positive snapshot steps entry": `{"agentsession_snapshot_steps": "0,10"}`, + "snapshot steps with lifecycle mode none": `{"lifecycle_mode": "none", "agentsession_snapshot_steps": "3,10"}`, + } { + t.Run(name, func(t *testing.T) { + if _, _, err := h.Apply([]byte(payload)); err == nil { + t.Errorf("Apply(%s) took the payload, want a refusal", payload) + } + if got := dynconfig.Get[knobs](h); got != want { + t.Errorf("a refused payload changed the config to %+v", got) + } + }) + } +} diff --git a/internal/benchmarking/boomer/boomerutil/lifecycle.go b/internal/benchmarking/boomer/boomerutil/lifecycle.go index f328f97bf2..97d635dd89 100644 --- a/internal/benchmarking/boomer/boomerutil/lifecycle.go +++ b/internal/benchmarking/boomer/boomerutil/lifecycle.go @@ -57,8 +57,8 @@ const ( conflictRetryJitter = 5 * time.Millisecond ) -// ClassifyLifecycleFailure maps an error from ResumeActor / SuspendActor / -// PauseActor onto what to do about it. Unrecognized codes are recoverable +// ClassifyLifecycleFailure maps an error from ResumeActor / SuspendActor +// onto what to do about it. Unrecognized codes are recoverable // until proven otherwise: a wrapped atelet error arrives as Internal. func ClassifyLifecycleFailure(err error) FailureAction { s, ok := status.FromError(err) diff --git a/internal/benchmarking/boomer/dynconfig/dynconfig.go b/internal/benchmarking/boomer/dynconfig/dynconfig.go index 8abf1fadee..e91633d01b 100644 --- a/internal/benchmarking/boomer/dynconfig/dynconfig.go +++ b/internal/benchmarking/boomer/dynconfig/dynconfig.go @@ -13,351 +13,309 @@ // limitations under the License. // Package dynconfig fetches and holds the boomer worker's runtime-mutable -// settings — the subset of locust flags the operator can change in the web -// UI form. The boomer wire protocol only carries num_users + spawn_rate, so -// these come over an HTTP side channel from the master's /boomer-config -// endpoint (common/boomer_config.py). +// settings: the locust flags the operator can change in the web UI form. +// The boomer wire protocol only carries num_users + spawn_rate, so these +// come over an HTTP side channel from the master's /boomer-config endpoint +// (common/boomer_config.py). +// +// The payload is a JSON object keyed by locust flag name in snake_case. +// This package does not know what the keys mean. A worker runs exactly one +// user class, known at launch, so the class hands the Holder its Codec at +// construction: the typed struct it reads its knobs from, with the +// defaults a key left unset falls back to and the rules a fetched value +// must pass. The Holder then stores that typed value, decodes each payload +// over those defaults so a payload stands on its own, refuses one the class +// cannot run on in favor of the last good config, and hands the class its +// config back through Get. Keys the class does not name are ignored. package dynconfig import ( + "bytes" "context" "encoding/json" "fmt" + "io" "log/slog" - "math" + "math/rand/v2" "net/http" + "reflect" "sync/atomic" "time" "github.com/myzhan/boomer" ) -// Resume modes. Explicit issues a ResumeActor RPC before sending traffic. -// Implicit issues no wake request at all: the actor stays suspended until a -// request reaches the atenet router, which wakes it while the request is -// parked. -const ( - ResumeModeExplicit = "explicit" - ResumeModeImplicit = "implicit" +// Codec is how a user class's config comes to be: the value a worker +// starts from, and the value a payload puts in force. Typed is the +// implementation a class declares over its knobs struct. +type Codec interface { + // Initial is the config before any payload: the class's defaults. + Initial() any + // Decode is the config payload, a JSON object, puts in force, validated. + // Each payload stands on its own: a key absent from it, or null in it, + // is at the class's default, not at the value before. The master serves + // every flag it knows, null for the ones the operator left blank, so a + // field cleared in the form takes the knob back to its default. A body + // that is not a JSON object, an empty one included, is refused. + Decode(payload []byte) (any, error) +} - ReadModeData = "data" - ReadModeDigest = "digest" +// Typed is the Codec over a class's knobs struct T, whose json tags name +// the keys it reads. Decode lays the payload over Defaults and runs +// Validate on the result. +type Typed[T any] struct { + Defaults T + Validate func(T) error +} - LifecycleModeSuspend = "suspend" - LifecycleModePause = "pause" -) +func (c Typed[T]) Initial() any { return c.Defaults } -// Config is the dynamic-mutable subset of boomer's behavior. Holder swaps -// it atomically so task goroutines read a consistent snapshot. -type Config struct { - MinWait time.Duration // gap between one actor's suspend and the VU's next resume, lower bound - MaxWait time.Duration // upper bound of the same gap +func (c Typed[T]) Decode(payload []byte) (any, error) { + next := c.Defaults + if err := json.Unmarshal(payload, &next); err != nil { + return nil, fmt.Errorf("decode config: %w", err) + } + if c.Validate != nil { + if err := c.Validate(next); err != nil { + return nil, err + } + } + return next, nil +} - MinLive time.Duration // time a GluttonUser actor stays resumed between its first ping and suspend, lower bound - MaxLive time.Duration // upper bound of the live window; zero (the default) suspends right after the ping +// Common is the slice of the payload the worker itself reads, whatever the +// user class. Every Holder decodes it alongside the class's config. +type Common struct { + TraceProbability float64 `json:"trace_probability"` +} - TraceProbability float64 +var commonCodec = Typed[Common]{ + Validate: func(c Common) error { + if c.TraceProbability < 0 || c.TraceProbability > 1 { + return fmt.Errorf("trace_probability must be between 0.0 and 1.0, got: %f", c.TraceProbability) + } + return nil + }, +} - ResumeMode string // ResumeModeExplicit | ResumeModeImplicit - LifecycleMode string // LifecycleModeSuspend | LifecycleModePause +// Holder holds the running class's config and the worker's Common slice, +// swapped together atomically so task goroutines read a consistent pair. +type Holder struct { + codec Codec + v atomic.Pointer[Snapshot] +} - DurDirFileSize int64 // bytes - DurDirReadMode string // ReadModeData | ReadModeDigest - DurDirTemplate string // ActorTemplate name +// Snapshot is the pair the holder holds at one instant: the class's config, +// of the codec's type, and the worker's Common slice. +type Snapshot struct { + Class any + Common Common +} - MemTarget string // resident RAM the GluttonUser fills via WriteRAM, suffixed (e.g. "2Gi"); "" disables - MemChurn string // RAM re-randomized in place each cycle via WriteRAM rotate, suffixed (e.g. "64Mi"); "" disables - MemRead string // RAM walked (one byte per page) via ReadRAM after each resume, suffixed (e.g. "1Gi") or "all"; "" disables - MaxPingsPerWake int // cap on pings a GluttonUser sends during one resume/suspend cycle; values < 1 read as 1 +// NewHolder returns a holder at codec's defaults. A nil codec is a class +// that reads no runtime config. +func NewHolder(codec Codec) *Holder { + if codec == nil { + codec = Typed[struct{}]{} + } + h := &Holder{codec: codec} + h.v.Store(&Snapshot{Class: codec.Initial()}) + return h +} - SweperfTemplate string // ActorTemplate name for the sweperf workload; "" falls back to default - SweperfTotalSteps int // total steps in trace; 0 falls back to default - SweperfNumCycles int // number of cycles to partition steps into; 0 falls back to default - SweperfPollIntervalMs int // /status poll interval in ms; 0 falls back to default +// Static is for tests: a holder fixed at cfg, with no validation. +func Static[T any](cfg T) *Holder { + return NewHolder(Typed[T]{Defaults: cfg}) +} - CPUCores int // goroutines each GluttonUser's actor spins via UseCPU; 0 disables - CPUDutyCycle float64 // fraction of one core each of those goroutines consumes, in [0, 1] +// Apply puts a payload in force, decoded over the defaults, and returns the +// snapshot that payload left in the holder, so a caller reports the value +// this payload put in force and not one a concurrent Apply stored since. A +// payload the class's codec or Common refuses leaves the holder as it was +// and comes back as the error, naming the key. changed is false when the +// payload left every value as it was, so a caller can keep an unchanged +// poll out of the log. +func (h *Holder) Apply(payload []byte) (applied Snapshot, changed bool, err error) { + class, err := h.codec.Decode(payload) + if err != nil { + return Snapshot{}, false, err + } + common, err := commonCodec.Decode(payload) + if err != nil { + return Snapshot{}, false, err + } + prev := h.v.Load() + next := &Snapshot{Class: class, Common: common.(Common)} + if next.Common == prev.Common && reflect.DeepEqual(next.Class, prev.Class) { + return *prev, false, nil + } + h.v.Store(next) + return *next, true, nil +} - AgentSessionScript string // built-in agent-session script variant; "" falls back to the default - AgentSessionScriptFile string // path to a script YAML on the worker; wins over AgentSessionScript when set - AgentSessionThinkScale float64 // multiplier on the script's per-step think times; 0 reads as 1.0 +// Load returns the class's current config as the codec's type; Get is the +// typed form. +func (h *Holder) Load() any { return h.v.Load().Class } - TotalActors int // spawn batch size; 0 keeps --total-actors - SpawnConcurrency int // actors the spawn batch creates concurrently; 0 keeps --spawn-concurrency - ActorDeadline time.Duration // per-actor timeout in the spawn batch; 0 keeps --actor-deadline +// Common returns the worker's current Common slice. +func (h *Holder) Common() Common { return h.v.Load().Common } - TargetRPS float64 // requests per second the whole worker paces to, for user classes that pace; 0 means unpaced +// Get is the class's current config. A nil holder reads as T's zero +// value; a holder built for another type is a programming error and +// panics with both types named. +func Get[T any](h *Holder) T { + var zero T + if h == nil { + return zero + } + cfg, ok := h.Load().(T) + if !ok { + panic(fmt.Sprintf("dynconfig: holder carries %T, Get asked for %T", h.Load(), zero)) + } + return cfg } -// Holder lets readers Load() the current Config and writers Store() a new -// one. Backed by atomic.Pointer for lock-free reads on the hot path. -type Holder struct { - v atomic.Pointer[Config] -} +// Seconds is a duration carried in the payload as a JSON number of seconds, +// which is how the locust flags spell every time value. +type Seconds time.Duration -func NewHolder(initial Config) *Holder { - h := &Holder{} - h.v.Store(&initial) - return h +// Duration converts to the time package's unit. +func (s Seconds) Duration() time.Duration { return time.Duration(s) } + +// String prints the duration the way time.Duration does, so a config in a +// log line reads 500ms rather than a count of nanoseconds. +func (s Seconds) String() string { return s.Duration().String() } + +func (s Seconds) MarshalJSON() ([]byte, error) { + return json.Marshal(time.Duration(s).Seconds()) } -func (h *Holder) Load() Config { return *h.v.Load() } +func (s *Seconds) UnmarshalJSON(data []byte) error { + if bytes.Equal(bytes.TrimSpace(data), []byte("null")) { + return nil + } + var secs float64 + if err := json.Unmarshal(data, &secs); err != nil { + return fmt.Errorf("seconds: %w", err) + } + *s = Seconds(secs * float64(time.Second)) + return nil +} -func (h *Holder) Store(c Config) { h.v.Store(&c) } +// WaitTime is the gap between one iteration of a user and the next, drawn +// uniformly from [MinWait, MaxWait]. Embed it in a class's knobs struct +// and call its Validate from the class's. +type WaitTime struct { + MinWait Seconds `json:"min_wait_time"` + MaxWait Seconds `json:"max_wait_time"` +} -// ProbabilityUpdater is the subset of trace.UpdatableSampler we touch here; -// kept as an interface so this package doesn't depend on the trace package. -type ProbabilityUpdater interface { - UpdateProbability(p float64) +func (w WaitTime) Validate() error { + if w.MinWait < 0 { + return fmt.Errorf("min_wait_time cannot be negative: %v", w.MinWait.Duration()) + } + if w.MaxWait < 0 { + return fmt.Errorf("max_wait_time cannot be negative: %v", w.MaxWait.Duration()) + } + if w.MaxWait < w.MinWait { + return fmt.Errorf("max_wait_time (%v) cannot be less than min_wait_time (%v)", w.MaxWait.Duration(), w.MinWait.Duration()) + } + return nil } -// payload mirrors the master's /boomer-config JSON. Fields are pointers so -// we can distinguish "absent" (leave current value) from "explicitly zero". -// The same shape is used for static --config-json input and the live -// /boomer-config endpoint, so master + Python runner + Go worker share one -// vocabulary for the boomer's runtime-tunable knobs. -type payload struct { - TraceProbability *float64 `json:"trace_probability"` - MinWaitTime *float64 `json:"min_wait_time"` - MaxWaitTime *float64 `json:"max_wait_time"` - MinLiveTime *float64 `json:"min_live_time"` - MaxLiveTime *float64 `json:"max_live_time"` - DurDirFileSize *float64 `json:"durdir_file_size_bytes"` - ResumeMode *string `json:"resume_mode"` - LifecycleMode *string `json:"lifecycle_mode"` - DurDirReadMode *string `json:"durdir_read_mode"` - DurDirTemplate *string `json:"durdir_template"` - MemTarget *string `json:"mem_target"` - MemChurn *string `json:"mem_churn"` - MemRead *string `json:"mem_read"` - CPUCores *float64 `json:"cpu_cores"` - CPUDutyCycle *float64 `json:"cpu_duty_cycle"` - MaxPingsPerWake *float64 `json:"max_pings_per_wake"` - SweperfTemplate *string `json:"sweperf_template"` - SweperfTotalSteps *float64 `json:"sweperf_total_steps"` - SweperfNumCycles *float64 `json:"sweperf_num_cycles"` - SweperfPollIntervalMs *float64 `json:"sweperf_poll_interval_ms"` - - AgentSessionScript *string `json:"agentsession_script"` - AgentSessionScriptFile *string `json:"agentsession_script_file"` - AgentSessionThinkScale *float64 `json:"agentsession_think_scale"` - - TotalActors *float64 `json:"total_actors"` - SpawnConcurrency *float64 `json:"spawn_concurrency"` - ActorDeadline *float64 `json:"actor_deadline"` - - TargetRPS *float64 `json:"target_rps"` +// Draw returns a wait from [MinWait, MaxWait]; an inverted or empty range +// yields MinWait. +func (w WaitTime) Draw() time.Duration { + return Uniform(w.MinWait.Duration(), w.MaxWait.Duration()) } -// Parse decodes a JSON blob (typically from a CLI flag) and merges its -// fields into `current`. Returns the merged Config — unset fields preserve -// `current`'s existing values, matching Fetch's behavior. -func Parse(jsonBytes []byte, current Config) (Config, error) { - if len(jsonBytes) == 0 { - return current, nil +// Uniform draws from [lo, hi]; an inverted or empty range yields lo. +func Uniform(lo, hi time.Duration) time.Duration { + if hi <= lo { + return lo } - var p payload - if err := json.Unmarshal(jsonBytes, &p); err != nil { - return current, fmt.Errorf("decode config json: %w", err) + return lo + time.Duration(rand.Float64()*float64(hi-lo)) +} + +// Resume and lifecycle modes. Explicit issues a ResumeActor RPC before +// sending traffic; implicit issues no wake request at all: the actor stays +// suspended until a request reaches the atenet router, which wakes it +// while the request is parked. Suspend writes the snapshot to durable +// storage; pause keeps it on the node. +const ( + ResumeModeExplicit = "explicit" + ResumeModeImplicit = "implicit" + + LifecycleModeSuspend = "suspend" + LifecycleModePause = "pause" + LifecycleModeNone = "none" +) + +// Lifecycle is how a class takes its actors off a worker and brings them +// back. The empty string is each class's own default; see the class. +type Lifecycle struct { + ResumeMode string `json:"resume_mode"` + LifecycleMode string `json:"lifecycle_mode"` +} + +func (l Lifecycle) Validate() error { + if l.ResumeMode != "" && l.ResumeMode != ResumeModeExplicit && l.ResumeMode != ResumeModeImplicit { + return fmt.Errorf("invalid resume_mode %q: must be %q or %q", l.ResumeMode, ResumeModeExplicit, ResumeModeImplicit) } - merged := p.merge(current) - if err := merged.Validate(); err != nil { - return current, fmt.Errorf("validate config: %w", err) + if l.LifecycleMode != "" && l.LifecycleMode != LifecycleModeSuspend && l.LifecycleMode != LifecycleModePause && l.LifecycleMode != LifecycleModeNone { + return fmt.Errorf("invalid lifecycle_mode %q: must be %q, %q, or %q", l.LifecycleMode, LifecycleModeSuspend, LifecycleModePause, LifecycleModeNone) } - return merged, nil + return nil } -// Fetch GETs `url` and merges any returned fields into `current`. Returns -// the merged Config (or current unchanged on a soft no-op response). -func Fetch(ctx context.Context, url string, current Config) (Config, error) { +// Fetch GETs url and returns the payload it serves. +func Fetch(ctx context.Context, url string) ([]byte, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { - return current, err + return nil, err } resp, err := http.DefaultClient.Do(req) if err != nil { - return current, fmt.Errorf("GET %s: %w", url, err) + return nil, fmt.Errorf("GET %s: %w", url, err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return current, fmt.Errorf("GET %s: status %d", url, resp.StatusCode) + return nil, fmt.Errorf("GET %s: status %d", url, resp.StatusCode) } - var p payload - if err := json.NewDecoder(resp.Body).Decode(&p); err != nil { - return current, fmt.Errorf("decode %s: %w", url, err) - } - merged := p.merge(current) - if err := merged.Validate(); err != nil { - return current, fmt.Errorf("validate %s: %w", url, err) + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("read %s: %w", url, err) } - return merged, nil + return body, nil } -// Validate checks that the config values are within legal ranges. -func (c Config) Validate() error { - if c.MinWait < 0 { - return fmt.Errorf("min_wait_time cannot be negative: %v", c.MinWait) - } - if c.MaxWait < 0 { - return fmt.Errorf("max_wait_time cannot be negative: %v", c.MaxWait) - } - if c.MaxWait < c.MinWait { - return fmt.Errorf("max_wait_time (%v) cannot be less than min_wait_time (%v)", c.MaxWait, c.MinWait) - } - if c.MinLive < 0 { - return fmt.Errorf("min_live_time cannot be negative: %v", c.MinLive) - } - if c.MaxLive < 0 { - return fmt.Errorf("max_live_time cannot be negative: %v", c.MaxLive) - } - if c.MaxLive < c.MinLive { - return fmt.Errorf("max_live_time (%v) cannot be less than min_live_time (%v)", c.MaxLive, c.MinLive) - } - if c.TraceProbability < 0 || c.TraceProbability > 1 { - return fmt.Errorf("trace_probability must be between 0.0 and 1.0, got: %f", c.TraceProbability) - } - if c.DurDirFileSize < 0 { - return fmt.Errorf("durdir_file_size_bytes cannot be negative: %d", c.DurDirFileSize) - } - if c.DurDirFileSize > math.MaxInt32 { - return fmt.Errorf("durdir_file_size_bytes cannot exceed %d (2 GiB), got: %d", math.MaxInt32, c.DurDirFileSize) - } - if c.ResumeMode != "" && c.ResumeMode != ResumeModeExplicit && c.ResumeMode != ResumeModeImplicit { - return fmt.Errorf("invalid resume_mode %q: must be %q or %q", c.ResumeMode, ResumeModeExplicit, ResumeModeImplicit) - } - if c.LifecycleMode != "" && c.LifecycleMode != LifecycleModeSuspend && c.LifecycleMode != LifecycleModePause { - return fmt.Errorf("invalid lifecycle_mode %q: must be %q or %q", c.LifecycleMode, LifecycleModeSuspend, LifecycleModePause) - } - if c.DurDirReadMode != "" && c.DurDirReadMode != ReadModeData && c.DurDirReadMode != ReadModeDigest { - return fmt.Errorf("invalid durdir_read_mode %q: must be %q or %q", c.DurDirReadMode, ReadModeData, ReadModeDigest) - } - if c.CPUCores < 0 { - return fmt.Errorf("cpu_cores cannot be negative: %d", c.CPUCores) - } - if c.CPUDutyCycle < 0 || c.CPUDutyCycle > 1 { - return fmt.Errorf("cpu_duty_cycle must be between 0.0 and 1.0, got: %f", c.CPUDutyCycle) - } - if c.SweperfTotalSteps < 0 { - return fmt.Errorf("sweperf_total_steps cannot be negative: %d", c.SweperfTotalSteps) - } - if c.SweperfNumCycles < 0 { - return fmt.Errorf("sweperf_num_cycles cannot be negative: %d", c.SweperfNumCycles) - } - if c.SweperfPollIntervalMs < 0 { - return fmt.Errorf("sweperf_poll_interval_ms cannot be negative: %d", c.SweperfPollIntervalMs) - } - if c.AgentSessionThinkScale < 0 { - return fmt.Errorf("agentsession_think_scale cannot be negative: %f", c.AgentSessionThinkScale) - } - if c.TotalActors < 0 { - return fmt.Errorf("total_actors cannot be negative: %d", c.TotalActors) - } - if c.SpawnConcurrency < 0 { - return fmt.Errorf("spawn_concurrency cannot be negative: %d", c.SpawnConcurrency) - } - if c.ActorDeadline < 0 { - return fmt.Errorf("actor_deadline cannot be negative: %v", c.ActorDeadline) - } - if c.TargetRPS < 0 { - return fmt.Errorf("target_rps cannot be negative: %v", c.TargetRPS) - } - // MaxPingsPerWake < 1 is treated as 1 at read time (see iterate() in - // glutton/lifecycle.go), so Config's zero value stays usable — no - // validate rejection here. - // MemTarget, MemChurn, and MemRead are passed to glutton verbatim - // (MemRead's "all" excepted, which the driver maps to an empty - // whole-array walk), which owns the parse; invalid values fail loudly - // there as GluttonFillRAM / GluttonChurnRAM / GluttonReadRAM errors. - return nil +// ProbabilityUpdater is the subset of trace.UpdatableSampler we touch here; +// kept as an interface so this package doesn't depend on the trace package. +type ProbabilityUpdater interface { + UpdateProbability(p float64) } -// merge folds the payload's set fields into `current`, leaving unset fields -// at their existing values. Used by both Parse (CLI input) and Fetch (HTTP -// pull) so the merge semantics are identical. -func (p payload) merge(current Config) Config { - out := current - if p.TraceProbability != nil { - out.TraceProbability = *p.TraceProbability - } - if p.MinWaitTime != nil { - out.MinWait = time.Duration(*p.MinWaitTime * float64(time.Second)) - } - if p.MaxWaitTime != nil { - out.MaxWait = time.Duration(*p.MaxWaitTime * float64(time.Second)) - } - if p.MinLiveTime != nil { - out.MinLive = time.Duration(*p.MinLiveTime * float64(time.Second)) - } - if p.MaxLiveTime != nil { - out.MaxLive = time.Duration(*p.MaxLiveTime * float64(time.Second)) - } - if p.DurDirFileSize != nil { - out.DurDirFileSize = int64(*p.DurDirFileSize) - } - if p.ResumeMode != nil { - out.ResumeMode = *p.ResumeMode - } - if p.LifecycleMode != nil { - out.LifecycleMode = *p.LifecycleMode - } - if p.DurDirReadMode != nil { - out.DurDirReadMode = *p.DurDirReadMode - } - if p.DurDirTemplate != nil { - out.DurDirTemplate = *p.DurDirTemplate - } - if p.MemTarget != nil { - out.MemTarget = *p.MemTarget - } - if p.MemChurn != nil { - out.MemChurn = *p.MemChurn - } - if p.MemRead != nil { - out.MemRead = *p.MemRead - } - if p.CPUCores != nil { - out.CPUCores = int(*p.CPUCores) - } - if p.CPUDutyCycle != nil { - out.CPUDutyCycle = *p.CPUDutyCycle - } - if p.MaxPingsPerWake != nil { - out.MaxPingsPerWake = int(*p.MaxPingsPerWake) - } - if p.SweperfTemplate != nil { - out.SweperfTemplate = *p.SweperfTemplate - } - if p.SweperfTotalSteps != nil { - out.SweperfTotalSteps = int(*p.SweperfTotalSteps) - } - if p.SweperfNumCycles != nil { - out.SweperfNumCycles = int(*p.SweperfNumCycles) - } - if p.SweperfPollIntervalMs != nil { - out.SweperfPollIntervalMs = int(*p.SweperfPollIntervalMs) - } - if p.AgentSessionScript != nil { - out.AgentSessionScript = *p.AgentSessionScript - } - if p.AgentSessionScriptFile != nil { - out.AgentSessionScriptFile = *p.AgentSessionScriptFile - } - if p.AgentSessionThinkScale != nil { - out.AgentSessionThinkScale = *p.AgentSessionThinkScale - } - if p.TotalActors != nil { - out.TotalActors = int(*p.TotalActors) - } - if p.SpawnConcurrency != nil { - out.SpawnConcurrency = int(*p.SpawnConcurrency) +// fetchAndApply fetches url into holder and, on a change, pushes the trace +// probability to the sampler and logs the config now in force. The error +// is a failed fetch or the holder's refusal, with the key it names. +func fetchAndApply(ctx context.Context, trigger, url string, holder *Holder, sampler ProbabilityUpdater) error { + body, err := Fetch(ctx, url) + if err != nil { + return err } - if p.ActorDeadline != nil { - out.ActorDeadline = time.Duration(*p.ActorDeadline * float64(time.Second)) + applied, changed, err := holder.Apply(body) + if err != nil { + return fmt.Errorf("%s: %w", url, err) } - if p.TargetRPS != nil { - out.TargetRPS = *p.TargetRPS + if !changed { + return nil } - return out + sampler.UpdateProbability(applied.Common.TraceProbability) + slog.Info("dynconfig applied", + slog.String("trigger", trigger), + slog.Float64("trace_probability", applied.Common.TraceProbability), + slog.String("config", fmt.Sprintf("%+v", applied.Class))) + return nil } // StartPoll fetches `url` every `interval` until `ctx` is done, and applies @@ -371,9 +329,11 @@ func (p payload) merge(current Config) Config { // it. The sample-rate sweep of benchmarking/observability.md is one such // shape: each of its steps holds 10 users. // -// `onError` gets each failed fetch. A caller must not exit the process there, -// as it does for a spawn: the worker holds the last good value, and one -// failed poll of a long run is not a reason to lose the run. +// `onError` gets each failed fetch and each refused payload. A caller must +// not exit the process there, as it does for a spawn: the worker holds the +// last good value, and one failed poll of a long run is not a reason to +// lose the run. Only a change goes to the log: a poll of each few seconds +// for the length of a soak would otherwise make a log that hides the run. func StartPoll( ctx context.Context, url string, @@ -394,7 +354,7 @@ func StartPoll( return case <-ticker.C: fetchCtx, cancel := context.WithTimeout(ctx, fetchTimeout) - next, err := Fetch(fetchCtx, url, holder.Load()) + err := fetchAndApply(fetchCtx, "poll", url, holder, sampler) cancel() if err != nil { // The end of the run stops a fetch that is in @@ -404,45 +364,7 @@ func StartPoll( return } onError(err) - continue - } - // Only a change goes to the log. A poll of each few seconds - // for the length of a soak makes a log that hides the run. - if next == holder.Load() { - continue } - holder.Store(next) - sampler.UpdateProbability(next.TraceProbability) - slog.Info("dynconfig applied", - slog.String("trigger", "poll"), - slog.Float64("trace_probability", next.TraceProbability), - slog.Duration("min_wait", next.MinWait), - slog.Duration("max_wait", next.MaxWait), - slog.Duration("min_live", next.MinLive), - slog.Duration("max_live", next.MaxLive), - slog.Int64("durdir_file_size_bytes", next.DurDirFileSize), - slog.String("resume_mode", next.ResumeMode), - slog.String("lifecycle_mode", next.LifecycleMode), - slog.String("durdir_read_mode", next.DurDirReadMode), - slog.String("durdir_template", next.DurDirTemplate), - slog.String("mem_target", next.MemTarget), - slog.String("mem_churn", next.MemChurn), - slog.String("mem_read", next.MemRead), - slog.Int("cpu_cores", next.CPUCores), - slog.Float64("cpu_duty_cycle", next.CPUDutyCycle), - slog.Int("max_pings_per_wake", next.MaxPingsPerWake), - slog.String("sweperf_template", next.SweperfTemplate), - slog.Int("sweperf_total_steps", next.SweperfTotalSteps), - slog.Int("sweperf_num_cycles", next.SweperfNumCycles), - slog.Int("sweperf_poll_interval_ms", next.SweperfPollIntervalMs), - slog.String("agentsession_script", next.AgentSessionScript), - slog.String("agentsession_script_file", next.AgentSessionScriptFile), - slog.Float64("agentsession_think_scale", next.AgentSessionThinkScale), - slog.Int("total_actors", next.TotalActors), - slog.Int("spawn_concurrency", next.SpawnConcurrency), - slog.Duration("actor_deadline", next.ActorDeadline), - slog.Float64("target_rps", next.TargetRPS), - ) } } }() @@ -451,53 +373,22 @@ func StartPoll( // SubscribeSpawn registers a boomer Events handler that fetches `url` on // each spawn message and applies the result to `holder` + `sampler`. Locust // sends a spawn message for every ramp step, so a long ramp fetches once per -// second. `onError` is invoked when a fetch fails, with `fetched` true if an -// earlier spawn fetch succeeded: the holder then still has a value from the -// master, and the caller can keep running on it. With `fetched` false the -// worker has only its command-line defaults, and callers typically exit. -// Returns an error if the event subscription itself fails (handler signature -// mismatch), which is a programmer error and should be treated as fatal too. +// second. `onError` is invoked when a fetch fails or the holder refuses the +// payload, with `fetched` true if an earlier spawn fetch was applied: the +// holder then still has a value from the master, and the caller can keep +// running on it. With `fetched` false the worker has only its command-line +// values, and callers typically exit. Returns an error if the event +// subscription itself fails (handler signature mismatch), which is a +// programmer error and should be treated as fatal too. func SubscribeSpawn(url string, holder *Holder, sampler ProbabilityUpdater, fetchTimeout time.Duration, onError func(err error, fetched bool)) error { var fetched atomic.Bool return boomer.Events.Subscribe("boomer:spawn", func(spawnCount int, spawnRate float64) { ctx, cancel := context.WithTimeout(context.Background(), fetchTimeout) defer cancel() - next, err := Fetch(ctx, url, holder.Load()) - if err != nil { + if err := fetchAndApply(ctx, "spawn", url, holder, sampler); err != nil { onError(err, fetched.Load()) return } fetched.Store(true) - holder.Store(next) - sampler.UpdateProbability(next.TraceProbability) - slog.Info("dynconfig applied", - slog.Float64("trace_probability", next.TraceProbability), - slog.Duration("min_wait", next.MinWait), - slog.Duration("max_wait", next.MaxWait), - slog.Duration("min_live", next.MinLive), - slog.Duration("max_live", next.MaxLive), - slog.Int64("durdir_file_size_bytes", next.DurDirFileSize), - slog.String("resume_mode", next.ResumeMode), - slog.String("lifecycle_mode", next.LifecycleMode), - slog.String("durdir_read_mode", next.DurDirReadMode), - slog.String("durdir_template", next.DurDirTemplate), - slog.String("mem_target", next.MemTarget), - slog.String("mem_churn", next.MemChurn), - slog.String("mem_read", next.MemRead), - slog.Int("cpu_cores", next.CPUCores), - slog.Float64("cpu_duty_cycle", next.CPUDutyCycle), - slog.Int("max_pings_per_wake", next.MaxPingsPerWake), - slog.String("sweperf_template", next.SweperfTemplate), - slog.Int("sweperf_total_steps", next.SweperfTotalSteps), - slog.Int("sweperf_num_cycles", next.SweperfNumCycles), - slog.Int("sweperf_poll_interval_ms", next.SweperfPollIntervalMs), - slog.String("agentsession_script", next.AgentSessionScript), - slog.String("agentsession_script_file", next.AgentSessionScriptFile), - slog.Float64("agentsession_think_scale", next.AgentSessionThinkScale), - slog.Int("total_actors", next.TotalActors), - slog.Int("spawn_concurrency", next.SpawnConcurrency), - slog.Duration("actor_deadline", next.ActorDeadline), - slog.Float64("target_rps", next.TargetRPS), - ) }) } diff --git a/internal/benchmarking/boomer/dynconfig/dynconfig_test.go b/internal/benchmarking/boomer/dynconfig/dynconfig_test.go index eff1e5c332..d5addf0195 100644 --- a/internal/benchmarking/boomer/dynconfig/dynconfig_test.go +++ b/internal/benchmarking/boomer/dynconfig/dynconfig_test.go @@ -19,6 +19,7 @@ import ( "fmt" "net/http" "net/http/httptest" + "strings" "sync" "sync/atomic" "testing" @@ -27,240 +28,204 @@ import ( "github.com/myzhan/boomer" ) -func TestParseValid(t *testing.T) { - jsonBlob := []byte(`{ - "trace_probability": 0.5, - "min_wait_time": 0.1, - "max_wait_time": 0.5, - "min_live_time": 9, - "max_live_time": 14, - "durdir_file_size_bytes": 1048576, - "resume_mode": "explicit", - "lifecycle_mode": "pause", - "durdir_read_mode": "data", - "durdir_template": "glutton-durdir-data", - "cpu_cores": 2, - "cpu_duty_cycle": 0.1, - "sweperf_template": "sweperf-astropy-7336", - "sweperf_total_steps": 21, - "sweperf_num_cycles": 4, - "sweperf_poll_interval_ms": 100, - "agentsession_script": "coding-session", - "agentsession_script_file": "/etc/agentsession/script.yaml", - "total_actors": 50, - "spawn_concurrency": 5, - "actor_deadline": 60.0, - "target_rps": 2500.5 - }`) +// knobs is the kind of struct a user class declares: its slice of the +// payload with its own defaults and rules. +type knobs struct { + WaitTime + Lifecycle + Template string `json:"test_template"` + Workers int `json:"test_workers"` +} + +var knobsCodec = Typed[knobs]{ + Defaults: knobs{Template: "stock", Workers: 1}, + Validate: func(k knobs) error { + if err := k.WaitTime.Validate(); err != nil { + return err + } + if err := k.Lifecycle.Validate(); err != nil { + return err + } + if k.Workers < 1 { + return fmt.Errorf("test_workers must be positive: %d", k.Workers) + } + return nil + }, +} - cfg, err := Parse(jsonBlob, Config{}) +func mustApply(t *testing.T, h *Holder, payload string) bool { + t.Helper() + _, changed, err := h.Apply([]byte(payload)) if err != nil { - t.Fatalf("Parse failed: %v", err) + t.Fatalf("Apply(%s): %v", payload, err) } + return changed +} - if cfg.TraceProbability != 0.5 { - t.Errorf("TraceProbability: got %f, want 0.5", cfg.TraceProbability) - } - if cfg.MinWait != 100*time.Millisecond { - t.Errorf("MinWait: got %v, want 100ms", cfg.MinWait) - } - if cfg.MaxWait != 500*time.Millisecond { - t.Errorf("MaxWait: got %v, want 500ms", cfg.MaxWait) - } - if cfg.MinLive != 9*time.Second { - t.Errorf("MinLive: got %v, want 9s", cfg.MinLive) - } - if cfg.MaxLive != 14*time.Second { - t.Errorf("MaxLive: got %v, want 14s", cfg.MaxLive) - } - if cfg.DurDirFileSize != 1048576 { - t.Errorf("DurDirFileSize: got %d, want 1048576", cfg.DurDirFileSize) - } - if cfg.ResumeMode != ResumeModeExplicit { - t.Errorf("ResumeMode: got %q, want %q", cfg.ResumeMode, ResumeModeExplicit) +// A payload stands on its own: it sets the keys it carries, puts the ones +// it nulls or omits back at the class's defaults, converts seconds to +// durations, and ignores keys the class does not name. +func TestApplyDecodesOverDefaults(t *testing.T) { + h := NewHolder(knobsCodec) + if got := Get[knobs](h); got != knobsCodec.Defaults { + t.Fatalf("fresh holder = %+v, want the defaults", got) + } + mustApply(t, h, `{ + "min_wait_time": 0.5, "max_wait_time": 2, + "resume_mode": "implicit", "lifecycle_mode": "none", + "test_workers": 3, + "someone_elses_key": "ignored" + }`) + want := knobs{ + WaitTime: WaitTime{MinWait: Seconds(500 * time.Millisecond), MaxWait: Seconds(2 * time.Second)}, + Lifecycle: Lifecycle{ResumeMode: ResumeModeImplicit, LifecycleMode: LifecycleModeNone}, + Template: "stock", + Workers: 3, } - if cfg.LifecycleMode != LifecycleModePause { - t.Errorf("LifecycleMode: got %q, want %q", cfg.LifecycleMode, LifecycleModePause) + if got := Get[knobs](h); got != want { + t.Errorf("after first payload =\n %+v, want\n %+v", got, want) } - if cfg.DurDirReadMode != ReadModeData { - t.Errorf("DurDirReadMode: got %q, want %q", cfg.DurDirReadMode, ReadModeData) + // A cleared form field comes as null, and a key the master does not + // know is absent; both are back at the default, not at the last value. + mustApply(t, h, `{"test_template": "big", "test_workers": null, "max_wait_time": null}`) + want = knobsCodec.Defaults + want.Template = "big" + if got := Get[knobs](h); got != want { + t.Errorf("after nulls and omissions =\n %+v, want\n %+v", got, want) } - if cfg.DurDirTemplate != "glutton-durdir-data" { - t.Errorf("DurDirTemplate: got %q, want glutton-durdir-data", cfg.DurDirTemplate) + if changed := mustApply(t, h, `{"test_template": "big"}`); changed { + t.Error("a payload that changes nothing reported a change") } - if cfg.CPUCores != 2 { - t.Errorf("CPUCores: got %d, want 2", cfg.CPUCores) + if changed := mustApply(t, h, `{}`); !changed { + t.Error("an empty object did not report the return to the defaults") } - if cfg.CPUDutyCycle != 0.1 { - t.Errorf("CPUDutyCycle: got %f, want 0.1", cfg.CPUDutyCycle) + if got := Get[knobs](h); got != knobsCodec.Defaults { + t.Errorf("after an empty object = %+v, want the defaults", got) } - if cfg.SweperfTemplate != "sweperf-astropy-7336" { - t.Errorf("SweperfTemplate: got %q, want sweperf-astropy-7336", cfg.SweperfTemplate) +} + +// Apply hands back the snapshot the payload left in the holder, so a log +// line written from it cannot show a concurrent Apply's values. +func TestApplyReturnsWhatItStored(t *testing.T) { + h := NewHolder(knobsCodec) + applied, changed, err := h.Apply([]byte(`{"test_workers": 4, "trace_probability": 0.5}`)) + if err != nil || !changed { + t.Fatalf("Apply = changed %v, err %v; want a change", changed, err) } - if cfg.SweperfTotalSteps != 21 { - t.Errorf("SweperfTotalSteps: got %d, want 21", cfg.SweperfTotalSteps) + if applied.Class != h.Load() || applied.Common != h.Common() { + t.Errorf("Apply returned %+v, holder has %+v / %+v", applied, h.Load(), h.Common()) } - if cfg.SweperfNumCycles != 4 { - t.Errorf("SweperfNumCycles: got %d, want 4", cfg.SweperfNumCycles) + if got := applied.Class.(knobs).Workers; got != 4 { + t.Errorf("applied test_workers = %d, want 4", got) } - if cfg.SweperfPollIntervalMs != 100 { - t.Errorf("SweperfPollIntervalMs: got %d, want 100", cfg.SweperfPollIntervalMs) +} + +// A payload the class's rules or Common refuse leaves the holder as it was, +// so a bad value from the master cannot take a run down. An empty body is +// one such payload: a 200 with nothing in it must not read as every +// default. +func TestApplyRefuses(t *testing.T) { + h := NewHolder(knobsCodec) + mustApply(t, h, `{"test_workers": 2, "trace_probability": 0.5}`) + before := Get[knobs](h) + for name, payload := range map[string]string{ + "class rule": `{"test_workers": 0}`, + "embedded": `{"max_wait_time": 1, "min_wait_time": 2}`, + "mode": `{"lifecycle_mode": "hibernate"}`, + "wrong type": `{"test_workers": "three"}`, + "bad seconds": `{"min_wait_time": "soon"}`, + "common rule": `{"trace_probability": 1.5}`, + "not json": ``, + "empty body": ``, + } { + t.Run(name, func(t *testing.T) { + _, changed, err := h.Apply([]byte(payload)) + if err == nil || changed { + t.Fatalf("Apply(%s) = changed %v, err %v; want a refusal", payload, changed, err) + } + if got := Get[knobs](h); got != before { + t.Errorf("a refused payload changed the config to %+v", got) + } + if got := h.Common().TraceProbability; got != 0.5 { + t.Errorf("a refused payload changed trace_probability to %v", got) + } + }) } - if cfg.AgentSessionScript != "coding-session" { - t.Errorf("AgentSessionScript: got %q, want coding-session", cfg.AgentSessionScript) +} + +// Get on the wrong type is a programming error that must not pass +// silently; a nil holder is the zero value, for classes that run without +// one in tests. +func TestGetTypeChecks(t *testing.T) { + if got := Get[knobs](nil); got != (knobs{}) { + t.Errorf("Get on a nil holder = %+v, want zero", got) + } + defer func() { + if recover() == nil { + t.Error("Get with the wrong type did not panic") + } + }() + Get[Common](NewHolder(knobsCodec)) +} + +func TestStaticAndNilCodec(t *testing.T) { + h := Static(knobs{Template: "t", WaitTime: WaitTime{MaxWait: Seconds(3 * time.Second)}}) + if got := Get[knobs](h); got.Template != "t" || got.MaxWait != Seconds(3*time.Second) { + t.Errorf("Static round trip = %+v", got) } - if cfg.AgentSessionScriptFile != "/etc/agentsession/script.yaml" { - t.Errorf("AgentSessionScriptFile: got %q", cfg.AgentSessionScriptFile) + // A class without knobs still gets Common. + h = NewHolder(nil) + mustApply(t, h, `{"trace_probability": 0.25, "anything": 1}`) + if got := h.Common().TraceProbability; got != 0.25 { + t.Errorf("trace_probability = %v, want 0.25", got) } - if cfg.TotalActors != 50 { - t.Errorf("TotalActors: got %d, want 50", cfg.TotalActors) +} + +// A config printed with %+v, as the dynconfig applied log line does, shows +// its durations as durations, not as counts of nanoseconds. +func TestSecondsPrintsAsDuration(t *testing.T) { + line := fmt.Sprintf("%+v", knobs{WaitTime: WaitTime{MaxWait: Seconds(500 * time.Millisecond)}}) + if !strings.Contains(line, "MaxWait:500ms") { + t.Errorf("printed config %q, want MaxWait:500ms in it", line) } - if cfg.SpawnConcurrency != 5 { - t.Errorf("SpawnConcurrency: got %d, want 5", cfg.SpawnConcurrency) +} + +func TestWaitTimeDraw(t *testing.T) { + window := WaitTime{MinWait: Seconds(10 * time.Millisecond), MaxWait: Seconds(50 * time.Millisecond)} + for i := 0; i < 100; i++ { + if got := window.Draw(); got < 10*time.Millisecond || got > 50*time.Millisecond { + t.Fatalf("Draw = %v, outside the window", got) + } } - if cfg.ActorDeadline != 60*time.Second { - t.Errorf("ActorDeadline: got %v, want 60s", cfg.ActorDeadline) + inverted := WaitTime{MinWait: Seconds(100 * time.Millisecond), MaxWait: Seconds(50 * time.Millisecond)} + if got := inverted.Draw(); got != 100*time.Millisecond { + t.Errorf("inverted Draw = %v, want the lower bound", got) } - if cfg.TargetRPS != 2500.5 { - t.Errorf("TargetRPS: got %v, want 2500.5", cfg.TargetRPS) + if got := Uniform(0, 0); got != 0 { + t.Errorf("Uniform(0, 0) = %v, want 0", got) } -} - -func TestParseInvalidValues(t *testing.T) { - tests := []struct { - name string - json string - }{ - { - name: "negative trace probability", - json: `{"trace_probability": -0.1}`, - }, - { - name: "trace probability > 1.0", - json: `{"trace_probability": 1.5}`, - }, - { - name: "negative min wait", - json: `{"min_wait_time": -1.0}`, - }, - { - name: "negative max wait", - json: `{"max_wait_time": -1.0}`, - }, - { - name: "max wait less than min wait", - json: `{"min_wait_time": 2.0, "max_wait_time": 1.0}`, - }, - { - name: "negative min live", - json: `{"min_live_time": -1.0}`, - }, - { - name: "negative max live", - json: `{"max_live_time": -1.0}`, - }, - { - name: "max live less than min live", - json: `{"min_live_time": 14.0, "max_live_time": 9.0}`, - }, - { - name: "negative file size", - json: `{"durdir_file_size_bytes": -100}`, - }, - { - name: "file size exceeds 2 GiB", - json: `{"durdir_file_size_bytes": 2147483648}`, - }, - { - name: "invalid resume mode", - json: `{"resume_mode": "invalid_mode"}`, - }, - { - name: "invalid lifecycle mode", - json: `{"lifecycle_mode": "invalid_lifecycle"}`, - }, - { - name: "invalid read mode", - json: `{"durdir_read_mode": "invalid_read"}`, - }, - { - name: "negative cpu cores", - json: `{"cpu_cores": -1}`, - }, - { - name: "negative cpu duty cycle", - json: `{"cpu_duty_cycle": -0.1}`, - }, - { - name: "cpu duty cycle > 1.0", - json: `{"cpu_duty_cycle": 1.5}`, - }, - { - name: "negative sweperf total steps", - json: `{"sweperf_total_steps": -1}`, - }, - { - name: "negative sweperf num cycles", - json: `{"sweperf_num_cycles": -1}`, - }, - { - name: "negative target rps", - json: `{"target_rps": -1}`, - }, - { - name: "negative sweperf poll interval", - json: `{"sweperf_poll_interval_ms": -1}`, - }, - { - name: "negative total actors", - json: `{"total_actors": -1}`, - }, - { - name: "negative spawn concurrency", - json: `{"spawn_concurrency": -1}`, - }, - { - name: "negative actor deadline", - json: `{"actor_deadline": -1.0}`, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - _, err := Parse([]byte(tt.json), Config{}) - if err == nil { - t.Errorf("expected Parse to fail for %s, got nil error", tt.name) - } - }) + if got := Uniform(3*time.Second, 3*time.Second); got != 3*time.Second { + t.Errorf("Uniform(3s, 3s) = %v, want 3s", got) } } -func TestFetchValidAndInvalid(t *testing.T) { +func TestFetchReportsErrors(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/valid", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"resume_mode": "implicit", "durdir_read_mode": "digest"}`)) + _, _ = w.Write([]byte(`{"resume_mode": "implicit"}`)) }) - mux.HandleFunc("/invalid", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"resume_mode": "bogus"}`)) + mux.HandleFunc("/busy", func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "busy", http.StatusServiceUnavailable) }) ts := httptest.NewServer(mux) defer ts.Close() - ctx := context.Background() - - cfg, err := Fetch(ctx, ts.URL+"/valid", Config{}) - if err != nil { - t.Fatalf("Fetch valid failed: %v", err) + body, err := Fetch(context.Background(), ts.URL+"/valid") + if err != nil || string(body) != `{"resume_mode": "implicit"}` { + t.Errorf("Fetch valid = %q, %v", body, err) } - if cfg.ResumeMode != ResumeModeImplicit || cfg.DurDirReadMode != ReadModeDigest { - t.Errorf("Fetch valid values mismatch: got %+v", cfg) - } - - _, err = Fetch(ctx, ts.URL+"/invalid", Config{}) - if err == nil { - t.Errorf("expected Fetch invalid to fail, got nil") + if _, err := Fetch(context.Background(), ts.URL+"/busy"); err == nil { + t.Error("Fetch of a 503 returned no error") } } @@ -301,7 +266,7 @@ func TestStartPollAppliesAChange(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - holder := NewHolder(Config{}) + holder := NewHolder(knobsCodec) sampler := &fakeSampler{} StartPoll(ctx, ts.URL, holder, sampler, 10*time.Millisecond, time.Second, func(err error) { t.Errorf("unexpected poll error: %v", err) }) @@ -314,7 +279,7 @@ func TestStartPollAppliesAChange(t *testing.T) { } time.Sleep(10 * time.Millisecond) } - if got := holder.Load().TraceProbability; got != 0.5 { + if got := holder.Common().TraceProbability; got != 0.5 { t.Fatalf("holder trace_probability = %v, want 0.5", got) } @@ -326,17 +291,47 @@ func TestStartPollAppliesAChange(t *testing.T) { } } +// A poll that brings a value the class refuses goes to onError and leaves +// the holder as it was. +func TestStartPollReportsRefusedConfig(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"test_workers": 0}`)) + })) + defer ts.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + holder := NewHolder(knobsCodec) + errs := make(chan error, 1) + StartPoll(ctx, ts.URL, holder, &fakeSampler{}, 10*time.Millisecond, time.Second, func(err error) { + select { + case errs <- err: + default: + } + }) + select { + case err := <-errs: + if !strings.Contains(err.Error(), "test_workers") { + t.Errorf("onError got %v, want the refusing key named", err) + } + case <-time.After(2 * time.Second): + t.Fatal("a refused poll never reached onError") + } + if got := Get[knobs](holder); got != knobsCodec.Defaults { + t.Errorf("a refused poll changed the holder to %+v", got) + } +} + func TestStartPollStopsWithTheContext(t *testing.T) { var hits atomic.Int64 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { hits.Add(1) - w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{}`)) })) defer ts.Close() ctx, cancel := context.WithCancel(context.Background()) - StartPoll(ctx, ts.URL, NewHolder(Config{}), &fakeSampler{}, + StartPoll(ctx, ts.URL, NewHolder(nil), &fakeSampler{}, 10*time.Millisecond, time.Second, func(error) {}) time.Sleep(60 * time.Millisecond) cancel() @@ -359,7 +354,6 @@ func TestSubscribeSpawnReportsPriorSuccess(t *testing.T) { http.Error(w, "busy", http.StatusServiceUnavailable) return } - w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"trace_probability": 0.25}`)) })) defer ts.Close() @@ -369,7 +363,7 @@ func TestSubscribeSpawnReportsPriorSuccess(t *testing.T) { fetched bool } var calls []call - holder := NewHolder(Config{}) + holder := NewHolder(nil) // boomer.Events is a process-wide bus and SubscribeSpawn owns the handler // value, so the subscription outlives this test. No other test publishes // boomer:spawn, and each test uses its own holder, so that is harmless. @@ -381,8 +375,8 @@ func TestSubscribeSpawnReportsPriorSuccess(t *testing.T) { fail.Store(true) boomer.Events.Publish("boomer:spawn", 1, 1.0) - if len(calls) != 1 || calls[0].fetched { - t.Fatalf("after a failure with no prior success: calls = %+v, want one with fetched=false", calls) + if len(calls) != 1 || calls[0].fetched || calls[0].err == nil { + t.Fatalf("after a failure with no prior success: calls = %+v, want one error with fetched=false", calls) } fail.Store(false) @@ -390,7 +384,7 @@ func TestSubscribeSpawnReportsPriorSuccess(t *testing.T) { if len(calls) != 1 { t.Fatalf("a successful fetch invoked onError: %+v", calls) } - if got := holder.Load().TraceProbability; got != 0.25 { + if got := holder.Common().TraceProbability; got != 0.25 { t.Fatalf("holder trace_probability = %v, want 0.25", got) } @@ -399,7 +393,7 @@ func TestSubscribeSpawnReportsPriorSuccess(t *testing.T) { if len(calls) != 2 || !calls[1].fetched { t.Fatalf("after a failure with a prior success: calls = %+v, want a second with fetched=true", calls) } - if got := holder.Load().TraceProbability; got != 0.25 { + if got := holder.Common().TraceProbability; got != 0.25 { t.Fatalf("failed fetch changed holder trace_probability to %v", got) } } @@ -412,7 +406,7 @@ func TestStartPollZeroInterval(t *testing.T) { })) defer ts.Close() - StartPoll(context.Background(), ts.URL, NewHolder(Config{}), &fakeSampler{}, + StartPoll(context.Background(), ts.URL, NewHolder(nil), &fakeSampler{}, 0, time.Second, func(error) {}) time.Sleep(50 * time.Millisecond) if hits.Load() != 0 { diff --git a/internal/benchmarking/boomer/glutton/cpuload_test.go b/internal/benchmarking/boomer/glutton/cpuload_test.go index 162cc70164..bb487fb3b0 100644 --- a/internal/benchmarking/boomer/glutton/cpuload_test.go +++ b/internal/benchmarking/boomer/glutton/cpuload_test.go @@ -19,13 +19,12 @@ import ( "net/http" "testing" - "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" "go.ate.dev/substrate/internal/benchmarking/glutton/fake" ) func TestEnsureCPULoadRequestsConfiguredLoad(t *testing.T) { srv := &fake.Server{} - u := newTestGluttonActor(t, srv, dynconfig.Config{CPUCores: 2, CPUDutyCycle: 0.1}) + u := newTestGluttonActor(t, srv, gluttonKnobs{CPUCores: 2, CPUDutyCycle: 0.1}) u.ensureCPULoad(context.Background()) @@ -49,7 +48,7 @@ func TestEnsureCPULoadRequestsConfiguredLoad(t *testing.T) { func TestEnsureCPULoadDisabledByDefault(t *testing.T) { srv := &fake.Server{} - u := newTestGluttonActor(t, srv, dynconfig.Config{}) + u := newTestGluttonActor(t, srv, gluttonKnobs{}) u.ensureCPULoad(context.Background()) @@ -63,7 +62,7 @@ func TestEnsureCPULoadDisabledByDefault(t *testing.T) { func TestEnsureCPULoadRetriesAfterFailure(t *testing.T) { srv := &fake.Server{Status: http.StatusServiceUnavailable} - u := newTestGluttonActor(t, srv, dynconfig.Config{CPUCores: 1, CPUDutyCycle: 0.5}) + u := newTestGluttonActor(t, srv, gluttonKnobs{CPUCores: 1, CPUDutyCycle: 0.5}) ctx := context.Background() u.ensureCPULoad(ctx) diff --git a/internal/benchmarking/boomer/glutton/durdir.go b/internal/benchmarking/boomer/glutton/durdir.go index 65f6bfa2b2..7adc227a06 100644 --- a/internal/benchmarking/boomer/glutton/durdir.go +++ b/internal/benchmarking/boomer/glutton/durdir.go @@ -22,7 +22,6 @@ import ( "fmt" "io" "log/slog" - "math/rand/v2" "net/http" "strings" "sync" @@ -65,6 +64,7 @@ func init() { Name: "durdir", LocustFile: "durdir.py", UserClass: durDirUserClass, + Config: durDirCodec, Init: initDurDir, }) } @@ -85,19 +85,14 @@ type durDirRuntime struct { } func (r *durDirRuntime) dynamicWait() time.Duration { - cfg := r.cfg.Dyn.Load() - if cfg.MaxWait <= cfg.MinWait { - return cfg.MinWait - } - jitter := cfg.MaxWait - cfg.MinWait - return cfg.MinWait + time.Duration(rand.Float64()*float64(jitter)) + return dynconfig.Get[durDirKnobs](r.cfg.Dyn).WaitTime.Draw() } func (r *durDirRuntime) iterate() { gid := boomerutil.GoroutineID() val, loaded := r.users.Load(gid) if !loaded { - dynCfg := r.cfg.Dyn.Load() + dynCfg := dynconfig.Get[durDirKnobs](r.cfg.Dyn) u, err := r.startUser(context.Background(), dynCfg) if err != nil { slog.Warn("durdir on_start failed; goroutine will retry next iter", @@ -109,15 +104,15 @@ func (r *durDirRuntime) iterate() { } user := val.(*durDirUser) - dynCfg := r.cfg.Dyn.Load() + dynCfg := dynconfig.Get[durDirKnobs](r.cfg.Dyn) ctx := context.Background() user.step(ctx, dynCfg) time.Sleep(r.dynamicWait()) } -func (r *durDirRuntime) startUser(ctx context.Context, dynCfg dynconfig.Config) (*durDirUser, error) { - tmpl := dynCfg.DurDirTemplate +func (r *durDirRuntime) startUser(ctx context.Context, dynCfg durDirKnobs) (*durDirUser, error) { + tmpl := dynCfg.Template if tmpl == "" { tmpl = defaultDurTemplate } @@ -146,7 +141,7 @@ func (r *durDirRuntime) startUser(ctx context.Context, dynCfg dynconfig.Config) } func (r *durDirRuntime) shutdown(ctx context.Context) { - dynCfg := r.cfg.Dyn.Load() + dynCfg := dynconfig.Get[durDirKnobs](r.cfg.Dyn) r.users.Range(func(_, val any) bool { u := val.(*durDirUser) u.hibernateAndDelete(ctx, dynCfg) @@ -214,9 +209,10 @@ func (u *durDirUser) resume(ctx context.Context, mode string) bool { } func (u *durDirUser) pause(ctx context.Context) { - _ = u.tracedCall(ctx, "PauseActor", func(callCtx context.Context, tr *metadata.MD) error { - _, err := u.cfg.APIStub.PauseActor(callCtx, &ateapipb.PauseActorRequest{ - Actor: u.ref(), + _ = u.tracedCall(ctx, "SuspendActorAsync", func(callCtx context.Context, tr *metadata.MD) error { + _, err := u.cfg.APIStub.SuspendActor(callCtx, &ateapipb.SuspendActorRequest{ + Actor: u.ref(), + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, }, grpc.Trailer(tr)) return err }) @@ -225,13 +221,14 @@ func (u *durDirUser) pause(ctx context.Context) { func (u *durDirUser) suspend(ctx context.Context) { _ = u.tracedCall(ctx, "SuspendActor", func(callCtx context.Context, tr *metadata.MD) error { _, err := u.cfg.APIStub.SuspendActor(callCtx, &ateapipb.SuspendActorRequest{ - Actor: u.ref(), + Actor: u.ref(), + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, }, grpc.Trailer(tr)) return err }) } -func (u *durDirUser) hibernate(ctx context.Context, dynCfg dynconfig.Config) { +func (u *durDirUser) hibernate(ctx context.Context, dynCfg durDirKnobs) { if dynCfg.LifecycleMode == dynconfig.LifecycleModePause { u.pause(ctx) } else { @@ -242,14 +239,16 @@ func (u *durDirUser) hibernate(ctx context.Context, dynCfg dynconfig.Config) { // hibernateAndDelete hibernates (suspends or pauses) the actor before deleting it. // The hibernate call is unmetered (teardown precondition, not benchmark latency), // while the delete is metered so true leaks still surface in failures.csv. -func (u *durDirUser) hibernateAndDelete(ctx context.Context, dynCfg dynconfig.Config) { +func (u *durDirUser) hibernateAndDelete(ctx context.Context, dynCfg durDirKnobs) { if dynCfg.LifecycleMode == dynconfig.LifecycleModePause { - _, _ = u.cfg.APIStub.PauseActor(ctx, &ateapipb.PauseActorRequest{ - Actor: u.ref(), + _, _ = u.cfg.APIStub.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: u.ref(), + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, }) } else { _, _ = u.cfg.APIStub.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ - Actor: u.ref(), + Actor: u.ref(), + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, }) } u.delete(ctx) @@ -287,19 +286,19 @@ func (u *durDirUser) tracedCall(ctx context.Context, name string, do func(contex return nil } -func (u *durDirUser) params(dynCfg dynconfig.Config) (int64, gluttonpb.ReadMode) { - fileSize := dynCfg.DurDirFileSize +func (u *durDirUser) params(dynCfg durDirKnobs) (int64, gluttonpb.ReadMode) { + fileSize := dynCfg.FileSize if fileSize <= 0 { fileSize = defaultFileSize } readMode := gluttonpb.ReadMode_READ_MODE_DATA - if dynCfg.DurDirReadMode == dynconfig.ReadModeDigest { + if dynCfg.ReadMode == readModeDigest { readMode = gluttonpb.ReadMode_READ_MODE_DIGEST_ONLY } return fileSize, readMode } -func (u *durDirUser) step(ctx context.Context, dynCfg dynconfig.Config) { +func (u *durDirUser) step(ctx context.Context, dynCfg durDirKnobs) { fileSize, readMode := u.params(dynCfg) // 1. Suspend or pause actor @@ -326,7 +325,7 @@ func (u *durDirUser) step(ctx context.Context, dynCfg dynconfig.Config) { } } -func (u *durDirUser) bootstrap(ctx context.Context, dynCfg dynconfig.Config) error { +func (u *durDirUser) bootstrap(ctx context.Context, dynCfg durDirKnobs) error { fileSize, readMode := u.params(dynCfg) if !u.resume(ctx, dynCfg.ResumeMode) { diff --git a/internal/benchmarking/boomer/glutton/durdir_test.go b/internal/benchmarking/boomer/glutton/durdir_test.go index bcc11330cc..a1ab0524ff 100644 --- a/internal/benchmarking/boomer/glutton/durdir_test.go +++ b/internal/benchmarking/boomer/glutton/durdir_test.go @@ -29,6 +29,7 @@ import ( "go.ate.dev/substrate/internal/benchmarking/boomer/userclass" "go.ate.dev/substrate/internal/benchmarking/glutton/fake" gluttonpb "go.ate.dev/substrate/internal/proto/glutton" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) func TestDurDirLoopSequence(t *testing.T) { @@ -38,25 +39,29 @@ func TestDurDirLoopSequence(t *testing.T) { lifecycleMode string wantGRPCCall []string wantHTTPCall []string + wantMode ateapipb.DurabilityMode }{ { name: "explicit resume mode", resumeMode: dynconfig.ResumeModeExplicit, wantGRPCCall: []string{"SuspendActor", "ResumeActor"}, wantHTTPCall: []string{fake.ReadDiskRoute, fake.ReadDiskRoute, fake.WriteDiskRoute}, + wantMode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS, }, { name: "implicit resume mode", resumeMode: dynconfig.ResumeModeImplicit, wantGRPCCall: []string{"SuspendActor"}, // No ResumeActor RPC! wantHTTPCall: []string{fake.ReadDiskRoute, fake.ReadDiskRoute, fake.WriteDiskRoute}, + wantMode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS, }, { name: "pause lifecycle mode", lifecycleMode: dynconfig.LifecycleModePause, resumeMode: dynconfig.ResumeModeExplicit, - wantGRPCCall: []string{"PauseActor", "ResumeActor"}, + wantGRPCCall: []string{"SuspendActor(async)", "ResumeActor"}, wantHTTPCall: []string{fake.ReadDiskRoute, fake.ReadDiskRoute, fake.WriteDiskRoute}, + wantMode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS, }, } @@ -66,16 +71,14 @@ func TestDurDirLoopSequence(t *testing.T) { fakeCtrl := &fakeControlClient{} cfg := &userclass.Config{ APIStub: fakeCtrl, - Dyn: dynconfig.NewHolder(dynconfig.Config{ - ResumeMode: tc.resumeMode, - LifecycleMode: tc.lifecycleMode, + Dyn: dynconfig.Static(durDirKnobs{ + Lifecycle: dynconfig.Lifecycle{ResumeMode: tc.resumeMode, LifecycleMode: tc.lifecycleMode}, }), } du := newTestDurDirUser(t, srv, cfg) du.expectedDigest = srv.HexDigest() - dynCfg := cfg.Dyn.Load() - du.step(context.Background(), dynCfg) + du.step(context.Background(), dynconfig.Get[durDirKnobs](cfg.Dyn)) if got := fakeCtrl.recordedCalls(); !reflect.DeepEqual(got, tc.wantGRPCCall) { t.Errorf("gRPC calls: got %v, want %v", got, tc.wantGRPCCall) @@ -83,6 +86,11 @@ func TestDurDirLoopSequence(t *testing.T) { if got := srv.RecordedPaths(); !reflect.DeepEqual(got, tc.wantHTTPCall) { t.Errorf("HTTP calls: got %v, want %v", got, tc.wantHTTPCall) } + for _, req := range fakeCtrl.recordedSuspendRequests() { + if got := req.GetOptions().GetMode(); got != tc.wantMode { + t.Errorf("suspend mode = %v, want %v", got, tc.wantMode) + } + } }) } } @@ -195,14 +203,14 @@ func TestDurDirBootstrapUsesConfiguredResumeMode(t *testing.T) { fakeCtrl := &fakeControlClient{} cfg := newTestConfig(t, srv, &userclass.Config{ APIStub: fakeCtrl, - Dyn: dynconfig.NewHolder(dynconfig.Config{ - DurDirFileSize: int64(len(srv.Data)), - ResumeMode: tc.resumeMode, + Dyn: dynconfig.Static(durDirKnobs{ + FileSize: int64(len(srv.Data)), + Lifecycle: dynconfig.Lifecycle{ResumeMode: tc.resumeMode}, }), }) rt := &durDirRuntime{cfg: cfg} - _, err := rt.startUser(context.Background(), cfg.Dyn.Load()) + _, err := rt.startUser(context.Background(), dynconfig.Get[durDirKnobs](cfg.Dyn)) if err != nil { t.Fatalf("startUser failed: %v", err) } @@ -221,14 +229,14 @@ func TestDurDirBootstrapFailureSuspendsBeforeDelete(t *testing.T) { fakeCtrl := &fakeControlClient{} cfg := newTestConfig(t, srv, &userclass.Config{ APIStub: fakeCtrl, - Dyn: dynconfig.NewHolder(dynconfig.Config{ - DurDirFileSize: 1024, - ResumeMode: dynconfig.ResumeModeExplicit, + Dyn: dynconfig.Static(durDirKnobs{ + FileSize: 1024, + Lifecycle: dynconfig.Lifecycle{ResumeMode: dynconfig.ResumeModeExplicit}, }), }) rt := &durDirRuntime{cfg: cfg} - _, err := rt.startUser(context.Background(), cfg.Dyn.Load()) + _, err := rt.startUser(context.Background(), dynconfig.Get[durDirKnobs](cfg.Dyn)) if err == nil { t.Fatalf("startUser expected error on failing server, got nil") } @@ -237,49 +245,63 @@ func TestDurDirBootstrapFailureSuspendsBeforeDelete(t *testing.T) { if len(calls) < 2 || calls[len(calls)-2] != "SuspendActor" || calls[len(calls)-1] != "DeleteActor" { t.Errorf("recordedCalls must end with [SuspendActor, DeleteActor], got %v", calls) } -} - -func TestDurDirShutdownSuspendsBeforeDelete(t *testing.T) { - fakeCtrl := &fakeControlClient{} - cfg := &userclass.Config{ - APIStub: fakeCtrl, + suspends := fakeCtrl.recordedSuspendRequests() + if len(suspends) == 0 { + t.Fatalf("got no SuspendActor requests") } - du := newTestDurDirUser(t, &fake.Server{}, cfg) - - rt := &durDirRuntime{cfg: du.cfg} - rt.users.Store(boomerutil.GoroutineID(), du) - rt.shutdown(context.Background()) - - calls := fakeCtrl.recordedCalls() - if len(calls) < 2 || calls[len(calls)-2] != "SuspendActor" || calls[len(calls)-1] != "DeleteActor" { - t.Errorf("recordedCalls must end with [SuspendActor, DeleteActor], got %v", calls) - } - reqs := fakeCtrl.recordedDeleteRequests() - if len(reqs) == 0 || !reqs[0].GetAnyState() { - t.Errorf("DeleteActor must set AnyState=true, got %v", reqs) + if got, want := suspends[len(suspends)-1].GetOptions().GetMode(), ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS; got != want { + t.Errorf("suspend mode = %v, want %v", got, want) } } -func TestDurDirShutdownPausesBeforeDelete(t *testing.T) { - fakeCtrl := &fakeControlClient{} - cfg := &userclass.Config{ - APIStub: fakeCtrl, - Dyn: dynconfig.NewHolder(dynconfig.Config{ - LifecycleMode: dynconfig.LifecycleModePause, - }), +func TestDurDirShutdownHibernatesBeforeDelete(t *testing.T) { + tests := []struct { + name string + lifecycleMode string + wantCalls []string + wantMode ateapipb.DurabilityMode + }{ + { + name: "suspend lifecycle mode", + wantCalls: []string{"SuspendActor", "DeleteActor"}, + wantMode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS, + }, + { + name: "pause lifecycle mode", + lifecycleMode: dynconfig.LifecycleModePause, + wantCalls: []string{"SuspendActor(async)", "DeleteActor"}, + wantMode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS, + }, } - du := newTestDurDirUser(t, &fake.Server{}, cfg) + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fakeCtrl := &fakeControlClient{} + cfg := &userclass.Config{ + APIStub: fakeCtrl, + Dyn: dynconfig.Static(durDirKnobs{ + Lifecycle: dynconfig.Lifecycle{LifecycleMode: tc.lifecycleMode}, + }), + } + du := newTestDurDirUser(t, &fake.Server{}, cfg) - rt := &durDirRuntime{cfg: du.cfg} - rt.users.Store(boomerutil.GoroutineID(), du) - rt.shutdown(context.Background()) + rt := &durDirRuntime{cfg: du.cfg} + rt.users.Store(boomerutil.GoroutineID(), du) + rt.shutdown(context.Background()) - calls := fakeCtrl.recordedCalls() - if len(calls) < 2 || calls[len(calls)-2] != "PauseActor" || calls[len(calls)-1] != "DeleteActor" { - t.Errorf("recordedCalls must end with [PauseActor, DeleteActor], got %v", calls) - } - reqs := fakeCtrl.recordedDeleteRequests() - if len(reqs) == 0 || !reqs[0].GetAnyState() { - t.Errorf("DeleteActor must set AnyState=true, got %v", reqs) + if got := fakeCtrl.recordedCalls(); !reflect.DeepEqual(got, tc.wantCalls) { + t.Errorf("recordedCalls = %v, want %v", got, tc.wantCalls) + } + suspends := fakeCtrl.recordedSuspendRequests() + if len(suspends) != 1 { + t.Fatalf("got %d SuspendActor requests, want 1", len(suspends)) + } + if got := suspends[0].GetOptions().GetMode(); got != tc.wantMode { + t.Errorf("suspend mode = %v, want %v", got, tc.wantMode) + } + reqs := fakeCtrl.recordedDeleteRequests() + if len(reqs) == 0 || !reqs[0].GetAnyState() { + t.Errorf("DeleteActor must set AnyState=true, got %v", reqs) + } + }) } } diff --git a/internal/benchmarking/boomer/glutton/fixture_test.go b/internal/benchmarking/boomer/glutton/fixture_test.go index 797aeae4d2..1d08dc1af8 100644 --- a/internal/benchmarking/boomer/glutton/fixture_test.go +++ b/internal/benchmarking/boomer/glutton/fixture_test.go @@ -32,6 +32,7 @@ type fakeControlClient struct { mu sync.Mutex calls []string deleteRequests []*ateapipb.DeleteActorRequest + suspendReqs []*ateapipb.SuspendActorRequest // resumeErrs is returned by successive ResumeActor calls, in order, until // it is drained; every call after that succeeds. resumeErrs []error @@ -91,18 +92,21 @@ func (f *fakeControlClient) ResumeActor(ctx context.Context, in *ateapipb.Resume func (f *fakeControlClient) SuspendActor(ctx context.Context, in *ateapipb.SuspendActorRequest, opts ...grpc.CallOption) (*ateapipb.SuspendActorResponse, error) { f.mu.Lock() defer f.mu.Unlock() - f.calls = append(f.calls, "SuspendActor") + f.calls = append(f.calls, suspendCallName(in)) + f.suspendReqs = append(f.suspendReqs, in) if err := nextErr(&f.suspendErrs); err != nil { return nil, err } return &ateapipb.SuspendActorResponse{}, nil } -func (f *fakeControlClient) PauseActor(ctx context.Context, in *ateapipb.PauseActorRequest, opts ...grpc.CallOption) (*ateapipb.PauseActorResponse, error) { - f.mu.Lock() - defer f.mu.Unlock() - f.calls = append(f.calls, "PauseActor") - return &ateapipb.PauseActorResponse{}, nil +// suspendCallName is the name a SuspendActor call is recorded under: an +// async suspend is told apart from a sync one. +func suspendCallName(in *ateapipb.SuspendActorRequest) string { + if in.GetOptions().GetMode() == ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS { + return "SuspendActor(async)" + } + return "SuspendActor" } func (f *fakeControlClient) DeleteActor(ctx context.Context, in *ateapipb.DeleteActorRequest, opts ...grpc.CallOption) (*ateapipb.Actor, error) { @@ -119,6 +123,12 @@ func (f *fakeControlClient) recordedCalls() []string { return append([]string(nil), f.calls...) } +func (f *fakeControlClient) recordedSuspendRequests() []*ateapipb.SuspendActorRequest { + f.mu.Lock() + defer f.mu.Unlock() + return append([]*ateapipb.SuspendActorRequest(nil), f.suspendReqs...) +} + func (f *fakeControlClient) recordedDeleteRequests() []*ateapipb.DeleteActorRequest { f.mu.Lock() defer f.mu.Unlock() @@ -140,7 +150,7 @@ func newTestConfig(t *testing.T, srv *fake.Server, cfg *userclass.Config) *userc cfg.Tracer = otel.Tracer("test") } if cfg.Dyn == nil { - cfg.Dyn = dynconfig.NewHolder(dynconfig.Config{}) + cfg.Dyn = dynconfig.Static(gluttonKnobs{}) } cfg.HTTPClient = ts.Client() cfg.RouterURL = ts.URL diff --git a/internal/benchmarking/boomer/glutton/knobs.go b/internal/benchmarking/boomer/glutton/knobs.go new file mode 100644 index 0000000000..ed9373ede4 --- /dev/null +++ b/internal/benchmarking/boomer/glutton/knobs.go @@ -0,0 +1,150 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package glutton + +import ( + "fmt" + "math" + "time" + + "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" +) + +// The three user classes in this package each read their own slice of the +// runtime config. The structs below name the keys; the sections carry the +// defaults a missing key falls back to and the rules a fetched value must +// pass before the worker takes it on. + +// gluttonKnobs is the GluttonUser slice: the wait and live windows, the +// lifecycle modes, the working-set sizes, the CPU load, and the ping cap. +type gluttonKnobs struct { + dynconfig.WaitTime + dynconfig.Lifecycle + // MinLive and MaxLive bound how long an actor stays resumed between its + // first ping and its suspend; a zero window suspends right after the ping. + MinLive dynconfig.Seconds `json:"min_live_time"` + MaxLive dynconfig.Seconds `json:"max_live_time"` + // MemTarget is the resident RAM filled via WriteRAM, suffixed ("2Gi"); + // MemChurn the RAM re-randomized in place each cycle; MemRead the RAM + // walked after each resume, or "all". Empty disables each. Glutton + // parses the sizes, so a bad one fails loudly as a GluttonFillRAM, + // GluttonChurnRAM, or GluttonReadRAM error rather than here. + MemTarget string `json:"mem_target"` + MemChurn string `json:"mem_churn"` + MemRead string `json:"mem_read"` + // CPUCores goroutines each burn CPUDutyCycle of one core via UseCPU; 0 + // disables. + CPUCores int `json:"cpu_cores"` + CPUDutyCycle float64 `json:"cpu_duty_cycle"` + // MaxPingsPerWake caps the pings sent during one resume/suspend cycle; + // values below 1 read as 1. + MaxPingsPerWake int `json:"max_pings_per_wake"` +} + +var gluttonCodec = dynconfig.Typed[gluttonKnobs]{ + Defaults: gluttonKnobs{ + WaitTime: dynconfig.WaitTime{MaxWait: dynconfig.Seconds(500 * time.Millisecond)}, + MaxPingsPerWake: 1, + }, + Validate: func(k gluttonKnobs) error { + if err := k.WaitTime.Validate(); err != nil { + return err + } + if err := k.Lifecycle.Validate(); err != nil { + return err + } + if k.MinLive < 0 { + return fmt.Errorf("min_live_time cannot be negative: %v", k.MinLive.Duration()) + } + if k.MaxLive < 0 { + return fmt.Errorf("max_live_time cannot be negative: %v", k.MaxLive.Duration()) + } + if k.MaxLive < k.MinLive { + return fmt.Errorf("max_live_time (%v) cannot be less than min_live_time (%v)", k.MaxLive.Duration(), k.MinLive.Duration()) + } + if k.CPUCores < 0 { + return fmt.Errorf("cpu_cores cannot be negative: %d", k.CPUCores) + } + if k.CPUDutyCycle < 0 || k.CPUDutyCycle > 1 { + return fmt.Errorf("cpu_duty_cycle must be between 0.0 and 1.0, got: %f", k.CPUDutyCycle) + } + return nil + }, +} + +// DurDir read modes: whether the actor ships the file's bytes back or only +// its digest. +const ( + readModeData = "data" + readModeDigest = "digest" +) + +// durDirKnobs is the DurdirUser slice. +type durDirKnobs struct { + dynconfig.WaitTime + dynconfig.Lifecycle + // FileSize is the file written and re-read each cycle, in bytes; 0 + // falls back to defaultFileSize. + FileSize int64 `json:"durdir_file_size_bytes"` + // ReadMode is readModeData or readModeDigest; empty reads as data. + ReadMode string `json:"durdir_read_mode"` + // Template is the ActorTemplate name; empty falls back to + // defaultDurTemplate. + Template string `json:"durdir_template"` +} + +var durDirCodec = dynconfig.Typed[durDirKnobs]{ + Validate: func(k durDirKnobs) error { + if err := k.WaitTime.Validate(); err != nil { + return err + } + if err := k.Lifecycle.Validate(); err != nil { + return err + } + if k.FileSize < 0 { + return fmt.Errorf("durdir_file_size_bytes cannot be negative: %d", k.FileSize) + } + if k.FileSize > math.MaxInt32 { + return fmt.Errorf("durdir_file_size_bytes cannot exceed %d (2 GiB), got: %d", math.MaxInt32, k.FileSize) + } + if k.ReadMode != "" && k.ReadMode != readModeData && k.ReadMode != readModeDigest { + return fmt.Errorf("invalid durdir_read_mode %q: must be %q or %q", k.ReadMode, readModeData, readModeDigest) + } + return nil + }, +} + +// spawnKnobs is the SpawnUser slice. Each zero keeps the matching +// boomer-worker flag. +type spawnKnobs struct { + TotalActors int `json:"total_actors"` + SpawnConcurrency int `json:"spawn_concurrency"` + ActorDeadline dynconfig.Seconds `json:"actor_deadline"` +} + +var spawnCodec = dynconfig.Typed[spawnKnobs]{ + Validate: func(k spawnKnobs) error { + if k.TotalActors < 0 { + return fmt.Errorf("total_actors cannot be negative: %d", k.TotalActors) + } + if k.SpawnConcurrency < 0 { + return fmt.Errorf("spawn_concurrency cannot be negative: %d", k.SpawnConcurrency) + } + if k.ActorDeadline < 0 { + return fmt.Errorf("actor_deadline cannot be negative: %v", k.ActorDeadline.Duration()) + } + return nil + }, +} diff --git a/internal/benchmarking/boomer/glutton/knobs_test.go b/internal/benchmarking/boomer/glutton/knobs_test.go new file mode 100644 index 0000000000..f5f28071a8 --- /dev/null +++ b/internal/benchmarking/boomer/glutton/knobs_test.go @@ -0,0 +1,140 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package glutton + +import ( + "reflect" + "testing" + "time" + + "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" +) + +// Each class codec starts a holder at its defaults, takes a payload naming +// every key the class reads, and refuses the values the class cannot run +// on. +func TestClassCodecs(t *testing.T) { + window := dynconfig.WaitTime{MinWait: dynconfig.Seconds(100 * time.Millisecond), MaxWait: dynconfig.Seconds(500 * time.Millisecond)} + for _, tc := range []struct { + name string + codec dynconfig.Codec + initial any + valid string + want any + invalid map[string]string + }{ + { + name: "glutton", + codec: gluttonCodec, + initial: gluttonKnobs{WaitTime: dynconfig.WaitTime{MaxWait: dynconfig.Seconds(500 * time.Millisecond)}, MaxPingsPerWake: 1}, + valid: `{ + "min_wait_time": 0.1, "max_wait_time": 0.5, + "min_live_time": 9, "max_live_time": 14, + "resume_mode": "explicit", "lifecycle_mode": "pause", + "mem_target": "2Gi", "mem_churn": "512Mi", "mem_read": "all", + "cpu_cores": 2, "cpu_duty_cycle": 0.1, + "max_pings_per_wake": 3 + }`, + want: gluttonKnobs{ + WaitTime: window, + Lifecycle: dynconfig.Lifecycle{ResumeMode: dynconfig.ResumeModeExplicit, LifecycleMode: dynconfig.LifecycleModePause}, + MinLive: dynconfig.Seconds(9 * time.Second), + MaxLive: dynconfig.Seconds(14 * time.Second), + MemTarget: "2Gi", + MemChurn: "512Mi", + MemRead: "all", + CPUCores: 2, + CPUDutyCycle: 0.1, + MaxPingsPerWake: 3, + }, + invalid: map[string]string{ + "negative min wait": `{"min_wait_time": -1}`, + "negative max wait": `{"max_wait_time": -1}`, + "max wait below min wait": `{"min_wait_time": 2, "max_wait_time": 1}`, + "negative min live": `{"min_live_time": -1}`, + "negative max live": `{"max_live_time": -1}`, + "max live below min live": `{"min_live_time": 14, "max_live_time": 9}`, + "invalid resume mode": `{"resume_mode": "invalid_mode"}`, + "invalid lifecycle mode": `{"lifecycle_mode": "invalid_lifecycle"}`, + "negative cpu cores": `{"cpu_cores": -1}`, + "negative cpu duty cycle": `{"cpu_duty_cycle": -0.1}`, + "cpu duty cycle above 1": `{"cpu_duty_cycle": 1.5}`, + "negative trace (common)": `{"trace_probability": -0.1}`, + "trace above 1 (common)": `{"trace_probability": 1.5}`, + "wrong type for a count": `{"cpu_cores": "two"}`, + }, + }, + { + name: "durdir", + codec: durDirCodec, + initial: durDirKnobs{}, + valid: `{ + "min_wait_time": 0.1, "max_wait_time": 0.5, + "resume_mode": "implicit", "lifecycle_mode": "suspend", + "durdir_file_size_bytes": 1048576, + "durdir_read_mode": "digest", + "durdir_template": "glutton-durdir-data" + }`, + want: durDirKnobs{ + WaitTime: window, + Lifecycle: dynconfig.Lifecycle{ResumeMode: dynconfig.ResumeModeImplicit, LifecycleMode: dynconfig.LifecycleModeSuspend}, + FileSize: 1048576, + ReadMode: readModeDigest, + Template: "glutton-durdir-data", + }, + invalid: map[string]string{ + "negative file size": `{"durdir_file_size_bytes": -100}`, + "file size over 2 GiB": `{"durdir_file_size_bytes": 2147483648}`, + "invalid read mode": `{"durdir_read_mode": "invalid_read"}`, + "invalid lifecycle mode": `{"lifecycle_mode": "invalid_lifecycle"}`, + "max wait below min wait": `{"min_wait_time": 2, "max_wait_time": 1}`, + }, + }, + { + name: "spawn", + codec: spawnCodec, + initial: spawnKnobs{}, + valid: `{"total_actors": 50, "spawn_concurrency": 5, "actor_deadline": 60}`, + want: spawnKnobs{TotalActors: 50, SpawnConcurrency: 5, ActorDeadline: dynconfig.Seconds(60 * time.Second)}, + invalid: map[string]string{ + "negative total actors": `{"total_actors": -1}`, + "negative spawn concurrency": `{"spawn_concurrency": -1}`, + "negative actor deadline": `{"actor_deadline": -1}`, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + h := dynconfig.NewHolder(tc.codec) + if got := h.Load(); !reflect.DeepEqual(got, tc.initial) { + t.Fatalf("fresh holder = %+v, want %+v", got, tc.initial) + } + if _, _, err := h.Apply([]byte(tc.valid)); err != nil { + t.Fatalf("Apply(valid): %v", err) + } + if got := h.Load(); !reflect.DeepEqual(got, tc.want) { + t.Errorf("decoded =\n %+v, want\n %+v", got, tc.want) + } + for name, payload := range tc.invalid { + t.Run(name, func(t *testing.T) { + if _, _, err := h.Apply([]byte(payload)); err == nil { + t.Errorf("Apply(%s) took the payload, want a refusal", payload) + } + if got := h.Load(); !reflect.DeepEqual(got, tc.want) { + t.Errorf("a refused payload changed the config to %+v", got) + } + }) + } + }) + } +} diff --git a/internal/benchmarking/boomer/glutton/lifecycle.go b/internal/benchmarking/boomer/glutton/lifecycle.go index e1e13e08b6..e50af663bc 100644 --- a/internal/benchmarking/boomer/glutton/lifecycle.go +++ b/internal/benchmarking/boomer/glutton/lifecycle.go @@ -76,6 +76,7 @@ func init() { Name: "glutton", LocustFile: "glutton.py", UserClass: userClass, + Config: gluttonCodec, Init: initPing, }) } @@ -141,7 +142,7 @@ func (r *taskRuntime) iterate() { // Going straight back to resume on a stranded actor would turn every // retryable hibernate failure into a FailedPrecondition and cost the - // actor anyway. SuspendActor and PauseActor are re-entrant, so finish + // actor anyway. SuspendActor is re-entrant in either mode, so finish // the hibernate and pick the cycle up next iteration. if actor.hibernatePending { if err := actor.hibernate(ctx); err != nil && actor.noteFailure(err) { @@ -150,11 +151,13 @@ func (r *taskRuntime) iterate() { return } - if err := actor.resume(ctx); err != nil { - if actor.noteFailure(err) { - user.replaceActor(ctx, actor) + if !actor.actorRunning || dynconfig.Get[gluttonKnobs](r.cfg.Dyn).LifecycleMode != dynconfig.LifecycleModeNone { + if err := actor.resume(ctx); err != nil { + if actor.noteFailure(err) { + user.replaceActor(ctx, actor) + } + return } - return } // Fill before the first suspend so every snapshot from cycle one on // carries the full working set; glutton keeps the allocations across @@ -181,7 +184,7 @@ func (r *taskRuntime) iterate() { // actor stays live for the full window. With the default zero window // the actor is hibernated right after the first ping. deadline := time.Now().Add(r.liveWait()) - maxPings := max(r.cfg.Dyn.Load().MaxPingsPerWake, 1) + maxPings := max(dynconfig.Get[gluttonKnobs](r.cfg.Dyn).MaxPingsPerWake, 1) actor.ping(ctx) for sent := 1; sent < maxPings; sent++ { gap := minPingGap + time.Duration(rand.Float64()*float64(maxPingGap-minPingGap)) @@ -249,7 +252,11 @@ func (r *taskRuntime) shutdown(ctx context.Context) { u := val.(*gluttonUser) for _, a := range u.actors { if a.actorRunning { - _ = a.hibernate(ctx) + if dynconfig.Get[gluttonKnobs](r.cfg.Dyn).LifecycleMode == dynconfig.LifecycleModePause { + _ = a.pause(ctx) + } else { + _ = a.suspend(ctx) + } } a.delete(ctx) } @@ -261,24 +268,15 @@ func (r *taskRuntime) shutdown(ctx context.Context) { // dynamicWait is the gap between suspending one actor and resuming the // VU's next one, drawn uniformly from [MinWait, MaxWait]. func (r *taskRuntime) dynamicWait() time.Duration { - cfg := r.cfg.Dyn.Load() - return uniformWait(cfg.MinWait, cfg.MaxWait) + return dynconfig.Get[gluttonKnobs](r.cfg.Dyn).WaitTime.Draw() } // liveWait is how long an actor stays resumed between its first ping and // its suspend, drawn uniformly from [MinLive, MaxLive]. The default zero // window suspends right after the ping. func (r *taskRuntime) liveWait() time.Duration { - cfg := r.cfg.Dyn.Load() - return uniformWait(cfg.MinLive, cfg.MaxLive) -} - -// uniformWait draws from [lo, hi]; an inverted or empty range yields lo. -func uniformWait(lo, hi time.Duration) time.Duration { - if hi <= lo { - return lo - } - return lo + time.Duration(rand.Float64()*float64(hi-lo)) + knobs := dynconfig.Get[gluttonKnobs](r.cfg.Dyn) + return dynconfig.Uniform(knobs.MinLive.Duration(), knobs.MaxLive.Duration()) } // gluttonUser is one VU (boomer goroutine). It owns --actors-per-user actors @@ -456,19 +454,25 @@ func (u *gluttonActor) resume(ctx context.Context) error { } // hibernate takes the actor off its worker by whichever operation the -// lifecycle mode selects: PauseActor keeps the snapshot on the node, while -// SuspendActor writes it to durable storage. +// lifecycle mode selects: an async SuspendActor keeps the snapshot on the +// node, a sync one writes it to durable storage, and LifecycleModeNone +// leaves the actor running on its worker. func (u *gluttonActor) hibernate(ctx context.Context) error { - if u.cfg.Dyn.Load().LifecycleMode == dynconfig.LifecycleModePause { + switch dynconfig.Get[gluttonKnobs](u.cfg.Dyn).LifecycleMode { + case dynconfig.LifecycleModeNone: + return nil + case dynconfig.LifecycleModePause: return u.pause(ctx) + default: + return u.suspend(ctx) } - return u.suspend(ctx) } func (u *gluttonActor) pause(ctx context.Context) error { - err := u.tracedCall(ctx, "PauseActor", func(callCtx context.Context, tr *metadata.MD) error { - _, err := u.cfg.APIStub.PauseActor(callCtx, &ateapipb.PauseActorRequest{ - Actor: u.ref(), + err := u.tracedCall(ctx, "SuspendActorAsync", func(callCtx context.Context, tr *metadata.MD) error { + _, err := u.cfg.APIStub.SuspendActor(callCtx, &ateapipb.SuspendActorRequest{ + Actor: u.ref(), + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, }, grpc.Trailer(tr)) return err }) @@ -483,7 +487,8 @@ func (u *gluttonActor) pause(ctx context.Context) error { func (u *gluttonActor) suspend(ctx context.Context) error { err := u.tracedCall(ctx, "SuspendActor", func(callCtx context.Context, tr *metadata.MD) error { _, err := u.cfg.APIStub.SuspendActor(callCtx, &ateapipb.SuspendActorRequest{ - Actor: u.ref(), + Actor: u.ref(), + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, }, grpc.Trailer(tr)) return err }) @@ -600,7 +605,7 @@ func (u *gluttonActor) ensureRAMFilled(ctx context.Context) { if u.ramFilled { return } - target := u.cfg.Dyn.Load().MemTarget + target := dynconfig.Get[gluttonKnobs](u.cfg.Dyn).MemTarget if target == "" { u.ramFilled = true return @@ -631,8 +636,8 @@ func (u *gluttonActor) ensureCPULoad(ctx context.Context) { if u.cpuLoaded { return } - dyn := u.cfg.Dyn.Load() - if dyn.CPUCores == 0 { + knobs := dynconfig.Get[gluttonKnobs](u.cfg.Dyn) + if knobs.CPUCores == 0 { u.cpuLoaded = true return } @@ -642,8 +647,8 @@ func (u *gluttonActor) ensureCPULoad(ctx context.Context) { start := time.Now() err := u.postProto(ctx, useCPUPath, &gluttonpb.UseCPURequest{ - NumCores: int32(dyn.CPUCores), - DutyCycle: dyn.CPUDutyCycle, + NumCores: int32(knobs.CPUCores), + DutyCycle: knobs.CPUDutyCycle, }, &gluttonpb.UseCPUResponse{}) clientLatency := time.Since(start) boomerutil.LogSampledTrace(span, "GluttonUseCPU", clientLatency, boomerutil.SourceClient, err) @@ -665,7 +670,7 @@ func (u *gluttonActor) ensureCPULoad(ctx context.Context) { // iteration, only after the fill has succeeded, and reports as its own // GluttonChurnRAM stats row. func (u *gluttonActor) churnRAM(ctx context.Context) { - churn := u.cfg.Dyn.Load().MemChurn + churn := dynconfig.Get[gluttonKnobs](u.cfg.Dyn).MemChurn if churn == "" || !u.ramFilled { return } @@ -691,7 +696,7 @@ func (u *gluttonActor) churnRAM(ctx context.Context) { // memory; on an eagerly-restored actor it degenerates to a fast in-memory // scan, so the two restore modes are directly comparable. func (u *gluttonActor) readRAM(ctx context.Context) { - read := u.cfg.Dyn.Load().MemRead + read := dynconfig.Get[gluttonKnobs](u.cfg.Dyn).MemRead if read == "" || !u.ramFilled { return } diff --git a/internal/benchmarking/boomer/glutton/lifecycle_test.go b/internal/benchmarking/boomer/glutton/lifecycle_test.go index 8673f5809f..1aa434c9d3 100644 --- a/internal/benchmarking/boomer/glutton/lifecycle_test.go +++ b/internal/benchmarking/boomer/glutton/lifecycle_test.go @@ -23,6 +23,7 @@ import ( "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" "go.ate.dev/substrate/internal/benchmarking/boomer/userclass" "go.ate.dev/substrate/internal/benchmarking/glutton/fake" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -33,8 +34,8 @@ func TestGluttonIterate_SuspendMode(t *testing.T) { cfg := newTestConfig(t, srv, &userclass.Config{ APIStub: fakeCtrl, Atespace: "bench-test", - Dyn: dynconfig.NewHolder(dynconfig.Config{ - LifecycleMode: dynconfig.LifecycleModeSuspend, + Dyn: dynconfig.Static(gluttonKnobs{ + Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModeSuspend}, }), }) @@ -45,8 +46,13 @@ func TestGluttonIterate_SuspendMode(t *testing.T) { if !slices.Contains(calls, "SuspendActor") { t.Errorf("expected SuspendActor in calls, got: %v", calls) } - if slices.Contains(calls, "PauseActor") { - t.Errorf("did not expect PauseActor in suspend mode calls, got: %v", calls) + if slices.Contains(calls, "SuspendActor(async)") { + t.Errorf("did not expect SuspendActor(async) in suspend mode calls, got: %v", calls) + } + for _, req := range fakeCtrl.recordedSuspendRequests() { + if got := req.GetOptions().GetMode(); got != ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS { + t.Errorf("suspend mode = %v, want %v", got, ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS) + } } } @@ -56,8 +62,8 @@ func TestGluttonIterate_PauseMode(t *testing.T) { cfg := newTestConfig(t, srv, &userclass.Config{ APIStub: fakeCtrl, Atespace: "bench-test", - Dyn: dynconfig.NewHolder(dynconfig.Config{ - LifecycleMode: dynconfig.LifecycleModePause, + Dyn: dynconfig.Static(gluttonKnobs{ + Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModePause}, }), }) @@ -65,22 +71,52 @@ func TestGluttonIterate_PauseMode(t *testing.T) { rt.iterate() calls := fakeCtrl.recordedCalls() - if !slices.Contains(calls, "PauseActor") { - t.Errorf("expected PauseActor in calls, got: %v", calls) + if !slices.Contains(calls, "SuspendActor(async)") { + t.Errorf("expected SuspendActor(async) in calls, got: %v", calls) } if slices.Contains(calls, "SuspendActor") { t.Errorf("did not expect SuspendActor in pause mode calls, got: %v", calls) } } +func TestGluttonIterate_NoneMode(t *testing.T) { + srv := &fake.Server{} + fakeCtrl := &fakeControlClient{} + cfg := newTestConfig(t, srv, &userclass.Config{ + APIStub: fakeCtrl, + Atespace: "bench-test", + Dyn: dynconfig.Static(gluttonKnobs{ + Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModeNone}, + }), + }) + + rt := &taskRuntime{cfg: cfg} + rt.iterate() + rt.iterate() + + calls := fakeCtrl.recordedCalls() + if slices.Contains(calls, "SuspendActor") || slices.Contains(calls, "SuspendActor(async)") { + t.Errorf("did not expect a sync or async SuspendActor during iterate in none mode, got: %v", calls) + } + if got := countCalls(calls, "ResumeActor"); got != 1 { + t.Errorf("ResumeActor count across 2 iterations = %d, want 1 (first activation only); calls = %v", got, calls) + } + + rt.shutdown(context.Background()) + afterShutdown := fakeCtrl.recordedCalls() + if len(afterShutdown) < 2 || afterShutdown[len(afterShutdown)-2] != "SuspendActor" || afterShutdown[len(afterShutdown)-1] != "DeleteActor" { + t.Errorf("shutdown in none mode must end with [SuspendActor, DeleteActor], got %v", afterShutdown) + } +} + func TestGluttonShutdown_PauseModeRunningActor(t *testing.T) { srv := &fake.Server{} fakeCtrl := &fakeControlClient{} cfg := newTestConfig(t, srv, &userclass.Config{ APIStub: fakeCtrl, Atespace: "bench-test", - Dyn: dynconfig.NewHolder(dynconfig.Config{ - LifecycleMode: dynconfig.LifecycleModePause, + Dyn: dynconfig.Static(gluttonKnobs{ + Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModePause}, }), }) @@ -95,8 +131,8 @@ func TestGluttonShutdown_PauseModeRunningActor(t *testing.T) { rt.shutdown(context.Background()) calls := fakeCtrl.recordedCalls() - if len(calls) < 2 || calls[len(calls)-2] != "PauseActor" || calls[len(calls)-1] != "DeleteActor" { - t.Errorf("recordedCalls must end with [PauseActor, DeleteActor], got %v", calls) + if len(calls) < 2 || calls[len(calls)-2] != "SuspendActor(async)" || calls[len(calls)-1] != "DeleteActor" { + t.Errorf("recordedCalls must end with [SuspendActor(async), DeleteActor], got %v", calls) } reqs := fakeCtrl.recordedDeleteRequests() if len(reqs) == 0 || !reqs[0].GetAnyState() { @@ -110,8 +146,8 @@ func TestGluttonShutdown_DeleteSetsAnyState(t *testing.T) { cfg := newTestConfig(t, srv, &userclass.Config{ APIStub: fakeCtrl, Atespace: "bench-test", - Dyn: dynconfig.NewHolder(dynconfig.Config{ - LifecycleMode: dynconfig.LifecycleModeSuspend, + Dyn: dynconfig.Static(gluttonKnobs{ + Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModeSuspend}, }), }) @@ -143,8 +179,8 @@ func newReplacementRuntime(t *testing.T, resumeErrs ...error) (*taskRuntime, *fa cfg := newTestConfig(t, &fake.Server{}, &userclass.Config{ APIStub: fakeCtrl, Atespace: "bench-test", - Dyn: dynconfig.NewHolder(dynconfig.Config{ - LifecycleMode: dynconfig.LifecycleModeSuspend, + Dyn: dynconfig.Static(gluttonKnobs{ + Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModeSuspend}, }), }) return &taskRuntime{cfg: cfg}, fakeCtrl @@ -231,8 +267,8 @@ func TestGluttonIterate_RetriesStrandedHibernate(t *testing.T) { cfg := newTestConfig(t, &fake.Server{}, &userclass.Config{ APIStub: fakeCtrl, Atespace: "bench-test", - Dyn: dynconfig.NewHolder(dynconfig.Config{ - LifecycleMode: dynconfig.LifecycleModeSuspend, + Dyn: dynconfig.Static(gluttonKnobs{ + Lifecycle: dynconfig.Lifecycle{LifecycleMode: dynconfig.LifecycleModeSuspend}, }), }) rt := &taskRuntime{cfg: cfg} diff --git a/internal/benchmarking/boomer/glutton/memfill_test.go b/internal/benchmarking/boomer/glutton/memfill_test.go index ac3499c028..4e671f4099 100644 --- a/internal/benchmarking/boomer/glutton/memfill_test.go +++ b/internal/benchmarking/boomer/glutton/memfill_test.go @@ -24,9 +24,9 @@ import ( gluttonpb "go.ate.dev/substrate/internal/proto/glutton" ) -func newTestGluttonActor(t *testing.T, srv *fake.Server, dyn dynconfig.Config) *gluttonActor { +func newTestGluttonActor(t *testing.T, srv *fake.Server, dyn gluttonKnobs) *gluttonActor { t.Helper() - cfg := newTestConfig(t, srv, &userclass.Config{Dyn: dynconfig.NewHolder(dyn)}) + cfg := newTestConfig(t, srv, &userclass.Config{Dyn: dynconfig.Static(dyn)}) return &gluttonActor{ cfg: cfg, actorName: "memactor", @@ -35,7 +35,7 @@ func newTestGluttonActor(t *testing.T, srv *fake.Server, dyn dynconfig.Config) * func TestEnsureRAMFilledRequestsTarget(t *testing.T) { srv := &fake.Server{} - u := newTestGluttonActor(t, srv, dynconfig.Config{MemTarget: "2Gi"}) + u := newTestGluttonActor(t, srv, gluttonKnobs{MemTarget: "2Gi"}) u.ensureRAMFilled(context.Background()) @@ -56,7 +56,7 @@ func TestEnsureRAMFilledRequestsTarget(t *testing.T) { func TestEnsureRAMFilledDisabledByDefault(t *testing.T) { srv := &fake.Server{} - u := newTestGluttonActor(t, srv, dynconfig.Config{}) + u := newTestGluttonActor(t, srv, gluttonKnobs{}) u.ensureRAMFilled(context.Background()) @@ -70,7 +70,7 @@ func TestEnsureRAMFilledDisabledByDefault(t *testing.T) { func TestChurnRAMOverwritesEachCycle(t *testing.T) { srv := &fake.Server{} - u := newTestGluttonActor(t, srv, dynconfig.Config{MemTarget: "1Gi", MemChurn: "64Mi"}) + u := newTestGluttonActor(t, srv, gluttonKnobs{MemTarget: "1Gi", MemChurn: "64Mi"}) ctx := context.Background() // Churn before fill is a no-op: there is nothing to overwrite yet. @@ -104,7 +104,7 @@ func TestChurnRAMOverwritesEachCycle(t *testing.T) { func TestChurnRAMDisabledByDefault(t *testing.T) { srv := &fake.Server{} - u := newTestGluttonActor(t, srv, dynconfig.Config{MemTarget: "1Gi"}) + u := newTestGluttonActor(t, srv, gluttonKnobs{MemTarget: "1Gi"}) ctx := context.Background() u.ensureRAMFilled(ctx) @@ -116,7 +116,7 @@ func TestChurnRAMDisabledByDefault(t *testing.T) { func TestReadRAMWalksAfterFill(t *testing.T) { srv := &fake.Server{} - u := newTestGluttonActor(t, srv, dynconfig.Config{MemTarget: "1Gi", MemRead: "all"}) + u := newTestGluttonActor(t, srv, gluttonKnobs{MemTarget: "1Gi", MemRead: "all"}) ctx := context.Background() // Read before fill is a no-op: there is nothing to walk yet. @@ -144,7 +144,7 @@ func TestReadRAMWalksAfterFill(t *testing.T) { func TestReadRAMPassesSizeVerbatim(t *testing.T) { srv := &fake.Server{} - u := newTestGluttonActor(t, srv, dynconfig.Config{MemTarget: "1Gi", MemRead: "512Mi"}) + u := newTestGluttonActor(t, srv, gluttonKnobs{MemTarget: "1Gi", MemRead: "512Mi"}) ctx := context.Background() u.ensureRAMFilled(ctx) @@ -158,7 +158,7 @@ func TestReadRAMPassesSizeVerbatim(t *testing.T) { func TestReadRAMDisabledByDefault(t *testing.T) { srv := &fake.Server{} - u := newTestGluttonActor(t, srv, dynconfig.Config{MemTarget: "1Gi"}) + u := newTestGluttonActor(t, srv, gluttonKnobs{MemTarget: "1Gi"}) ctx := context.Background() u.ensureRAMFilled(ctx) @@ -170,7 +170,7 @@ func TestReadRAMDisabledByDefault(t *testing.T) { func TestEnsureRAMFilledRetriesAfterFailure(t *testing.T) { srv := &fake.Server{Status: 503} - u := newTestGluttonActor(t, srv, dynconfig.Config{MemTarget: "1Mi"}) + u := newTestGluttonActor(t, srv, gluttonKnobs{MemTarget: "1Mi"}) u.ensureRAMFilled(context.Background()) if u.ramFilled { diff --git a/internal/benchmarking/boomer/glutton/spawn.go b/internal/benchmarking/boomer/glutton/spawn.go index d14e0a74f8..28175db539 100644 --- a/internal/benchmarking/boomer/glutton/spawn.go +++ b/internal/benchmarking/boomer/glutton/spawn.go @@ -31,6 +31,7 @@ import ( "go.ate.dev/substrate/internal/ateinterceptors" "go.ate.dev/substrate/internal/atenet" "go.ate.dev/substrate/internal/benchmarking/boomer/boomerutil" + "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" bmetrics "go.ate.dev/substrate/internal/benchmarking/boomer/metrics" "go.ate.dev/substrate/internal/benchmarking/boomer/userclass" gluttonpb "go.ate.dev/substrate/internal/proto/glutton" @@ -75,6 +76,7 @@ func init() { Name: "spawn", LocustFile: "spawn.py", UserClass: spawnUserClass, + Config: spawnCodec, Init: initSpawn, }) } @@ -161,17 +163,15 @@ func (r *spawnRuntime) runBatch(ctx context.Context) { spawnConcurrency := r.cfg.SpawnConcurrency deadline := r.cfg.ActorDeadline // Values set in the web UI form (dynconfig) override the flags. - if r.cfg.Dyn != nil { - dyn := r.cfg.Dyn.Load() - if dyn.TotalActors > 0 { - totalActors = dyn.TotalActors - } - if dyn.SpawnConcurrency > 0 { - spawnConcurrency = dyn.SpawnConcurrency - } - if dyn.ActorDeadline > 0 { - deadline = dyn.ActorDeadline - } + knobs := dynconfig.Get[spawnKnobs](r.cfg.Dyn) + if knobs.TotalActors > 0 { + totalActors = knobs.TotalActors + } + if knobs.SpawnConcurrency > 0 { + spawnConcurrency = knobs.SpawnConcurrency + } + if knobs.ActorDeadline > 0 { + deadline = knobs.ActorDeadline.Duration() } spawnConcurrency = min(spawnConcurrency, totalActors) if spawnConcurrency < 1 { diff --git a/internal/benchmarking/boomer/glutton/spawn_test.go b/internal/benchmarking/boomer/glutton/spawn_test.go index 9c33f73bdd..9e52a87bce 100644 --- a/internal/benchmarking/boomer/glutton/spawn_test.go +++ b/internal/benchmarking/boomer/glutton/spawn_test.go @@ -646,10 +646,10 @@ func TestSpawnRunBatch_DynConfigOverride(t *testing.T) { t.Run("overrides when set", func(t *testing.T) { fakeAPI := &fakeControlClient{} - dyn := dynconfig.NewHolder(dynconfig.Config{ + dyn := dynconfig.Static(spawnKnobs{ TotalActors: 3, SpawnConcurrency: 2, - ActorDeadline: 5 * time.Second, + ActorDeadline: dynconfig.Seconds(5 * time.Second), }) cfg := &userclass.Config{ APIStub: fakeAPI, @@ -684,7 +684,7 @@ func TestSpawnRunBatch_DynConfigOverride(t *testing.T) { t.Run("falls back to cfg when dynconfig is zero", func(t *testing.T) { fakeAPI := &fakeControlClient{} - dyn := dynconfig.NewHolder(dynconfig.Config{ + dyn := dynconfig.Static(spawnKnobs{ TotalActors: 0, // 0 = unset SpawnConcurrency: 0, ActorDeadline: 0, diff --git a/internal/benchmarking/boomer/glutton/wait_test.go b/internal/benchmarking/boomer/glutton/wait_test.go index 56aef7b17d..a65d2b2640 100644 --- a/internal/benchmarking/boomer/glutton/wait_test.go +++ b/internal/benchmarking/boomer/glutton/wait_test.go @@ -25,22 +25,22 @@ import ( func TestUniformWaitStaysInRange(t *testing.T) { lo, hi := 200*time.Millisecond, time.Second for i := 0; i < 1000; i++ { - got := uniformWait(lo, hi) + got := dynconfig.Uniform(lo, hi) if got < lo || got > hi { - t.Fatalf("uniformWait(%v, %v) = %v, outside range", lo, hi, got) + t.Fatalf("dynconfig.Uniform(%v, %v) = %v, outside range", lo, hi, got) } } } func TestUniformWaitDegenerateRanges(t *testing.T) { - if got := uniformWait(0, 0); got != 0 { - t.Errorf("uniformWait(0, 0) = %v, want 0", got) + if got := dynconfig.Uniform(0, 0); got != 0 { + t.Errorf("dynconfig.Uniform(0, 0) = %v, want 0", got) } - if got := uniformWait(3*time.Second, 3*time.Second); got != 3*time.Second { + if got := dynconfig.Uniform(3*time.Second, 3*time.Second); got != 3*time.Second { t.Errorf("equal bounds: got %v, want 3s", got) } // An inverted range yields the lower bound rather than a negative wait. - if got := uniformWait(5*time.Second, time.Second); got != 5*time.Second { + if got := dynconfig.Uniform(5*time.Second, time.Second); got != 5*time.Second { t.Errorf("inverted bounds: got %v, want 5s", got) } } @@ -48,9 +48,8 @@ func TestUniformWaitDegenerateRanges(t *testing.T) { // The wait window and the live window read different config fields, so a // run that sets only one of them must not leak into the other. func TestWaitAndLiveWindowsAreIndependent(t *testing.T) { - rt := &taskRuntime{cfg: &userclass.Config{Dyn: dynconfig.NewHolder(dynconfig.Config{ - MinWait: 200 * time.Millisecond, - MaxWait: time.Second, + rt := &taskRuntime{cfg: &userclass.Config{Dyn: dynconfig.Static(gluttonKnobs{ + WaitTime: dynconfig.WaitTime{MinWait: dynconfig.Seconds(200 * time.Millisecond), MaxWait: dynconfig.Seconds(time.Second)}, })}} for i := 0; i < 100; i++ { if got := rt.liveWait(); got != 0 { @@ -61,7 +60,7 @@ func TestWaitAndLiveWindowsAreIndependent(t *testing.T) { } } - rt.cfg.Dyn.Store(dynconfig.Config{MinLive: 9 * time.Second, MaxLive: 14 * time.Second}) + rt.cfg.Dyn = dynconfig.Static(gluttonKnobs{MinLive: dynconfig.Seconds(9 * time.Second), MaxLive: dynconfig.Seconds(14 * time.Second)}) for i := 0; i < 100; i++ { if got := rt.dynamicWait(); got != 0 { t.Fatalf("dynamicWait with zero wait window = %v, want 0", got) diff --git a/internal/benchmarking/boomer/resumepaused/knobs_test.go b/internal/benchmarking/boomer/resumepaused/knobs_test.go new file mode 100644 index 0000000000..9d63c47860 --- /dev/null +++ b/internal/benchmarking/boomer/resumepaused/knobs_test.go @@ -0,0 +1,49 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resumepaused + +import ( + "testing" + + "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" +) + +// The codec takes a target rate, reads an absent one as unpaced, and +// refuses a negative one in favor of the config before it. +func TestKnobsCodec(t *testing.T) { + h := dynconfig.NewHolder(knobsCodec) + if got := dynconfig.Get[knobs](h); got != (knobs{}) { + t.Fatalf("fresh holder = %+v, want unpaced", got) + } + if _, _, err := h.Apply([]byte(`{"target_rps": 2500.5, "someone_elses_key": 1}`)); err != nil { + t.Fatalf("Apply(valid): %v", err) + } + want := knobs{TargetRPS: 2500.5} + if got := dynconfig.Get[knobs](h); got != want { + t.Errorf("decoded = %+v, want %+v", got, want) + } + if _, _, err := h.Apply([]byte(`{"target_rps": -1}`)); err == nil { + t.Error("Apply(negative target_rps) took the payload, want a refusal") + } + if got := dynconfig.Get[knobs](h); got != want { + t.Errorf("a refused payload changed the config to %+v", got) + } + if _, _, err := h.Apply([]byte(`{"target_rps": null}`)); err != nil { + t.Fatalf("Apply(null): %v", err) + } + if got := dynconfig.Get[knobs](h); got != (knobs{}) { + t.Errorf("a null target_rps left %+v, want unpaced", got) + } +} diff --git a/internal/benchmarking/boomer/resumepaused/resumepaused.go b/internal/benchmarking/boomer/resumepaused/resumepaused.go index 585edef2ef..020080686b 100644 --- a/internal/benchmarking/boomer/resumepaused/resumepaused.go +++ b/internal/benchmarking/boomer/resumepaused/resumepaused.go @@ -37,6 +37,7 @@ import ( "github.com/google/uuid" "go.ate.dev/substrate/internal/benchmarking/boomer/boomerutil" + "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" bmetrics "go.ate.dev/substrate/internal/benchmarking/boomer/metrics" "go.ate.dev/substrate/internal/benchmarking/boomer/userclass" "go.ate.dev/substrate/pkg/proto/ateapipb" @@ -53,10 +54,29 @@ func init() { Name: "resumepaused", LocustFile: "resumepaused.py", UserClass: userClass, + Config: knobsCodec, Init: initResumePaused, }) } +// knobs is the ResumePausedUser slice of the runtime config. The master +// populates target_rps from the --target-rps locust flag +// (common/rate_config.py) or the ladder step's @rps suffix. +type knobs struct { + // TargetRPS is the cold resumes per second this worker paces to; zero + // means unpaced. + TargetRPS float64 `json:"target_rps"` +} + +var knobsCodec = dynconfig.Typed[knobs]{ + Validate: func(k knobs) error { + if k.TargetRPS < 0 { + return fmt.Errorf("target_rps cannot be negative: %v", k.TargetRPS) + } + return nil + }, +} + // initResumePaused ensures the atespace and returns the task. Each user // creates and warms its own actor on its first iteration. func initResumePaused(cfg *userclass.Config) (taskFn func(), shutdown func(context.Context)) { @@ -236,15 +256,16 @@ func (r *runtime) cycle(name string) { } } -// recover pauses an actor after a failed cycle, so it is cold for the next -// one. It is load on the server like any other call, so it is recorded, as -// PauseActorRecovery to keep it apart from the measured pauses. An actor -// already paused fails it with FailedPrecondition; the actor is cold, which -// is all recovery wants, so that counts as a success. +// recover suspends an actor asynchronously after a failed cycle, so it is cold +// for the next one. It is load on the server like any other call, so it is +// recorded, as SuspendActorAsyncRecovery to keep it apart from the measured +// suspends. An actor already suspended fails it with FailedPrecondition; the +// actor is cold, which is all recovery wants, so that counts as a success. func (r *runtime) recover(ctx context.Context, name string) { - _ = r.call(ctx, "PauseActorRecovery", func(ctx context.Context) error { - _, err := r.cfg.APIStub.PauseActor(ctx, &ateapipb.PauseActorRequest{ - Actor: r.ref(name), + _ = r.call(ctx, "SuspendActorAsyncRecovery", func(ctx context.Context) error { + _, err := r.cfg.APIStub.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: r.ref(name), + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, }) if status.Code(err) == codes.FailedPrecondition { return nil @@ -269,7 +290,7 @@ func (r *runtime) startCycle() bool { // current TargetRPS first. Zero means unpaced. func (r *runtime) pace(ctx context.Context) { want := rate.Inf - if target := r.cfg.Dyn.Load().TargetRPS; target > 0 { + if target := dynconfig.Get[knobs](r.cfg.Dyn).TargetRPS; target > 0 { want = rate.Limit(target) } if r.limiter.Limit() != want { @@ -296,12 +317,13 @@ func (r *runtime) resume(ctx context.Context, metricName, name string) error { }) } -// pause calls PauseActor. suffix keeps warm-up calls apart from the measured -// ones. +// pause suspends an actor asynchronously, keeping its snapshot on the node. +// suffix keeps warm-up calls apart from the measured ones. func (r *runtime) pause(ctx context.Context, suffix, name string) error { - return r.call(ctx, "PauseActor"+suffix, func(ctx context.Context) error { - _, err := r.cfg.APIStub.PauseActor(ctx, &ateapipb.PauseActorRequest{ - Actor: r.ref(name), + return r.call(ctx, "SuspendActorAsync"+suffix, func(ctx context.Context) error { + _, err := r.cfg.APIStub.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: r.ref(name), + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, }) return err }) diff --git a/internal/benchmarking/boomer/resumepaused/resumepaused_test.go b/internal/benchmarking/boomer/resumepaused/resumepaused_test.go index 64dc06759c..aa6eaa57f9 100644 --- a/internal/benchmarking/boomer/resumepaused/resumepaused_test.go +++ b/internal/benchmarking/boomer/resumepaused/resumepaused_test.go @@ -49,7 +49,7 @@ type fakeControlClient struct { // resumeFailures is how many ResumeActor calls fail with Unavailable, // without changing state, before resumes succeed. resumeFailures int - // pauseErr, when set, fails every PauseActor without changing state. + // pauseErr, when set, fails every async SuspendActor without changing state. pauseErr error // atespaceErr, when set, fails CreateAtespace. atespaceErr error @@ -149,11 +149,15 @@ func (f *fakeControlClient) ResumeActor(ctx context.Context, in *ateapipb.Resume return nil, status.Error(codes.Aborted, "actor crashed") } -func (f *fakeControlClient) PauseActor(ctx context.Context, in *ateapipb.PauseActorRequest, opts ...grpc.CallOption) (*ateapipb.PauseActorResponse, error) { +// SuspendActor fakes only an async suspend, which is how the user pauses. +func (f *fakeControlClient) SuspendActor(ctx context.Context, in *ateapipb.SuspendActorRequest, opts ...grpc.CallOption) (*ateapipb.SuspendActorResponse, error) { f.mu.Lock() defer f.mu.Unlock() name := in.GetActor().GetName() - f.record("PauseActor", name) + f.record("SuspendActor(async)", name) + if in.GetOptions().GetMode() != ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS { + return nil, status.Error(codes.Unimplemented, "only async suspend is faked") + } if f.pauseErr != nil { return nil, f.pauseErr } @@ -163,7 +167,7 @@ func (f *fakeControlClient) PauseActor(ctx context.Context, in *ateapipb.PauseAc // A paused actor is SUSPENDED holding a node-local snapshot; the fake // tracks only the state. f.states[name] = ateapipb.ActorState_ACTOR_STATE_SUSPENDED - return &ateapipb.PauseActorResponse{}, nil + return &ateapipb.SuspendActorResponse{}, nil } func (f *fakeControlClient) DeleteActor(ctx context.Context, in *ateapipb.DeleteActorRequest, opts ...grpc.CallOption) (*ateapipb.Actor, error) { @@ -202,7 +206,7 @@ func newTestRuntime(fake *fakeControlClient) *runtime { r := newRuntime(&userclass.Config{ APIStub: fake, Atespace: "bench", - Dyn: dynconfig.NewHolder(dynconfig.Config{}), + Dyn: dynconfig.Static(knobs{}), Tracer: otel.Tracer("test"), }) r.retryBackoff = time.Millisecond @@ -288,7 +292,7 @@ func TestFirstIterationMakesTheUsersActorCold(t *testing.T) { t.Fatalf("got user actors %v, want 1", actors) } name := actors[0] - want := []string{"CreateActor " + name, "ResumeActor " + name, "PauseActor " + name} + want := []string{"CreateActor " + name, "ResumeActor " + name, "SuspendActor(async) " + name} if !slices.Equal(fake.calls, want) { t.Errorf("calls: got %v, want %v", fake.calls, want) } @@ -314,7 +318,7 @@ func TestIterateResumesThenPausesTheUsersActor(t *testing.T) { for range 3 { r.iterate() } - want := []string{"ResumeActor " + name, "PauseActor " + name} + want := []string{"ResumeActor " + name, "SuspendActor(async) " + name} want = slices.Concat(want, want, want) if !slices.Equal(fake.calls, want) { t.Errorf("calls: got %v, want %v", fake.calls, want) @@ -359,7 +363,7 @@ func TestEachUserCyclesItsOwnActor(t *testing.T) { func TestSetupIsNotPaced(t *testing.T) { fake := newFake() r := newTestRuntime(fake) - r.cfg.Dyn.Store(dynconfig.Config{TargetRPS: 1}) + r.cfg.Dyn = dynconfig.Static(knobs{TargetRPS: 1}) start := time.Now() setUpUsers(t, r, 5) if elapsed := time.Since(start); elapsed > 500*time.Millisecond { @@ -435,17 +439,17 @@ func TestIterateFailsAResumeOfARunningActor(t *testing.T) { fake.setState(name, ateapipb.ActorState_ACTOR_STATE_RUNNING) fake.resetCalls() before := requestCount(t, "ResumeActorCold", "failure") - recoveries := requestCount(t, "PauseActorRecovery", "success") + recoveries := requestCount(t, "SuspendActorAsyncRecovery", "success") r.iterate() if got := requestCount(t, "ResumeActorCold", "failure") - before; got != 1 { t.Errorf("ResumeActorCold failures grew by %v, want 1", got) } - if got := requestCount(t, "PauseActorRecovery", "success") - recoveries; got != 1 { - t.Errorf("PauseActorRecovery successes grew by %v, want 1", got) + if got := requestCount(t, "SuspendActorAsyncRecovery", "success") - recoveries; got != 1 { + t.Errorf("SuspendActorAsyncRecovery successes grew by %v, want 1", got) } // Recovery pauses the running actor. - want := []string{"ResumeActor " + name, "PauseActor " + name} + want := []string{"ResumeActor " + name, "SuspendActor(async) " + name} if !slices.Equal(fake.calls, want) { t.Errorf("calls: got %v, want %v", fake.calls, want) } @@ -475,7 +479,7 @@ func TestIteratePacesResumes(t *testing.T) { r := newTestRuntime(fake) r.iterate() - r.cfg.Dyn.Store(dynconfig.Config{TargetRPS: 200}) + r.cfg.Dyn = dynconfig.Static(knobs{TargetRPS: 200}) start := time.Now() for range 21 { r.iterate() @@ -494,27 +498,27 @@ func TestIterateRecordsTheRecoveryPauseOfAPausedActorAsASuccess(t *testing.T) { fake.resetCalls() fake.resumeErr = status.Error(codes.Aborted, "lease held") resumeFailures := requestCount(t, "ResumeActorCold", "failure") - pauseFailures := requestCount(t, "PauseActor", "failure") - recoverySuccesses := requestCount(t, "PauseActorRecovery", "success") - recoveryFailures := requestCount(t, "PauseActorRecovery", "failure") + pauseFailures := requestCount(t, "SuspendActorAsync", "failure") + recoverySuccesses := requestCount(t, "SuspendActorAsyncRecovery", "success") + recoveryFailures := requestCount(t, "SuspendActorAsyncRecovery", "failure") // The resume fails with the actor still paused, so the recovery pause // finds it paused: a success, as the actor is cold. r.iterate() - if got := fake.callsTo("PauseActor"); !slices.Equal(got, []string{name}) { - t.Errorf("PauseActor calls = %v, want the recovery pause of %s", got, name) + if got := fake.callsTo("SuspendActor(async)"); !slices.Equal(got, []string{name}) { + t.Errorf("async SuspendActor calls = %v, want the recovery pause of %s", got, name) } if got := requestCount(t, "ResumeActorCold", "failure") - resumeFailures; got != 1 { t.Errorf("ResumeActorCold failures grew by %v, want 1", got) } - if got := requestCount(t, "PauseActor", "failure") - pauseFailures; got != 0 { - t.Errorf("PauseActor failures grew by %v, want 0", got) + if got := requestCount(t, "SuspendActorAsync", "failure") - pauseFailures; got != 0 { + t.Errorf("SuspendActorAsync failures grew by %v, want 0", got) } - if got := requestCount(t, "PauseActorRecovery", "success") - recoverySuccesses; got != 1 { - t.Errorf("PauseActorRecovery successes grew by %v, want 1", got) + if got := requestCount(t, "SuspendActorAsyncRecovery", "success") - recoverySuccesses; got != 1 { + t.Errorf("SuspendActorAsyncRecovery successes grew by %v, want 1", got) } - if got := requestCount(t, "PauseActorRecovery", "failure") - recoveryFailures; got != 0 { - t.Errorf("PauseActorRecovery failures grew by %v, want 0", got) + if got := requestCount(t, "SuspendActorAsyncRecovery", "failure") - recoveryFailures; got != 0 { + t.Errorf("SuspendActorAsyncRecovery failures grew by %v, want 0", got) } } @@ -523,17 +527,17 @@ func TestIterateRecordsAFailedRecoveryPause(t *testing.T) { r := newTestRuntime(fake) r.iterate() fake.pauseErr = status.Error(codes.Unavailable, "down") - pauseFailures := requestCount(t, "PauseActor", "failure") - recoveryFailures := requestCount(t, "PauseActorRecovery", "failure") + pauseFailures := requestCount(t, "SuspendActorAsync", "failure") + recoveryFailures := requestCount(t, "SuspendActorAsyncRecovery", "failure") // The resume succeeds, the measured pause fails, and so does the // recovery pause of the running actor. r.iterate() - if got := requestCount(t, "PauseActor", "failure") - pauseFailures; got != 1 { - t.Errorf("PauseActor failures grew by %v, want 1", got) + if got := requestCount(t, "SuspendActorAsync", "failure") - pauseFailures; got != 1 { + t.Errorf("SuspendActorAsync failures grew by %v, want 1", got) } - if got := requestCount(t, "PauseActorRecovery", "failure") - recoveryFailures; got != 1 { - t.Errorf("PauseActorRecovery failures grew by %v, want 1", got) + if got := requestCount(t, "SuspendActorAsyncRecovery", "failure") - recoveryFailures; got != 1 { + t.Errorf("SuspendActorAsyncRecovery failures grew by %v, want 1", got) } } diff --git a/internal/benchmarking/boomer/sweperf/knobs_test.go b/internal/benchmarking/boomer/sweperf/knobs_test.go new file mode 100644 index 0000000000..8b4f9ef5e8 --- /dev/null +++ b/internal/benchmarking/boomer/sweperf/knobs_test.go @@ -0,0 +1,64 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package sweperf + +import ( + "testing" + "time" + + "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" +) + +// The codec takes a payload naming every key the class reads and refuses +// the values it cannot run on. +func TestKnobsCodec(t *testing.T) { + h := dynconfig.NewHolder(knobsCodec) + if got := dynconfig.Get[knobs](h); got != (knobs{}) { + t.Fatalf("fresh holder = %+v, want the zero defaults", got) + } + if _, _, err := h.Apply([]byte(`{ + "min_wait_time": 0.1, "max_wait_time": 0.5, + "sweperf_template": "sweperf-astropy-7336", + "sweperf_total_steps": 21, + "sweperf_num_cycles": 4, + "sweperf_poll_interval_ms": 100 + }`)); err != nil { + t.Fatalf("Apply(valid): %v", err) + } + want := knobs{ + WaitTime: dynconfig.WaitTime{MinWait: dynconfig.Seconds(100 * time.Millisecond), MaxWait: dynconfig.Seconds(500 * time.Millisecond)}, + Template: "sweperf-astropy-7336", + TotalSteps: 21, + NumCycles: 4, + PollIntervalMs: 100, + } + if got := dynconfig.Get[knobs](h); got != want { + t.Errorf("decoded =\n %+v, want\n %+v", got, want) + } + for name, payload := range map[string]string{ + "negative total steps": `{"sweperf_total_steps": -1}`, + "negative num cycles": `{"sweperf_num_cycles": -1}`, + "negative poll interval": `{"sweperf_poll_interval_ms": -1}`, + "max wait below min wait": `{"min_wait_time": 2, "max_wait_time": 1}`, + } { + t.Run(name, func(t *testing.T) { + if _, _, err := h.Apply([]byte(payload)); err == nil { + t.Errorf("Apply(%s) took the payload, want a refusal", payload) + } + if got := dynconfig.Get[knobs](h); got != want { + t.Errorf("a refused payload changed the config to %+v", got) + } + }) + } +} diff --git a/internal/benchmarking/boomer/sweperf/sweperf.go b/internal/benchmarking/boomer/sweperf/sweperf.go index 598b1523d3..878e84c5e6 100644 --- a/internal/benchmarking/boomer/sweperf/sweperf.go +++ b/internal/benchmarking/boomer/sweperf/sweperf.go @@ -29,7 +29,6 @@ import ( "fmt" "io" "log/slog" - "math/rand/v2" "net/http" "strings" "sync" @@ -40,6 +39,7 @@ import ( "go.ate.dev/substrate/internal/ateinterceptors" "go.ate.dev/substrate/internal/atenet" "go.ate.dev/substrate/internal/benchmarking/boomer/boomerutil" + "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" bmetrics "go.ate.dev/substrate/internal/benchmarking/boomer/metrics" "go.ate.dev/substrate/internal/benchmarking/boomer/userclass" "go.ate.dev/substrate/pkg/proto/ateapipb" @@ -86,10 +86,40 @@ func init() { Name: "sweperf", LocustFile: "sweperf.py", UserClass: sweperfUserClass, + Config: knobsCodec, Init: initSweperf, }) } +// knobs is the SweperfUser slice of the runtime config. Each zero falls +// back to the default above; the master populates the keys from the +// --sweperf-* locust flags (common/sweperf_config.py). +type knobs struct { + dynconfig.WaitTime + Template string `json:"sweperf_template"` + TotalSteps int `json:"sweperf_total_steps"` + NumCycles int `json:"sweperf_num_cycles"` + PollIntervalMs int `json:"sweperf_poll_interval_ms"` +} + +var knobsCodec = dynconfig.Typed[knobs]{ + Validate: func(k knobs) error { + if err := k.WaitTime.Validate(); err != nil { + return err + } + if k.TotalSteps < 0 { + return fmt.Errorf("sweperf_total_steps cannot be negative: %d", k.TotalSteps) + } + if k.NumCycles < 0 { + return fmt.Errorf("sweperf_num_cycles cannot be negative: %d", k.NumCycles) + } + if k.PollIntervalMs < 0 { + return fmt.Errorf("sweperf_poll_interval_ms cannot be negative: %d", k.PollIntervalMs) + } + return nil + }, +} + // chunk partitions totalSteps instructions into numCycles slices. type chunk struct { start int @@ -204,19 +234,19 @@ func canceled(ctx context.Context) bool { // the --sweperf-* locust flags (common/sweperf_config.py); an unset field // falls back to the built-in default below. func (r *sweperfRuntime) resolveConfig() (string, int, int) { - dyn := r.cfg.Dyn.Load() + dyn := dynconfig.Get[knobs](r.cfg.Dyn) - template := dyn.SweperfTemplate + template := dyn.Template if template == "" { template = defaultSweperfTemplate } - totalSteps := dyn.SweperfTotalSteps + totalSteps := dyn.TotalSteps if totalSteps <= 0 { totalSteps = defaultSweperfTotalSteps } - numCycles := dyn.SweperfNumCycles + numCycles := dyn.NumCycles if numCycles <= 0 { numCycles = defaultSweperfNumCycles } @@ -228,7 +258,7 @@ func (r *sweperfRuntime) resolveConfig() (string, int, int) { // (--sweperf-poll-interval-ms), or the default when unset. Read per job, so a // mid-run change applies to the next cycle. func pollInterval(cfg *userclass.Config) time.Duration { - if ms := cfg.Dyn.Load().SweperfPollIntervalMs; ms > 0 { + if ms := dynconfig.Get[knobs](cfg.Dyn).PollIntervalMs; ms > 0 { return time.Duration(ms) * time.Millisecond } return defaultSweperfPollInterval @@ -237,12 +267,7 @@ func pollInterval(cfg *userclass.Config) time.Duration { // dynamicWait is the think time between cycles: a uniform draw from // [MinWait, MaxWait), or MinWait when the range is empty. func (r *sweperfRuntime) dynamicWait() time.Duration { - cfg := r.cfg.Dyn.Load() - if cfg.MaxWait <= cfg.MinWait { - return cfg.MinWait - } - jitter := cfg.MaxWait - cfg.MinWait - return cfg.MinWait + time.Duration(rand.Float64()*float64(jitter)) + return dynconfig.Get[knobs](r.cfg.Dyn).WaitTime.Draw() } // iterate is the boomer task function, one call per goroutine per iteration. @@ -255,8 +280,22 @@ func (r *sweperfRuntime) iterate() { if !r.enter() { return } + gid := boomerutil.GoroutineID() + cycleBefore := 0 + if val, ok := r.users.Load(gid); ok { + cycleBefore = val.(*sweperfUser).cycleIndex + } r.runCycle() - time.Sleep(r.dynamicWait()) + + wait := r.dynamicWait() + // Think time counts toward a task only between its cycles: this call + // completed a cycle and the session is still bound, so it was not the last. + if val, ok := r.users.Load(gid); ok { + if u := val.(*sweperfUser); u.cycleIndex > cycleBefore { + u.loopThink += wait + } + } + time.Sleep(wait) } // runCycle is the part of iterate that shutdown waits for: it may start, @@ -300,7 +339,7 @@ func (r *sweperfRuntime) bindUser(gid int64) *sweperfUser { // startUser creates an actor and blocks until its sandbox serves, so the // caller gets a session that is ready to take cycles. Resolves the config per // session, which lets a value changed mid-run apply to later sessions. On -// failure it tears the actor down and logs the error with the actor and +// failure it tears the actor down and logs the error with the user, actor and // template, so a failed start can be tied to its task. func (r *sweperfRuntime) startUser(ctx context.Context) (_ *sweperfUser, err error) { tmpl, totalSteps, numCycles := r.resolveConfig() @@ -313,6 +352,9 @@ func (r *sweperfRuntime) startUser(ctx context.Context) (_ *sweperfUser, err err userClass: sweperfUserClass, chunks: chunks, cycleIndex: 0, + // bindUser and every cycle of this session run on this goroutine. + userID: boomerutil.GoroutineID(), + taskStart: time.Now(), } slog.Info("Creating new sweperf user session", @@ -322,6 +364,7 @@ func (r *sweperfRuntime) startUser(ctx context.Context) (_ *sweperfUser, err err defer func() { if err != nil { slog.Warn("sweperf on_start failed; goroutine will retry next iter", + slog.Int64("user", u.userID), slog.String("actor", u.actorName), slog.String("template", u.templateName), slog.String("err", err.Error()), @@ -341,6 +384,7 @@ func (r *sweperfRuntime) startUser(ctx context.Context) (_ *sweperfUser, err err // CreateActor runs detached from ctx: canceled mid-call, the actor might // exist without this session knowing to delete it. createCtx, cancelCreate := r.teardownCtx() + startupStart := time.Now() err = u.create(createCtx) cancelCreate() if err != nil { @@ -361,6 +405,7 @@ func (r *sweperfRuntime) startUser(ctx context.Context) (_ *sweperfUser, err err u.teardown(tctx) return nil, fmt.Errorf("pollLiveness: %w", err) } + u.startupDur = time.Since(startupStart) // pollLiveness left the actor running, so cycle 1's resume is a no-op. u.awake = true @@ -478,6 +523,16 @@ type sweperfUser struct { loopCEL time.Duration loopWall time.Duration loopFailed bool + // Facts for the task record (logTaskRecord). + userID int64 // boomer goroutine ID, unique per boomer process + taskStart time.Time // when startUser began + startupDur time.Duration // CreateActor until the sandbox answered /status + loopThink time.Duration // think time between this task's cycles + loopRetries int // failed resumes, each of which retries its cycle + resumeMax time.Duration // slowest non-no-op ResumeActor, as recorded on its row + ttfeMax time.Duration // slowest resume-to-ack; zero when no cycle after the no-op resume got an async ack + failStep string // first failed step: resume, execute or suspend + failCycle int // cycle of failStep } // ref is the control-plane reference to this user's actor. @@ -528,11 +583,11 @@ func (u *sweperfUser) create(ctx context.Context) error { }) } -// resume wakes the actor for the next cycle and reports whether it worked. A -// failure is the caller's cue to abandon the cycle rather than talk to a -// sandbox that is not running. -func (u *sweperfUser) resume(ctx context.Context) bool { - err := u.tracedCallRecord(ctx, "ResumeActor", !u.awake, func(callCtx context.Context, tr *metadata.MD) error { +// resume wakes the actor for the next cycle and reports the latency recorded +// on its ResumeActor row and whether it worked. A failure is the caller's cue +// to abandon the cycle rather than talk to a sandbox that is not running. +func (u *sweperfUser) resume(ctx context.Context) (time.Duration, bool) { + latency, err := u.tracedCallRecord(ctx, "ResumeActor", !u.awake, func(callCtx context.Context, tr *metadata.MD) error { _, err := u.cfg.APIStub.ResumeActor(callCtx, &ateapipb.ResumeActorRequest{ Actor: u.ref(), }, grpc.Trailer(tr)) @@ -540,9 +595,9 @@ func (u *sweperfUser) resume(ctx context.Context) bool { }) if err != nil { slog.Error("ResumeActor failed", slog.String("actor", u.actorName), slog.String("err", err.Error())) - return false + return latency, false } - return true + return latency, true } // suspend snapshots the actor and puts it to sleep, ending a cycle. It reports @@ -550,7 +605,8 @@ func (u *sweperfUser) resume(ctx context.Context) bool { func (u *sweperfUser) suspend(ctx context.Context) bool { err := u.tracedCall(ctx, "SuspendActor", func(callCtx context.Context, tr *metadata.MD) error { _, err := u.cfg.APIStub.SuspendActor(callCtx, &ateapipb.SuspendActorRequest{ - Actor: u.ref(), + Actor: u.ref(), + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, }, grpc.Trailer(tr)) return err }) @@ -592,12 +648,13 @@ func (u *sweperfUser) teardown(ctx context.Context) { // the figure excludes the client's own queueing. The client figure goes out as // a _rtt row when the two differ. func (u *sweperfUser) tracedCall(ctx context.Context, name string, do func(context.Context, *metadata.MD) error) error { - return u.tracedCallRecord(ctx, name, true, do) + _, err := u.tracedCallRecord(ctx, name, true, do) + return err } // tracedCallRecord is tracedCall with optional success rows; failures are -// always recorded. -func (u *sweperfUser) tracedCallRecord(ctx context.Context, name string, recordSuccess bool, do func(context.Context, *metadata.MD) error) error { +// always recorded. It also returns the latency it records. +func (u *sweperfUser) tracedCallRecord(ctx context.Context, name string, recordSuccess bool, do func(context.Context, *metadata.MD) error) (time.Duration, error) { ctx, span := u.cfg.Tracer.Start(ctx, name) defer span.End() @@ -606,7 +663,7 @@ func (u *sweperfUser) tracedCallRecord(ctx context.Context, name string, recordS err := do(ctx, &tr) clientLatency := time.Since(start) if err != nil && canceled(ctx) { - return err + return clientLatency, err } latency, source := boomerutil.ElapsedFromMD(tr, ateinterceptors.ServerElapsedTrailer, clientLatency) @@ -619,12 +676,12 @@ func (u *sweperfUser) tracedCallRecord(ctx context.Context, name string, recordS boomerutil.LogSampledTrace(span, name, latency, source, err) if err != nil { bmetrics.RecordFailure("grpc", name, u.userClass, latency, err.Error()) - return err + return latency, err } if recordSuccess { bmetrics.RecordSuccess("grpc", name, u.userClass, latency, 0) } - return nil + return latency, nil } // recordClientRTT publishes the client wall clock for an RPC whose own row carries server time. @@ -693,6 +750,15 @@ func (u *sweperfUser) isDone() bool { return u.cycleIndex >= len(u.chunks) } +// totalSteps is the number of trace steps the session's cycles cover. The +// chunks are contiguous from step 0, so it is the last chunk's end. +func (u *sweperfUser) totalSteps() int { + if len(u.chunks) == 0 { + return 0 + } + return u.chunks[len(u.chunks)-1].end +} + // step runs one cycle: resume the actor, replay that cycle's slice of the // trace, then suspend it again. The final cycle skips the suspend, since the // actor is deleted straight after. @@ -717,25 +783,38 @@ func (u *sweperfUser) step(ctx context.Context) { // Summed per cycle so the between-cycle think time stays out of the total. defer func() { u.loopWall += time.Since(resumeStart) }() // A cycle cut short by shutdown returns without recording or advancing. - if !u.resume(ctx) { + resumeLatency, resumed := u.resume(ctx) + if !resumed { if canceled(ctx) { return } u.loopFailed = true + u.loopRetries++ + u.noteFailure("resume", cycleNum) bmetrics.RecordFailure(derivedMetricMethod, resumeToFirstExecMetric, u.userClass, time.Since(resumeStart), "ResumeActor failed") - slog.Error("ResumeActor failed in step", slog.String("actor", u.actorName), slog.Int("cycle", cycleNum)) + slog.Error("ResumeActor failed in step", + slog.String("actor", u.actorName), + slog.Int("cycle", cycleNum), + slog.Float64("wall_ms", boomerutil.MsFloat(time.Since(resumeStart))), + ) return } u.awake = false + if !noopResume { + u.resumeMax = max(u.resumeMax, resumeLatency) + } // 2. Execute step range chunk inside sandbox ackAt, execDur, err := u.execute(ctx, cycleNum, chunk.start, chunk.end) if err != nil && canceled(ctx) { return } + cycleCEL := execDur if err != nil { u.loopFailed = true + u.noteFailure("execute", cycleNum) + cycleCEL = 0 slog.Error("execute steps failed", slog.String("actor", u.actorName), slog.Int("cycle", cycleNum), @@ -744,6 +823,11 @@ func (u *sweperfUser) step(ctx context.Context) { } else { u.loopCEL += execDur } + var ttfe time.Duration + if !noopResume && !ackAt.IsZero() { + ttfe = ackAt.Sub(resumeStart) + u.ttfeMax = max(u.ttfeMax, ttfe) + } // A no-op resume would record a near-zero sample; failures still count. if !noopResume || ackAt.IsZero() { u.recordResumeToFirstExec(resumeStart, ackAt, err) @@ -751,17 +835,32 @@ func (u *sweperfUser) step(ctx context.Context) { u.recordCycleCEL(execDur, err) // 3. Suspend actor, unless this was the last cycle - if cycleNum < len(u.chunks) && !u.suspend(ctx) { - if canceled(ctx) { - return + suspended := true + if cycleNum < len(u.chunks) { + if suspended = u.suspend(ctx); !suspended { + if canceled(ctx) { + return + } + u.loopFailed = true + u.noteFailure("suspend", cycleNum) } - u.loopFailed = true } - slog.Info("Completed sweperf cycle", + attrs := []any{ slog.String("actor", u.actorName), slog.Int("cycle", cycleNum), - ) + slog.Float64("cel_ms", boomerutil.MsFloat(cycleCEL)), + slog.Float64("wall_ms", boomerutil.MsFloat(time.Since(resumeStart))), + slog.Bool("ok", err == nil && suspended), + } + // A no-op resume and a reply without an async ack have nothing to report. + if !noopResume { + attrs = append(attrs, slog.Float64("resume_ms", boomerutil.MsFloat(resumeLatency))) + } + if ttfe > 0 { + attrs = append(attrs, slog.Float64("ttfe_ms", boomerutil.MsFloat(ttfe))) + } + slog.Info("Completed sweperf cycle", attrs...) u.cycleIndex++ } @@ -798,8 +897,9 @@ func (u *sweperfUser) recordCycleCEL(execDur time.Duration, err error) { } // recordTaskMetrics reports the finished trajectory's container compute time -// and its wall clock, then rearms the counters for the next pass. +// and its wall clock, logs its task record, then resets the loop counters. func (u *sweperfUser) recordTaskMetrics() { + u.logTaskRecord() if u.loopFailed { bmetrics.RecordFailure(derivedMetricMethod, taskCELMetric, u.userClass, u.loopCEL, "cycle failure in trajectory") @@ -814,6 +914,43 @@ func (u *sweperfUser) recordTaskMetrics() { u.loopFailed = false } +// logTaskRecord logs one line of facts about the finished task, so a separate +// script can decide whether it was good. Nothing here judges the task. The +// cycle and step counts are the ones startUser resolved for this task, so +// records from before and after a mid-run config change stay distinguishable. +func (u *sweperfUser) logTaskRecord() { + attrs := []any{ + slog.Int("v", 1), + slog.Int64("user", u.userID), + slog.String("actor", u.actorName), + slog.String("template", u.templateName), + slog.Int("cycles", len(u.chunks)), + slog.Int("steps", u.totalSteps()), + slog.Time("task_start", u.taskStart), + slog.Time("end", time.Now()), + slog.Float64("startup_ms", boomerutil.MsFloat(u.startupDur)), + slog.Float64("cel_ms", boomerutil.MsFloat(u.loopCEL)), + slog.Float64("wall_ms", boomerutil.MsFloat(u.loopWall)), + slog.Float64("think_ms", boomerutil.MsFloat(u.loopThink)), + slog.Float64("resume_max_ms", boomerutil.MsFloat(u.resumeMax)), + slog.Int("retries", u.loopRetries), + slog.Bool("failed", u.loopFailed), + slog.String("fail_step", u.failStep), + slog.Int("fail_cycle", u.failCycle), + } + if u.ttfeMax > 0 { + attrs = append(attrs, slog.Float64("ttfe_max_ms", boomerutil.MsFloat(u.ttfeMax))) + } + slog.Info("sweperf task record", attrs...) +} + +// noteFailure keeps the task's first failed step and its cycle. +func (u *sweperfUser) noteFailure(step string, cycle int) { + if u.failStep == "" { + u.failStep, u.failCycle = step, cycle + } +} + // executeRequest is the /execute body: the 1-based, inclusive range of trace // steps to replay. type executeRequest struct { diff --git a/internal/benchmarking/boomer/sweperf/sweperf_test.go b/internal/benchmarking/boomer/sweperf/sweperf_test.go index 98698991d8..1c63c66c33 100644 --- a/internal/benchmarking/boomer/sweperf/sweperf_test.go +++ b/internal/benchmarking/boomer/sweperf/sweperf_test.go @@ -104,6 +104,7 @@ type fakeControlClient struct { resumeErr error suspendErr error deleteErr error + suspendReqs []*ateapipb.SuspendActorRequest // Make ResumeActor take time and report a smaller server-side elapsed. resumeDelay time.Duration resumeElapsedUs string @@ -160,6 +161,7 @@ func (f *fakeControlClient) SuspendActor(ctx context.Context, in *ateapipb.Suspe f.mu.Lock() defer f.mu.Unlock() f.calls = append(f.calls, "SuspendActor") + f.suspendReqs = append(f.suspendReqs, in) if f.suspendErr != nil { return nil, f.suspendErr } @@ -200,7 +202,7 @@ func newTestConfig(t *testing.T, handler http.Handler) (*userclass.Config, *http HTTPClient: ts.Client(), RouterURL: ts.URL, Atespace: "benchmark-test", - Dyn: dynconfig.NewHolder(dynconfig.Config{}), + Dyn: dynconfig.Static(knobs{}), Tracer: otel.Tracer("test-sweperf"), } return cfg, ts, fakeCtrl @@ -293,7 +295,7 @@ func TestResolveConfig(t *testing.T) { t.Run("default fallback values", func(t *testing.T) { rt := &sweperfRuntime{ cfg: &userclass.Config{ - Dyn: dynconfig.NewHolder(dynconfig.Config{}), + Dyn: dynconfig.Static(knobs{}), }, } tmpl, steps, cycles := rt.resolveConfig() @@ -311,10 +313,10 @@ func TestResolveConfig(t *testing.T) { t.Run("dynamic config overrides", func(t *testing.T) { rt := &sweperfRuntime{ cfg: &userclass.Config{ - Dyn: dynconfig.NewHolder(dynconfig.Config{ - SweperfTemplate: "custom-template", - SweperfTotalSteps: 50, - SweperfNumCycles: 5, + Dyn: dynconfig.Static(knobs{ + Template: "custom-template", + TotalSteps: 50, + NumCycles: 5, }), }, } @@ -339,7 +341,7 @@ func TestPollInterval(t *testing.T) { {0, defaultSweperfPollInterval}, {250, 250 * time.Millisecond}, } { - cfg := &userclass.Config{Dyn: dynconfig.NewHolder(dynconfig.Config{SweperfPollIntervalMs: tt.ms})} + cfg := &userclass.Config{Dyn: dynconfig.Static(knobs{PollIntervalMs: tt.ms})} if got := pollInterval(cfg); got != tt.want { t.Errorf("pollInterval(%d) = %v, want %v", tt.ms, got, tt.want) } @@ -404,6 +406,14 @@ func TestSweperfUserCycleSequence(t *testing.T) { if !reflect.DeepEqual(grpcCalls, wantGRPCCalls) { t.Errorf("gRPC calls: got %v, want %v", grpcCalls, wantGRPCCalls) } + fakeCtrl.mu.Lock() + suspendReqs := fakeCtrl.suspendReqs + fakeCtrl.mu.Unlock() + for _, req := range suspendReqs { + if got := req.GetOptions().GetMode(); got != ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS { + t.Errorf("suspend mode = %v, want %v", got, ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS) + } + } mu.Lock() defer mu.Unlock() @@ -478,7 +488,7 @@ func TestControlClientErrors(t *testing.T) { fakeCtrl.resumeErr = errors.New("resume RPC failed") u := &sweperfUser{cfg: cfg, actorName: "act"} - if ok := u.resume(context.Background()); ok { + if _, ok := u.resume(context.Background()); ok { t.Errorf("resume = true on RPC failure, want false") } }) @@ -580,7 +590,7 @@ func TestSweperfPollLiveness(t *testing.T) { json.NewEncoder(w).Encode(statusResponse{Status: "up"}) }) cfg, _, _ := newTestConfig(t, handler) - cfg.Dyn = dynconfig.NewHolder(dynconfig.Config{SweperfPollIntervalMs: 10}) + cfg.Dyn = dynconfig.Static(knobs{PollIntervalMs: 10}) u := &sweperfUser{cfg: cfg, actorName: "act"} start := time.Now() @@ -594,36 +604,26 @@ func TestSweperfPollLiveness(t *testing.T) { }) } -func TestStartupFailureLogsActorAndTemplate(t *testing.T) { +func TestStartupFailureLogsUserActorAndTemplate(t *testing.T) { + logs := captureLogs(t) cfg, _, fakeCtrl := newTestConfig(t, asyncJobHandler()) fakeCtrl.createActorErr = errors.New("quota exceeded") r := newSweperfRuntime(cfg) - var buf bytes.Buffer - prev := slog.Default() - slog.SetDefault(slog.New(slog.NewJSONHandler(&buf, nil))) - t.Cleanup(func() { slog.SetDefault(prev) }) - if _, err := r.startUser(context.Background()); err == nil { t.Fatalf("startUser succeeded despite CreateActor failing") } - var failure map[string]any - for _, line := range strings.Split(strings.TrimSpace(buf.String()), "\n") { - var rec map[string]any - if json.Unmarshal([]byte(line), &rec) == nil && strings.Contains(fmt.Sprint(rec["msg"]), "on_start failed") { - failure = rec - } - } - if failure == nil { - t.Fatalf("no startup failure logged; got:\n%s", buf.String()) - } + failure := onlyLine(t, logs(), startupFailedMsg) if actor, _ := failure["actor"].(string); !strings.HasPrefix(actor, "sb-") { t.Errorf("failure log actor = %q, want the new actor's name", actor) } if got := failure["template"]; got != defaultSweperfTemplate { t.Errorf("failure log template = %v, want %q", got, defaultSweperfTemplate) } + if got := failure["user"]; got != float64(boomerutil.GoroutineID()) { + t.Errorf("failure log user = %v, want this goroutine's ID", got) + } } func TestSweperfPollJobCompletion(t *testing.T) { @@ -683,9 +683,8 @@ func TestDynamicWait(t *testing.T) { t.Run("returns MinWait when MaxWait <= MinWait", func(t *testing.T) { rt := &sweperfRuntime{ cfg: &userclass.Config{ - Dyn: dynconfig.NewHolder(dynconfig.Config{ - MinWait: 100 * time.Millisecond, - MaxWait: 50 * time.Millisecond, + Dyn: dynconfig.Static(knobs{ + WaitTime: dynconfig.WaitTime{MinWait: dynconfig.Seconds(100 * time.Millisecond), MaxWait: dynconfig.Seconds(50 * time.Millisecond)}, }), }, } @@ -699,9 +698,8 @@ func TestDynamicWait(t *testing.T) { maxW := 50 * time.Millisecond rt := &sweperfRuntime{ cfg: &userclass.Config{ - Dyn: dynconfig.NewHolder(dynconfig.Config{ - MinWait: minW, - MaxWait: maxW, + Dyn: dynconfig.Static(knobs{ + WaitTime: dynconfig.WaitTime{MinWait: dynconfig.Seconds(minW), MaxWait: dynconfig.Seconds(maxW)}, }), }, } @@ -722,9 +720,8 @@ func TestInitSweperfAndTaskFn(t *testing.T) { } }) cfg, _, _ := newTestConfig(t, handler) - cfg.Dyn = dynconfig.NewHolder(dynconfig.Config{ - MinWait: 1 * time.Millisecond, - MaxWait: 2 * time.Millisecond, + cfg.Dyn = dynconfig.Static(knobs{ + WaitTime: dynconfig.WaitTime{MinWait: dynconfig.Seconds(time.Millisecond), MaxWait: dynconfig.Seconds(2 * time.Millisecond)}, }) taskFn, shutdownFn := initSweperf(cfg) @@ -864,7 +861,7 @@ func TestTracedCallRecordsBothLatencies(t *testing.T) { u := &sweperfUser{cfg: cfg, actorName: "act", userClass: sweperfUserClass} before := readStats(t) - if ok := u.resume(context.Background()); !ok { + if _, ok := u.resume(context.Background()); !ok { t.Fatalf("resume failed") } @@ -887,7 +884,7 @@ func TestTracedCallSkipsRTTRowWithoutTrailer(t *testing.T) { u := &sweperfUser{cfg: cfg, actorName: "act", userClass: sweperfUserClass} before := readStats(t) - if ok := u.resume(context.Background()); !ok { + if _, ok := u.resume(context.Background()); !ok { t.Fatalf("resume failed") } @@ -1175,6 +1172,9 @@ func TestStepSuspendFailureFailsTask(t *testing.T) { if !u.loopFailed { t.Errorf("loopFailed = false after SuspendActor failed; the task would be recorded as a success") } + if u.failStep != "suspend" || u.failCycle != 1 { + t.Errorf("first failure = %q in cycle %d, want suspend in cycle 1", u.failStep, u.failCycle) + } } func TestStepSyncExitSkipsResumeToFirstExec(t *testing.T) { @@ -1301,7 +1301,7 @@ func TestTracedCallSkipsFailureWhenCanceled(t *testing.T) { cancel() before := readStats(t) - if ok := u.resume(ctx); ok { + if _, ok := u.resume(ctx); ok { t.Fatalf("resume = true on RPC failure, want false") } for _, name := range []string{"ResumeActor", "ResumeActor_rtt"} { @@ -1326,7 +1326,7 @@ func TestShutdownMidCycleRecordsNoFailures(t *testing.T) { } }) cfg, _, fakeCtrl := newTestConfig(t, handler) - cfg.Dyn = dynconfig.NewHolder(dynconfig.Config{SweperfPollIntervalMs: 10}) + cfg.Dyn = dynconfig.Static(knobs{PollIntervalMs: 10}) r := newSweperfRuntime(cfg) // The last cycle, so an interrupted task would also emit task rows. u := &sweperfUser{cfg: cfg, actorName: "act-1", userClass: sweperfUserClass, chunks: []chunk{{0, 5}}} @@ -1478,3 +1478,256 @@ func TestShutdownSweepsAfterInflightWaitCap(t *testing.T) { t.Errorf("recordedCalls = %v, want the bound actor swept despite the stuck iteration", got) } } + +const ( + taskRecordMsg = "sweperf task record" + completedCycleMsg = "Completed sweperf cycle" + shutdownMsg = "sweperf shutdown started" + startupFailedMsg = "sweperf on_start failed; goroutine will retry next iter" +) + +// syncWriter serializes writes from goroutines logging while a test reads. +type syncWriter struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (w *syncWriter) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + return w.buf.Write(p) +} + +// captureLogs routes slog to JSON in memory for the rest of the test and +// returns a reader of every line logged so far, in order. +func captureLogs(t *testing.T) func() []map[string]any { + t.Helper() + w := &syncWriter{} + prev := slog.Default() + slog.SetDefault(slog.New(slog.NewJSONHandler(w, nil))) + t.Cleanup(func() { slog.SetDefault(prev) }) + return func() []map[string]any { + w.mu.Lock() + defer w.mu.Unlock() + var out []map[string]any + for _, line := range strings.Split(w.buf.String(), "\n") { + var rec map[string]any + if json.Unmarshal([]byte(line), &rec) == nil { + out = append(out, rec) + } + } + return out + } +} + +// linesWith returns the lines whose msg is msg. +func linesWith(lines []map[string]any, msg string) []map[string]any { + var out []map[string]any + for _, l := range lines { + if l["msg"] == msg { + out = append(out, l) + } + } + return out +} + +// onlyLine returns the single line whose msg is msg. +func onlyLine(t *testing.T, lines []map[string]any, msg string) map[string]any { + t.Helper() + got := linesWith(lines, msg) + if len(got) != 1 { + t.Fatalf("%d %q lines, want 1: %v", len(got), msg, got) + } + return got[0] +} + +func TestTaskRecordForCleanTask(t *testing.T) { + logs := captureLogs(t) + cfg, _, fakeCtrl := newTestConfig(t, asyncJobHandler()) + fakeCtrl.resumeElapsedUs = "5000" + cfg.Dyn = dynconfig.Static(knobs{ + WaitTime: dynconfig.WaitTime{MinWait: dynconfig.Seconds(3 * time.Millisecond), MaxWait: dynconfig.Seconds(3 * time.Millisecond)}, + TotalSteps: 10, NumCycles: 2, + }) + r := newSweperfRuntime(cfg) + t.Cleanup(func() { r.shutdown(context.Background()) }) + + r.iterate() // starts the actor and runs cycle 1 (no-op resume) + r.iterate() // cycle 2, the last, ends the task + lines := logs() + + cycles := linesWith(lines, completedCycleMsg) + if len(cycles) != 2 { + t.Fatalf("%d completed cycles, want 2", len(cycles)) + } + first, last := cycles[0], cycles[1] + if first["resume_ms"] != nil || first["ttfe_ms"] != nil { + t.Errorf("cycle 1 resume_ms = %v, ttfe_ms = %v; want neither for a no-op resume", + first["resume_ms"], first["ttfe_ms"]) + } + if first["cel_ms"] != 1200.0 || first["ok"] != true { + t.Errorf("cycle 1 cel_ms = %v, ok = %v; want 1200, true", first["cel_ms"], first["ok"]) + } + // The last cycle is not suspended, and is still ok. + if last["ok"] != true || last["resume_ms"] != 5.0 || last["ttfe_ms"] == nil { + t.Errorf("cycle 2 ok = %v, resume_ms = %v, ttfe_ms = %v; want true, the server's 5ms, set", + last["ok"], last["resume_ms"], last["ttfe_ms"]) + } + + rec := onlyLine(t, lines, taskRecordMsg) + // One think time between the two cycles; none after the last. + want := map[string]any{"v": 1.0, "user": float64(boomerutil.GoroutineID()), "template": defaultSweperfTemplate, + "cycles": 2.0, "steps": 10.0, + "cel_ms": 2400.0, "think_ms": 3.0, "resume_max_ms": 5.0, "retries": 0.0, + "failed": false, "fail_step": "", "fail_cycle": 0.0} + for k, v := range want { + if rec[k] != v { + t.Errorf("record %s = %v, want %v", k, rec[k], v) + } + } + if actor, _ := rec["actor"].(string); !strings.HasPrefix(actor, "sb-") { + t.Errorf("record actor = %q, want the session's sb- actor", actor) + } + if rec["startup_ms"].(float64) <= 0 || rec["ttfe_max_ms"] == nil { + t.Errorf("record startup_ms = %v, ttfe_max_ms = %v; want > 0 and set", rec["startup_ms"], rec["ttfe_max_ms"]) + } + if sum := first["wall_ms"].(float64) + last["wall_ms"].(float64); rec["wall_ms"].(float64) < sum { + t.Errorf("record wall_ms = %v, want at least the cycles' %v", rec["wall_ms"], sum) + } + start, _ := time.Parse(time.RFC3339Nano, rec["task_start"].(string)) + end, _ := time.Parse(time.RFC3339Nano, rec["end"].(string)) + if start.IsZero() || !end.After(start) { + t.Errorf("record task_start = %v, end = %v; want start set and before end", rec["task_start"], rec["end"]) + } +} + +func TestTaskRecordResumeFailureAndRetry(t *testing.T) { + logs := captureLogs(t) + cfg, _, fakeCtrl := newTestConfig(t, asyncJobHandler()) + cfg.Dyn = dynconfig.Static(knobs{ + WaitTime: dynconfig.WaitTime{MinWait: dynconfig.Seconds(4 * time.Millisecond), MaxWait: dynconfig.Seconds(4 * time.Millisecond)}, + }) + r := newSweperfRuntime(cfg) + u := &sweperfUser{cfg: cfg, actorName: "act", userClass: sweperfUserClass, chunks: []chunk{{0, 5}, {5, 10}}} + r.users.Store(boomerutil.GoroutineID(), u) + + fakeCtrl.resumeErr = errors.New("resume failed") + r.iterate() // cycle 1 fails at resume + fakeCtrl.mu.Lock() + fakeCtrl.resumeErr = nil + fakeCtrl.mu.Unlock() + r.iterate() // cycle 1 retried + r.iterate() // cycle 2 ends the task + lines := logs() + + rec := onlyLine(t, lines, taskRecordMsg) + // The wait after the failed resume is a retry, not think time. + want := map[string]any{"retries": 1.0, "failed": true, "fail_step": "resume", "fail_cycle": 1.0, "think_ms": 4.0, + "cycles": 2.0, "steps": 10.0} + for k, v := range want { + if rec[k] != v { + t.Errorf("record %s = %v, want %v", k, rec[k], v) + } + } + // The failed attempt has no cycle line, but its wall time is part of the task's. + sum := onlyLine(t, lines, "ResumeActor failed in step")["wall_ms"].(float64) + for _, c := range linesWith(lines, completedCycleMsg) { + sum += c["wall_ms"].(float64) + } + if rec["wall_ms"].(float64) < sum { + t.Errorf("record wall_ms = %v, want at least %v including the failed resume", rec["wall_ms"], sum) + } +} + +func TestTaskRecordCountsFollowConfigChange(t *testing.T) { + logs := captureLogs(t) + cfg, _, _ := newTestConfig(t, asyncJobHandler()) + cfg.Dyn = dynconfig.Static(knobs{TotalSteps: 4, NumCycles: 2}) + r := newSweperfRuntime(cfg) + t.Cleanup(func() { r.shutdown(context.Background()) }) + + r.iterate() // task 1, cycle 1 + // A change mid-task applies from the next task on. + if _, _, err := cfg.Dyn.Apply([]byte(`{"sweperf_total_steps": 3, "sweperf_num_cycles": 5}`)); err != nil { + t.Fatalf("Apply: %v", err) + } + r.iterate() // task 1, cycle 2, the last + for range 3 { + r.iterate() // task 2: five cycles over three steps run as three + } + + recs := linesWith(logs(), taskRecordMsg) + if len(recs) != 2 { + t.Fatalf("%d task records, want 2", len(recs)) + } + for i, want := range []struct{ cycles, steps float64 }{{2, 4}, {3, 3}} { + if recs[i]["cycles"] != want.cycles || recs[i]["steps"] != want.steps { + t.Errorf("record %d cycles = %v, steps = %v; want %v, %v", + i+1, recs[i]["cycles"], recs[i]["steps"], want.cycles, want.steps) + } + } +} + +func TestTaskRecordExecuteFailureOnLastCycle(t *testing.T) { + logs := captureLogs(t) + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "boom", http.StatusInternalServerError) + }) + cfg, _, _ := newTestConfig(t, handler) + u := &sweperfUser{cfg: cfg, actorName: "act", userClass: sweperfUserClass, chunks: []chunk{{0, 5}}} + + u.step(context.Background()) + u.recordTaskMetrics() + lines := logs() + + if cyc := onlyLine(t, lines, completedCycleMsg); cyc["ok"] != false || cyc["cel_ms"] != 0.0 || cyc["ttfe_ms"] != nil { + t.Errorf("cycle ok = %v, cel_ms = %v, ttfe_ms = %v; want false, 0, unset", cyc["ok"], cyc["cel_ms"], cyc["ttfe_ms"]) + } + rec := onlyLine(t, lines, taskRecordMsg) + if rec["failed"] != true || rec["fail_step"] != "execute" || rec["fail_cycle"] != 1.0 || rec["ttfe_max_ms"] != nil { + t.Errorf("record failed = %v, fail_step = %v, fail_cycle = %v, ttfe_max_ms = %v; want true, execute, 1, unset", + rec["failed"], rec["fail_step"], rec["fail_cycle"], rec["ttfe_max_ms"]) + } +} + +func TestShutdownCutsOffInFlightCycle(t *testing.T) { + for _, tc := range []struct { + name string + chunks []chunk + }{ + {name: "last cycle", chunks: []chunk{{0, 5}}}, + {name: "earlier cycle", chunks: []chunk{{0, 5}, {5, 10}}}, + } { + t.Run(tc.name, func(t *testing.T) { + logs := captureLogs(t) + entered := make(chan struct{}) + var once sync.Once + handler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + // Read the body so the server notices when the client cancels. + json.NewDecoder(req.Body).Decode(&executeRequest{}) + once.Do(func() { close(entered) }) + <-req.Context().Done() // the chunk is still running when shutdown begins + }) + cfg, _, _ := newTestConfig(t, handler) + r := newSweperfRuntime(cfg) + u := &sweperfUser{cfg: cfg, actorName: "act", userClass: sweperfUserClass, chunks: tc.chunks} + + // Sessions are keyed by goroutine, so bind u on the goroutine that iterates. + go func() { + r.users.Store(boomerutil.GoroutineID(), u) + r.iterate() + }() + <-entered + r.shutdown(context.Background()) + lines := logs() + + onlyLine(t, lines, shutdownMsg) + if cycs := linesWith(lines, completedCycleMsg); len(cycs) != 0 { + t.Errorf("completed cycle logged for a canceled cycle: %v", cycs) + } + if recs := linesWith(lines, taskRecordMsg); len(recs) != 0 { + t.Errorf("task record logged for a canceled task: %v", recs) + } + }) + } +} diff --git a/internal/benchmarking/boomer/userclass/config.go b/internal/benchmarking/boomer/userclass/config.go index 28789ba3ec..9eef062ba4 100644 --- a/internal/benchmarking/boomer/userclass/config.go +++ b/internal/benchmarking/boomer/userclass/config.go @@ -35,9 +35,10 @@ type Config struct { // Atespace every actor this worker creates lives in. Required; caller // is responsible for having ensured it exists (see EnsureAtespace). Atespace string - // Dyn is the runtime-mutable config (wait-time bounds, trace - // probability). Required — every per-iteration read goes through it, - // so tests can mutate it without touching glutton internals. + // Dyn is the runtime-mutable config, built from the class's + // Entry.Config codec. Required: a class reads its typed knobs from it + // through dynconfig.Get on every iteration, so tests can swap the + // holder without touching class internals. Dyn *dynconfig.Holder // Tracer anchors sampled spans; falls back to the otel global if nil. Tracer trace.Tracer diff --git a/internal/benchmarking/boomer/userclass/registry.go b/internal/benchmarking/boomer/userclass/registry.go index f429c6a9b1..f198e4ca91 100644 --- a/internal/benchmarking/boomer/userclass/registry.go +++ b/internal/benchmarking/boomer/userclass/registry.go @@ -19,6 +19,8 @@ import ( "fmt" "slices" "sync" + + "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" ) // Entry declares one user class: the flag value that selects it, the Locust @@ -31,6 +33,12 @@ type Entry struct { // UserClass is the Python class name. It must equal boomer.Task.Name or // the master's spawn messages never match and no users start. UserClass string + // Config is the class's runtime config codec: a dynconfig.Typed over + // the struct it reads its knobs from, with its defaults and rules. The + // worker builds the holder from it, so a payload the class cannot run + // on is refused at fetch time rather than failing in the middle of a + // run. Nil when the class reads no runtime config. + Config dynconfig.Codec // Init builds the boomer task func and its shutdown hook. Init func(*Config) (task func(), shutdown func(context.Context)) } diff --git a/internal/e2e/atenet_dataplane.go b/internal/e2e/atenet_dataplane.go index 38235eb892..ba8a71e4aa 100644 --- a/internal/e2e/atenet_dataplane.go +++ b/internal/e2e/atenet_dataplane.go @@ -36,6 +36,7 @@ type AtenetDataplane interface { ParkingBudgetStatus() int IsEgressPolicyDenied(status int, body string) bool EgressUpgradeDeniedBody() string + EgressConnectDeniedResponse() (int, string) PlatformMetricPrefixes([]string) []string RouteDurationSeen(context.Context, string) (bool, error) ScrapeMetrics(context.Context) (string, error) @@ -81,6 +82,10 @@ func (envoyAtenetDataplane) EgressUpgradeDeniedBody() string { return "WebSocket egress is not supported" } +func (envoyAtenetDataplane) EgressConnectDeniedResponse() (int, string) { + return http.StatusForbidden, "CONNECT egress is not supported" +} + func (envoyAtenetDataplane) IsEgressPolicyDenied(status int, body string) bool { return (status == http.StatusForbidden && strings.Contains(body, "egress denied")) || (status == http.StatusBadGateway && strings.Contains(body, "request failed")) @@ -150,6 +155,10 @@ func (agentGatewayAtenetDataplane) EgressUpgradeDeniedBody() string { return "HTTP upgrades, including CONNECT, are denied for actor egress" } +func (agentGatewayAtenetDataplane) EgressConnectDeniedResponse() (int, string) { + return http.StatusMethodNotAllowed, "method not allowed" +} + func (agentGatewayAtenetDataplane) IsEgressPolicyDenied(status int, body string) bool { return status == http.StatusForbidden && strings.Contains(body, "actor egress policy denied") } diff --git a/internal/e2e/collector_metrics.go b/internal/e2e/collector_metrics.go index 4ba3e0c7b4..45aad928c5 100644 --- a/internal/e2e/collector_metrics.go +++ b/internal/e2e/collector_metrics.go @@ -56,6 +56,8 @@ var PlatformMetricPrefixes = []string{ "ate_actor_restore_duration", "ate_actor_checkpoint_duration", "atelet_snapshot_size", + "atelet_snapshot_upload_overdue_actors", + "atelet_snapshot_undurable_age_max", "atenet_router_route_duration", } @@ -230,16 +232,16 @@ func scrapeEgressEnvoyPod(ctx context.Context, config *rest.Config, clientset ku // harness: kind has no place to query log records. const LifecycleEventMetric = "substrate_actor_state_changes" -// LifecycleEventCounts returns the count the collector holds for each -// ate.actor.state. The counter is cumulative and the collector outlives any one -// test, so compare two reads rather than asserting a state is merely present: -// a stale count from an earlier run would pass a presence check even with the -// exporter turned off. +// LifecycleEventSeries returns each lifecycle series the collector holds, keyed +// by ate.actor.state and then by the series' full name and labels. The counter +// is cumulative and the collector outlives any one test, so compare two reads +// rather than asserting a state is merely present: a stale count from an +// earlier run would pass a presence check even with the exporter turned off. // -// One state can appear on several lines, one per emitting ateapi instance, so -// the counts are summed. -func LifecycleEventCounts(scrape string) map[string]float64 { - counts := map[string]float64{} +// Series are kept apart, one per emitting instance, otherwise a series the +// exporter expired looks like the count went down. +func LifecycleEventSeries(scrape string) map[string]map[string]float64 { + series := map[string]map[string]float64{} for _, line := range strings.Split(scrape, "\n") { name := metricNameFromLine(line) if name != LifecycleEventMetric && !strings.HasPrefix(name, LifecycleEventMetric+"_") { @@ -249,22 +251,40 @@ func LifecycleEventCounts(scrape string) map[string]float64 { if state == "" { continue } - fields := strings.Fields(line) - v, err := strconv.ParseFloat(fields[len(fields)-1], 64) + line = strings.TrimSpace(line) + i := strings.LastIndexAny(line, " \t") + if i < 0 { + continue + } + v, err := strconv.ParseFloat(line[i+1:], 64) if err != nil { continue } - counts[state] += v + if series[state] == nil { + series[state] = map[string]float64{} + } + series[state][strings.TrimSpace(line[:i])] = v } - return counts + return series } -// StatesNotAdvanced returns the states whose count did not rise between the two -// reads. An empty result means every state was emitted during the window. -func StatesNotAdvanced(before, after map[string]float64, states []string) []string { +// StatesNotAdvanced returns the states with no series that appeared or changed +// between the two reads. An empty result means every state was emitted during +// the window. +// +// A drop counts, since deltatocumulative only restarts a stale stream on a new +// count. A series that vanished does not, the exporter expires idle series. +func StatesNotAdvanced(before, after map[string]map[string]float64, states []string) []string { var stale []string for _, s := range states { - if after[s] <= before[s] { + advanced := false + for key, v := range after[s] { + if old, ok := before[s][key]; !ok || v != old { + advanced = true + break + } + } + if !advanced { stale = append(stale, s) } } diff --git a/internal/e2e/collector_metrics_test.go b/internal/e2e/collector_metrics_test.go index 2cdb46e6f9..ce95e8f5db 100644 --- a/internal/e2e/collector_metrics_test.go +++ b/internal/e2e/collector_metrics_test.go @@ -17,6 +17,8 @@ package e2e import ( "sort" "testing" + + "github.com/google/go-cmp/cmp" ) // A trimmed sample of what the Collector's prometheus exporter emits: HELP/TYPE @@ -166,81 +168,110 @@ substrate_actor_state_changes_total{service_name="ateapi",ate_actor_state="crash substrate_spans_total{service_name="ateapi"} 40 ` -func TestLifecycleEventCounts(t *testing.T) { +func TestLifecycleEventSeries(t *testing.T) { tests := []struct { name string scrape string - want map[string]float64 + want map[string]map[string]float64 }{ { - name: "every counted state, summed over instances", + name: "every counted state, one entry per instance", scrape: sampleLifecycleScrape, - want: map[string]float64{"resuming": 2, "running": 5, "crashed": 1}, + want: map[string]map[string]float64{ + "resuming": {`substrate_actor_state_changes_total{service_name="ateapi",ate_actor_state="resuming"}`: 2}, + "running": { + `substrate_actor_state_changes_total{service_name="ateapi",ate_actor_state="running"}`: 2, + `substrate_actor_state_changes_total{service_name="ateapi",instance="other",ate_actor_state="running"}`: 3, + }, + "crashed": {`substrate_actor_state_changes_total{service_name="ateapi",ate_actor_state="crashed"}`: 1}, + }, }, { name: "no events counted", scrape: sampleScrape, - want: map[string]float64{}, + want: map[string]map[string]float64{}, }, { name: "empty scrape", scrape: "", - want: map[string]float64{}, + want: map[string]map[string]float64{}, }, { name: "a series with no state label contributes nothing", scrape: "substrate_actor_state_changes_total{service_name=\"ateapi\"} 3\n", - want: map[string]float64{}, + want: map[string]map[string]float64{}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := LifecycleEventCounts(tt.scrape) - if len(got) != len(tt.want) { - t.Fatalf("LifecycleEventCounts() = %v, want %v", got, tt.want) - } - for state, want := range tt.want { - if got[state] != want { - t.Errorf("LifecycleEventCounts()[%q] = %v, want %v", state, got[state], want) - } + got := LifecycleEventSeries(tt.scrape) + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("LifecycleEventSeries() mismatch (-want +got):\n%s", diff) } }) } } func TestStatesNotAdvanced(t *testing.T) { + const a, b = `m{instance="a"}`, `m{instance="b"}` + type series = map[string]map[string]float64 tests := []struct { name string - before, after map[string]float64 + before, after series states []string want []string }{ { name: "every state advanced", - before: map[string]float64{"running": 1, "crashed": 0}, - after: map[string]float64{"running": 3, "crashed": 1}, + before: series{"running": {a: 1}, "crashed": {a: 0}}, + after: series{"running": {a: 3}, "crashed": {a: 1}}, states: []string{"running", "crashed"}, }, { name: "a stale count from an earlier run is not an advance", - before: map[string]float64{"running": 3, "crashed": 1}, - after: map[string]float64{"running": 3, "crashed": 1}, + before: series{"running": {a: 3}, "crashed": {a: 1}}, + after: series{"running": {a: 3}, "crashed": {a: 1}}, states: []string{"running", "crashed"}, want: []string{"running", "crashed"}, }, { name: "a state that never appeared", - before: map[string]float64{}, - after: map[string]float64{"running": 2}, + before: series{}, + after: series{"running": {a: 2}}, states: []string{"running", "suspended"}, want: []string{"suspended"}, }, { name: "first observation of a state counts as an advance", - before: map[string]float64{}, - after: map[string]float64{"running": 1}, + before: series{}, + after: series{"running": {a: 1}}, states: []string{"running"}, }, + { + name: "a new instance's series counts as an advance", + before: series{"crashed": {a: 3}}, + after: series{"crashed": {a: 3, b: 1}}, + states: []string{"crashed"}, + }, + { + name: "an expired series does not hide an advance on another", + before: series{"crashed": {a: 3, b: 1}}, + after: series{"crashed": {b: 2}}, + states: []string{"crashed"}, + }, + { + name: "an expired series alone is not an advance", + before: series{"crashed": {a: 3, b: 1}}, + after: series{"crashed": {b: 1}}, + states: []string{"crashed"}, + want: []string{"crashed"}, + }, + { + name: "a restarted stream counts as an advance", + before: series{"crashed": {a: 3}}, + after: series{"crashed": {a: 1}}, + states: []string{"crashed"}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/e2e/fixture.go b/internal/e2e/fixture.go index 4c35de5bc3..daeb254af5 100644 --- a/internal/e2e/fixture.go +++ b/internal/e2e/fixture.go @@ -39,10 +39,6 @@ func substrateTemplateSubstitutions(bucket, name string, trustBundles []string) "${FIXTURE_SUFFIX}": "-" + name, } blocks = map[string]string{ - // gvisor-default is the cluster-wide SandboxConfig - // manifests/ate-install ships; config_name is required, so the - // fixtures name it explicitly. - "${TEMPLATE_SANDBOX_CONFIG}": "sandboxConfig:\n sandboxClass: gvisor\n configName: gvisor-default", // Off unless the caller opts in; see WithTrustBundle. "${TEMPLATE_TRUST_BUNDLE}": "", } @@ -60,9 +56,6 @@ func substrateTemplateSubstitutions(bucket, name string, trustBundles []string) } inline["${FIXTURE_SUFFIX}"] = "-" + SandboxClassMicroVM + "-" + name - // The cluster-wide SandboxConfig hack/install-microvm-deps.sh installs; - // a missing or stale one fails loudly at template creation. - blocks["${TEMPLATE_SANDBOX_CONFIG}"] = "sandboxConfig:\n sandboxClass: microvm\n configName: microvm" return inline, blocks } diff --git a/internal/e2e/fixtures/capabilities/capabilities-templates.yaml.tmpl b/internal/e2e/fixtures/capabilities/capabilities-templates.yaml.tmpl index c0fe161dc2..c132736f3c 100644 --- a/internal/e2e/fixtures/capabilities/capabilities-templates.yaml.tmpl +++ b/internal/e2e/fixtures/capabilities/capabilities-templates.yaml.tmpl @@ -42,7 +42,6 @@ resources: quantity: "1" - name: memory quantity: 512Mi -${TEMPLATE_SANDBOX_CONFIG} snapshotConfig: storageLocation: gs://${BUCKET_NAME}/ate-e2e-caps-default${FIXTURE_SUFFIX}/ @@ -79,6 +78,5 @@ resources: quantity: "1" - name: memory quantity: 512Mi -${TEMPLATE_SANDBOX_CONFIG} snapshotConfig: storageLocation: gs://${BUCKET_NAME}/ate-e2e-caps-exact${FIXTURE_SUFFIX}/ diff --git a/internal/e2e/fixtures/probe/main.go b/internal/e2e/fixtures/probe/main.go index e1c3a113b0..7a732bb4e2 100644 --- a/internal/e2e/fixtures/probe/main.go +++ b/internal/e2e/fixtures/probe/main.go @@ -224,6 +224,19 @@ func writefile(w http.ResponseWriter, r *http.Request) { writeJSON(w, resp) } +// lookup resolves ?host= with the Go resolver, which reads /etc/hosts before +// asking DNS, so a test can assert on the names the sandbox resolves locally. +func lookup(w http.ResponseWriter, r *http.Request) { + host := r.URL.Query().Get("host") + resp := map[string]any{"host": host} + if addrs, err := net.DefaultResolver.LookupHost(r.Context(), host); err == nil { + resp["addrs"] = addrs + } else { + resp["error"] = err.Error() + } + writeJSON(w, resp) +} + // readAllAt reads f's full contents from offset 0 without moving its offset, // so concurrent requests do not interleave seeks on the shared fd. func readAllAt(f *os.File) ([]byte, error) { @@ -474,6 +487,7 @@ func main() { mux.HandleFunc("/fetch", fetch) mux.HandleFunc("/readfile", readfile) mux.HandleFunc("/writefile", writefile) + mux.HandleFunc("/lookup", lookup) mux.HandleFunc("/resources", resources) mux.HandleFunc("/capabilities", capabilities) mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) diff --git a/internal/e2e/fixtures/probe/probe-multiactor-template.yaml.tmpl b/internal/e2e/fixtures/probe/probe-multiactor-template.yaml.tmpl index fdfa031a94..01a35f7cc2 100644 --- a/internal/e2e/fixtures/probe/probe-multiactor-template.yaml.tmpl +++ b/internal/e2e/fixtures/probe/probe-multiactor-template.yaml.tmpl @@ -52,6 +52,5 @@ resources: quantity: "1" - name: memory quantity: 256Mi -${TEMPLATE_SANDBOX_CONFIG} snapshotConfig: storageLocation: gs://${BUCKET_NAME}/ate-e2e${FIXTURE_SUFFIX}/ diff --git a/internal/e2e/fixtures/probe/probe-parking-template.yaml.tmpl b/internal/e2e/fixtures/probe/probe-parking-template.yaml.tmpl index 120513cfe0..92f7f7ce29 100644 --- a/internal/e2e/fixtures/probe/probe-parking-template.yaml.tmpl +++ b/internal/e2e/fixtures/probe/probe-parking-template.yaml.tmpl @@ -51,6 +51,5 @@ resources: quantity: "1" - name: memory quantity: 512Mi -${TEMPLATE_SANDBOX_CONFIG} snapshotConfig: storageLocation: gs://${BUCKET_NAME}/ate-e2e${FIXTURE_SUFFIX}/ diff --git a/internal/e2e/fixtures/probe/probe-sized-template.yaml.tmpl b/internal/e2e/fixtures/probe/probe-sized-template.yaml.tmpl index b1ed5d078f..aea06d8029 100644 --- a/internal/e2e/fixtures/probe/probe-sized-template.yaml.tmpl +++ b/internal/e2e/fixtures/probe/probe-sized-template.yaml.tmpl @@ -45,6 +45,5 @@ resources: quantity: "2" - name: memory quantity: 512Mi -${TEMPLATE_SANDBOX_CONFIG} snapshotConfig: storageLocation: gs://${BUCKET_NAME}/ate-e2e${FIXTURE_SUFFIX}/ diff --git a/internal/e2e/fixtures/probe/probe-template.yaml.tmpl b/internal/e2e/fixtures/probe/probe-template.yaml.tmpl index a5e22aebab..29e2bf9486 100644 --- a/internal/e2e/fixtures/probe/probe-template.yaml.tmpl +++ b/internal/e2e/fixtures/probe/probe-template.yaml.tmpl @@ -62,6 +62,5 @@ resources: quantity: "1" - name: memory quantity: 512Mi -${TEMPLATE_SANDBOX_CONFIG} snapshotConfig: storageLocation: gs://${BUCKET_NAME}/ate-e2e-probe${FIXTURE_SUFFIX}/ diff --git a/internal/e2e/fixtures/security/atespace-template.yaml.tmpl b/internal/e2e/fixtures/security/atespace-template.yaml.tmpl index ae64bf69a2..0cda6a3407 100644 --- a/internal/e2e/fixtures/security/atespace-template.yaml.tmpl +++ b/internal/e2e/fixtures/security/atespace-template.yaml.tmpl @@ -76,6 +76,5 @@ resources: quantity: "1" - name: memory quantity: 512Mi -${TEMPLATE_SANDBOX_CONFIG} snapshotConfig: storageLocation: gs://${BUCKET_NAME}/ate-e2e${FIXTURE_SUFFIX}/ diff --git a/internal/e2e/fixtures/testserver/grpcecho-template.yaml.tmpl b/internal/e2e/fixtures/testserver/grpcecho-template.yaml.tmpl index e894e82c4e..e3d85ac91f 100644 --- a/internal/e2e/fixtures/testserver/grpcecho-template.yaml.tmpl +++ b/internal/e2e/fixtures/testserver/grpcecho-template.yaml.tmpl @@ -51,6 +51,5 @@ resources: quantity: "1" - name: memory quantity: 512Mi -${TEMPLATE_SANDBOX_CONFIG} snapshotConfig: storageLocation: gs://${BUCKET_NAME}/ate-e2e-grpcecho${FIXTURE_SUFFIX}/ diff --git a/internal/e2e/fixtures/testserver/http.go b/internal/e2e/fixtures/testserver/http.go index 6da26f7ad0..95ba7aaa7e 100644 --- a/internal/e2e/fixtures/testserver/http.go +++ b/internal/e2e/fixtures/testserver/http.go @@ -88,6 +88,23 @@ func newHTTPHandler(body, authorizationFile string) http.Handler { return mux } +// acceptingConnect answers every CONNECT with 200, as a forward proxy does +// when it opens the tunnel, and hands anything else to next. Nothing is +// relayed: the server stands in for a proxy only so that a client whose +// CONNECT is refused can tell the refusal did not come from here. A ServeMux +// alone would answer a CONNECT 404, which is harder to tell from one. +func acceptingConnect(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodConnect { + next.ServeHTTP(w, r) + return + } + // A 2xx to a CONNECT has no body; say so, or the server frames one. + w.Header().Set("Content-Length", "0") + w.WriteHeader(http.StatusOK) + }) +} + // dial opens a TCP connection to ?address= within ?timeout= // (default 3s) and closes it again. A suite uses it to check what this pod // can reach, such as whether a NetworkPolicy lets the pod's traffic through. @@ -130,15 +147,20 @@ func newHTTPCmd() *cobra.Command { certFile string keyFile string body string + acceptConnect bool ) cmd := &cobra.Command{ Use: "http", - Short: "Serve an HTTP/1.1 origin answering /healthz, /echoheaders, POST /fetch and /dial.", + Short: "Serve an HTTP/1.1 origin answering /healthz, /echoheaders, POST /fetch and /dial, and with --accept-connect any CONNECT.", Args: cobra.NoArgs, RunE: func(_ *cobra.Command, _ []string) error { + handler := newHTTPHandler(body, authorizationFile) + if acceptConnect { + handler = acceptingConnect(handler) + } server := &http.Server{ Addr: listenAddress, - Handler: newHTTPHandler(body, authorizationFile), + Handler: handler, ReadHeaderTimeout: 10 * time.Second, WriteTimeout: 2 * time.Minute, } @@ -151,6 +173,7 @@ func newHTTPCmd() *cobra.Command { cmd.Flags().StringVar(&keyFile, "tls-key", "", "PEM private key file; with --tls-cert, also serves HTTPS on :8443.") cmd.Flags().StringVar(&authorizationFile, "authorization-file", "", "Enable /credential, requiring a Bearer token matching this file.") cmd.Flags().StringVar(&body, "body", "", "Response body for /healthz.") + cmd.Flags().BoolVar(&acceptConnect, "accept-connect", false, "Answer CONNECT requests 200, as a forward proxy would, without relaying anything.") return cmd } diff --git a/internal/e2e/fixtures/testserver/http_test.go b/internal/e2e/fixtures/testserver/http_test.go index 27569aa90c..79fd6254d0 100644 --- a/internal/e2e/fixtures/testserver/http_test.go +++ b/internal/e2e/fixtures/testserver/http_test.go @@ -15,10 +15,13 @@ package main import ( + "bufio" "crypto/tls" "crypto/x509" "encoding/json" + "fmt" "io" + "net" "net/http" "net/http/httptest" "net/url" @@ -332,3 +335,54 @@ func TestCredentialHandler(t *testing.T) { t.Fatalf("unreadable credential file: status=%d, want 500", resp.Code) } } + +// sendConnect opens a connection to originURL's host, sends it a CONNECT for +// target, and returns the status it answered with. +func sendConnect(t *testing.T, originURL, target string) int { + t.Helper() + parsed, err := url.Parse(originURL) + if err != nil { + t.Fatalf("parsing %s: %v", originURL, err) + } + conn, err := net.DialTimeout("tcp", parsed.Host, 2*time.Second) + if err != nil { + t.Fatalf("dialing %s: %v", parsed.Host, err) + } + defer conn.Close() + if err := conn.SetDeadline(time.Now().Add(2 * time.Second)); err != nil { + t.Fatalf("setting deadline: %v", err) + } + if _, err := fmt.Fprintf(conn, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n", target, target); err != nil { + t.Fatalf("sending CONNECT %s: %v", target, err) + } + response, err := http.ReadResponse(bufio.NewReader(conn), &http.Request{Method: http.MethodConnect}) + if err != nil { + t.Fatalf("reading the answer to CONNECT %s: %v", target, err) + } + defer response.Body.Close() + return response.StatusCode +} + +func TestHTTPAcceptConnectAnswersCONNECT(t *testing.T) { + server := startOriginServer(t, acceptingConnect(newHTTPHandler("", "")), "", "") + if status := sendConnect(t, server.httpURL, "example.com:443"); status != http.StatusOK { + t.Errorf("CONNECT status = %d, want %d", status, http.StatusOK) + } + // Anything else is still the origin. + client := &http.Client{Timeout: 2 * time.Second} + response, err := client.Get(server.httpURL + "/healthz") + if err != nil { + t.Fatalf("GET /healthz: %v", err) + } + response.Body.Close() + if response.StatusCode != http.StatusOK { + t.Errorf("GET /healthz status = %d, want %d", response.StatusCode, http.StatusOK) + } +} + +func TestHTTPRefusesCONNECTByDefault(t *testing.T) { + server := startOriginServer(t, newHTTPHandler("", ""), "", "") + if status := sendConnect(t, server.httpURL, "example.com:443"); status/100 == 2 { + t.Errorf("CONNECT status = %d, want the origin not to open a tunnel", status) + } +} diff --git a/internal/e2e/fixtures/testserver/websocket-template.yaml.tmpl b/internal/e2e/fixtures/testserver/websocket-template.yaml.tmpl index 16e8d95f33..c9166a545a 100644 --- a/internal/e2e/fixtures/testserver/websocket-template.yaml.tmpl +++ b/internal/e2e/fixtures/testserver/websocket-template.yaml.tmpl @@ -37,6 +37,5 @@ resources: quantity: "1" - name: memory quantity: 512Mi -${TEMPLATE_SANDBOX_CONFIG} snapshotConfig: storageLocation: gs://${BUCKET_NAME}/ate-e2e-websocket${FIXTURE_SUFFIX}/ diff --git a/internal/e2e/sandbox.go b/internal/e2e/sandbox.go index e8de901130..878c155bc0 100644 --- a/internal/e2e/sandbox.go +++ b/internal/e2e/sandbox.go @@ -21,7 +21,7 @@ import ( ) // SandboxClassMicroVM is the kata + cloud-hypervisor runtime, spelled as the -// WorkerPool spec.sandboxClasses[].name and ActorTemplate sandboxClass spell it. +// WorkerPool spec.sandboxClasses[].name spells it. const SandboxClassMicroVM = "microvm" // sandboxClassEnv selects which runtime's fixtures the suites build actors @@ -52,8 +52,8 @@ type Fixture struct { // SubstrateFixture identifies an installed substrate ActorTemplate (the proto // resource created through the ate API, not the CRD) plus the CRD WorkerPool // backing it. Suites copy the resolved runtime — container images, sandbox -// config, sandbox size — out of the template, and the worker image and sandbox -// class out of the pool. +// size — out of the template, and the worker image and sandbox class out of +// the pool. type SubstrateFixture struct { // Atespace and Name locate the ActorTemplate for GetActorTemplate. Atespace string @@ -184,8 +184,7 @@ func fixtureSubstitutions(bucket, name string) (inline, blocks map[string]string "${FIXTURE_SUFFIX}": "-" + name, } blocks = map[string]string{ - "${WORKERPOOL_RUNTIME}": " sandboxClasses:\n - name: gvisor", - "${TEMPLATE_SANDBOX_CLASS}": "", + "${WORKERPOOL_RUNTIME}": " sandboxClasses:\n - name: gvisor", // Off unless the caller opts in; see WithTrustBundle. "${TEMPLATE_TRUST_BUNDLE}": "", } @@ -195,12 +194,6 @@ func fixtureSubstitutions(bucket, name string) (inline, blocks map[string]string inline["${WORKER_IMAGE}"] = "ko://go.ate.dev/substrate/cmd/ateworker-microvm" inline["${FIXTURE_SUFFIX}"] = "-" + SandboxClassMicroVM + "-" + name - // The micro-VM ActorTemplates name the cluster-wide SandboxConfig - // hack/install-microvm-deps.sh installs (configName: microvm), so a - // missing or stale one fails loudly. The pool only selects the class. blocks["${WORKERPOOL_RUNTIME}"] = " sandboxClasses:\n - name: microvm" - // Must match the WorkerPool's: a snapshot is not portable across sandbox - // classes, so only same-class pools are eligible to run these actors. - blocks["${TEMPLATE_SANDBOX_CLASS}"] = " sandboxClass: microvm" return inline, blocks } diff --git a/internal/e2e/sandbox_test.go b/internal/e2e/sandbox_test.go index 6e545dd52a..ca78b69f97 100644 --- a/internal/e2e/sandbox_test.go +++ b/internal/e2e/sandbox_test.go @@ -122,8 +122,8 @@ func memoryLimit(tmpl *ateapipb.ActorTemplate) string { } // TestRenderSubstrateFixtures_GVisor pins the default rendering: every -// micro-VM block is gone, no placeholder survives, and the templates name the -// cluster-wide gvisor-default SandboxConfig. +// micro-VM block is gone, no placeholder survives, and the templates select +// their fixture's pool. func TestRenderSubstrateFixtures_GVisor(t *testing.T) { t.Setenv(sandboxClassEnv, "") for _, fixture := range substrateFixtures { @@ -142,14 +142,6 @@ func TestRenderSubstrateFixtures_GVisor(t *testing.T) { } for _, tmpl := range templates { name := tmpl.GetMetadata().GetName() - if got := tmpl.GetSandboxConfig().GetSandboxClass(); got != "gvisor" { - t.Errorf("template %s sandboxClass = %q, want gvisor", name, got) - } - // The templates name the cluster-wide gvisor-default - // SandboxConfig explicitly: config_name is required. - if got := tmpl.GetSandboxConfig().GetConfigName(); got != "gvisor-default" { - t.Errorf("template %s configName = %q, want gvisor-default", name, got) - } // An inline placeholder with an empty value must substitute, not // delete its line: the location is what the golden snapshot needs. location := tmpl.GetSnapshotConfig().GetStorageLocation() @@ -171,8 +163,7 @@ func TestRenderSubstrateFixtures_GVisor(t *testing.T) { } // TestRenderSubstrateFixtures_MicroVM pins the micro-VM rendering: the -// templates name the cluster-wide SandboxConfig and match the pool's class -// and carry limits, and the snapshots land under their own prefix. +// templates carry limits and the snapshots land under their own prefix. func TestRenderSubstrateFixtures_MicroVM(t *testing.T) { t.Setenv(sandboxClassEnv, SandboxClassMicroVM) for _, fixture := range substrateFixtures { @@ -191,14 +182,6 @@ func TestRenderSubstrateFixtures_MicroVM(t *testing.T) { } for _, tmpl := range templates { name := tmpl.GetMetadata().GetName() - if got := tmpl.GetSandboxConfig().GetSandboxClass(); got != SandboxClassMicroVM { - t.Errorf("template %s sandboxClass = %q, want microvm — it must match the pool's or no worker is eligible", name, got) - } - // Named explicitly (see fixture.go), so a missing or stale - // microvm install fails loudly. - if got := tmpl.GetSandboxConfig().GetConfigName(); got != "microvm" { - t.Errorf("template %s configName = %q, want microvm", name, got) - } // The API rejects a template without limits. if memoryLimit(tmpl) == "" { t.Errorf("template %s declares no memory limit", name) diff --git a/internal/e2e/suites/combinedvolumes/combinedvolumes_test.go b/internal/e2e/suites/combinedvolumes/combinedvolumes_test.go index c0cc7a39aa..d46373d338 100644 --- a/internal/e2e/suites/combinedvolumes/combinedvolumes_test.go +++ b/internal/e2e/suites/combinedvolumes/combinedvolumes_test.go @@ -404,7 +404,7 @@ func TestCombinedVolumes(t *testing.T) { }) t.Run("SurvivesSuspendResume", func(t *testing.T) { - if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef.ToObjectRef()}); err != nil { + if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef.ToObjectRef(), Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}); err != nil { t.Fatalf("SuspendActor: %v", err) } diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index 69ab16fb91..0aa113e8a8 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -174,7 +174,8 @@ func TestActorSnapshotLifecycle(t *testing.T) { _, _ = clients.SubstrateAPI.DeleteTag(context.Background(), &ateapipb.DeleteTagRequest{Tag: tagRef}) }) suspended, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: sourceName}, + Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: sourceName}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, }) if err != nil { t.Fatalf("failed to suspend source Actor: %v", err) @@ -306,62 +307,12 @@ func TestDurableDirLifecycle(t *testing.T) { wantMemoryAfterSuspend: 1, wantFileAfterSuspend: 3, wantRootfsAfterSuspend: 3, - microVMOnly: true, - }, - }, - { - // Suspend from paused uploads the MEMORY local snapshot. - name: "preferredFidelity:MEMORY, suspend from paused", - tc: actorLifecycleTestCase{ - fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, - wantMemoryAfterPause: 2, - wantFileAfterPause: 2, - wantRootfsAfterPause: 2, - wantMemoryAfterSuspend: 3, - wantFileAfterSuspend: 3, - wantRootfsAfterSuspend: 3, - wantSnapshotFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, - suspendWhilePaused: true, - }, - }, - { - // Suspend from paused uploads the VOLUMES local snapshot. - name: "preferredFidelity:VOLUMES, suspend from paused", - tc: actorLifecycleTestCase{ - fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, - wantMemoryAfterPause: 1, - wantFileAfterPause: 2, - wantRootfsAfterPause: 1, - wantMemoryAfterSuspend: 1, - wantFileAfterSuspend: 3, - wantRootfsAfterSuspend: 1, - wantSnapshotFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, - suspendWhilePaused: true, - }, - }, - { - // Suspend from paused uploads the ROOTFS local snapshot. - name: "preferredFidelity:ROOTFS, suspend from paused", - tc: actorLifecycleTestCase{ - fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, - wantMemoryAfterPause: 1, - wantFileAfterPause: 2, - wantRootfsAfterPause: 2, - wantMemoryAfterSuspend: 1, - wantFileAfterSuspend: 3, - wantRootfsAfterSuspend: 3, - wantSnapshotFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, - suspendWhilePaused: true, - microVMOnly: true, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - if test.tc.microVMOnly && !e2e.IsMicroVM() { - t.Skipf("Skipping %s: only the micro-VM runtime serves this fidelity", test.name) - } t.Parallel() runActorLifecycleTestCase(t, "durabledir-lifecycle", createActorTemplate, test.tc) }) @@ -369,9 +320,7 @@ func TestDurableDirLifecycle(t *testing.T) { } // TestMultipleDurableDirLifecycle covers an Actor with TWO durable-dir volumes: -// both must survive pause/resume and suspend/resume independently. Only the -// micro-VM runtime supports more than one — gVisor templates are still capped at -// one by the ActorTemplate CEL rules, so the template would be rejected there. +// both must survive pause/resume and suspend/resume independently. func TestMultipleDurableDirLifecycle(t *testing.T) { t.Parallel() tests := []struct { @@ -411,16 +360,12 @@ func TestMultipleDurableDirLifecycle(t *testing.T) { wantFileAfterSuspend: 3, wantRootfsAfterSuspend: 3, checkSecondFileCounter: true, - microVMOnly: true, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - if test.tc.microVMOnly && !e2e.IsMicroVM() { - t.Skipf("Skipping %s: micro-VM-only case", test.name) - } t.Parallel() runActorLifecycleTestCase(t, "multi-durabledir-lifecycle", createActorTemplateWithTwoDurableDirs, test.tc) }) @@ -571,7 +516,8 @@ func TestExternalVolume_NodeMigration(t *testing.T) { t.Logf("Suspending Actor %q...", actorName) if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorName}, + Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorName}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, }); err != nil { t.Fatalf("failed to suspend Actor: %v", err) } @@ -639,16 +585,6 @@ type actorLifecycleTestCase struct { // wantSnapshotFidelity, when set, asserts the content scope recorded // on the ActorSnapshot the suspend produced. wantSnapshotFidelity ateapipb.SnapshotFidelity - - // microVMOnly skips the case outside the micro-VM environment. - microVMOnly bool - - // suspendWhilePaused pauses the actor again before the suspend step, so - // the suspend runs from paused: it must upload the node-local pause - // snapshot instead of checkpointing a running workload. The counter - // expectations are unchanged — both origins capture the state after the - // second call. - suspendWhilePaused bool } func runActorLifecycleTestCase(t *testing.T, prefix string, createTemplate func(context.Context, *testing.T, *e2e.Clients, *e2e.Namespace, ateapipb.SnapshotFidelity) (*ateapipb.ActorTemplate, error), tc actorLifecycleTestCase) { @@ -715,8 +651,9 @@ func runActorLifecycleTestCase(t *testing.T, prefix string, createTemplate func( // Pausing the actor // t.Logf("Pausing Actor %q...", actorID) - paused, err := clients.SubstrateAPI.PauseActor(ctx, &ateapipb.PauseActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, + paused, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, }) if err != nil { t.Fatalf("failed to pause Actor: %v", err) @@ -755,18 +692,10 @@ func runActorLifecycleTestCase(t *testing.T, prefix string, createTemplate func( // // Suspending the actor // - if tc.suspendWhilePaused { - t.Logf("Pausing Actor %q before suspending...", actorID) - if _, err := clients.SubstrateAPI.PauseActor(ctx, &ateapipb.PauseActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, - }); err != nil { - t.Fatalf("failed to pause Actor before suspend: %v", err) - } - waitForActorPaused(ctx, t, clients, actorID) - } t.Logf("Suspending Actor %q...", actorID) suspended, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, + Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, }) if err != nil { t.Fatalf("failed to suspend Actor: %v", err) @@ -776,20 +705,6 @@ func runActorLifecycleTestCase(t *testing.T, prefix string, createTemplate func( t.Errorf("durable snapshot stamp after suspend mismatch (-assignment +snapshot):\n%s", diff) } - if tc.suspendWhilePaused { - // The suspend must end the node pinning: a suspended actor's snapshot - // lives in object storage, not on a node. - suspendedActor, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, - }) - if err != nil { - t.Fatalf("failed to get suspended Actor: %v", err) - } - if hasLocalSnapshot(suspendedActor.GetStatus()) { - t.Errorf("suspended Actor still carries LocalSnapshot: %v", suspendedActor.GetStatus().GetSnapshots()) - } - } - if tc.wantSnapshotFidelity != ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED { validateSnapshotFidelity(ctx, t, clients, actorID, tc.wantSnapshotFidelity) } @@ -1010,8 +925,9 @@ func pauseActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj * // Pausing the actor t.Logf("Pausing Actor %q...", actorName) - if _, err := clients.SubstrateAPI.PauseActor(ctx, &ateapipb.PauseActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorName}, + if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorName}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, }); err != nil { t.Fatalf("failed to pause Actor: %v", err) } @@ -1036,7 +952,8 @@ func pauseActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj * // Suspending the actor before deletion t.Logf("Suspending Actor %q before deletion...", actorName) if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorName}, + Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorName}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, }); err != nil { t.Fatalf("failed to suspend Actor: %v", err) } @@ -1091,7 +1008,8 @@ func suspendActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj // Suspending the actor t.Logf("Suspending Actor %q...", actorName) if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorName}, + Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorName}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, }); err != nil { t.Fatalf("failed to suspend Actor: %v", err) } @@ -1116,7 +1034,8 @@ func suspendActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj // Suspending the actor before deletion t.Logf("Suspending Actor %q before deletion...", actorName) if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorName}, + Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorName}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, }); err != nil { t.Fatalf("failed to suspend Actor: %v", err) } @@ -1167,7 +1086,7 @@ func revertActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj } validateCounterResponse(t, resp, "before snapshot", 1, 1) - suspended, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef}) + suspended, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}) if err != nil { t.Fatalf("failed to suspend Actor: %v", err) } @@ -1219,7 +1138,7 @@ func revertActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj // Revert from paused. Resume prefers a local snapshot over the external // one, so clearing LocalSnapshot is what prevents this pause's count=2 // checkpoint from restoring on the next resume. - if _, err := clients.SubstrateAPI.PauseActor(ctx, &ateapipb.PauseActorRequest{Actor: actorRef}); err != nil { + if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}}); err != nil { t.Fatalf("failed to pause Actor: %v", err) } waitForActorPaused(ctx, t, clients, actorName) @@ -1376,8 +1295,9 @@ func deletePausedActorAnyState(ctx context.Context, t *testing.T, clients *e2e.C // 3. Pausing the actor t.Logf("Pausing Actor %q...", actorName) - if _, err := clients.SubstrateAPI.PauseActor(ctx, &ateapipb.PauseActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorName}, + if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorName}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, }); err != nil { t.Fatalf("failed to pause Actor: %v", err) } @@ -1489,8 +1409,7 @@ const ( // createActorTemplateWithTwoDurableDirs builds a template with a SECOND // durable-dir volume alongside the one the demo already declares, and points the // counter's second file counter at it, so both volumes are written on every -// request. Only the micro-VM runtime accepts this: gVisor templates are still -// capped at one durable-dir volume by the ActorTemplate CEL rules. +// request. func createActorTemplateWithTwoDurableDirs(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, fidelity ateapipb.SnapshotFidelity) (*ateapipb.ActorTemplate, error) { modify := func(at *ateapipb.ActorTemplate) { for _, c := range at.GetContainers() { @@ -1892,7 +1811,7 @@ func TestRevertCrashedActor(t *testing.T) { } validateCounterResponse(t, resp, "before snapshot", 1, 1) - suspended, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef}) + suspended, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}) if err != nil { t.Fatalf("failed to suspend Actor: %v", err) } diff --git a/internal/e2e/suites/demo/golden_test.go b/internal/e2e/suites/demo/golden_test.go new file mode 100644 index 0000000000..23f1de0796 --- /dev/null +++ b/internal/e2e/suites/demo/golden_test.go @@ -0,0 +1,128 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package demo + +import ( + "context" + "testing" + "time" + + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestGoldenSnapshotLifecycle(t *testing.T) { + for _, fidelity := range []ateapipb.SnapshotFidelity{ + ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, + } { + t.Run(fidelity.String(), func(t *testing.T) { + ctx := t.Context() + clients := e2e.GetClients() + ns := e2e.CreateNamespace(t) + env, err := e2e.CheckEnv("BUCKET_NAME") + if err != nil { + t.Fatal(err) + } + // A distinct atespace catches placement in a system atespace and + // keeps cleanup independent of the suite's shared demo atespace. + atespace := "golden-" + ns.Name + t.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + _, _ = clients.SubstrateAPI.DeleteAtespace(cleanupCtx, &ateapipb.DeleteAtespaceRequest{Atespace: &ateapipb.ObjectRef{Name: atespace}}) + }) + created := e2e.CreateSubstrateCounterTemplate(ctx, t, clients, ns.Name, e2e.SubstrateTemplateOptions{ + Atespace: atespace, Name: "counter", PoolName: "counter", PoolReplicas: 1, + Labels: map[string]string{"golden-e2e": ns.Name}, + SnapshotConfig: &ateapipb.SnapshotConfig{ + StorageLocation: "gs://" + env["BUCKET_NAME"] + "/" + atespace, + PreferredFidelity: fidelity, + }, + }) + + templateRef := e2e.TemplateRef(created) + + tmpl, err := clients.SubstrateAPI.GetActorTemplate(ctx, &ateapipb.GetActorTemplateRequest{ActorTemplate: templateRef}) + if err != nil { + t.Fatal(err) + } + tagRef := tmpl.GetStatus().GetGoldenSnapshotStatus().GetGoldenTag() + if tagRef.GetAtespace() != atespace { + t.Fatalf("golden tag atespace = %q, want %q", tagRef.GetAtespace(), atespace) + } + if got := tagRef.GetName(); got != tmpl.GetMetadata().GetUid() { + t.Fatalf("golden tag name = %q, want template UID %q", got, tmpl.GetMetadata().GetUid()) + } + tag, err := clients.SubstrateAPI.GetTag(ctx, &ateapipb.GetTagRequest{Tag: tagRef}) + if err != nil { + t.Fatal(err) + } + if source := tag.GetSourceActor(); source.GetAtespace() != atespace || source.GetName() != tmpl.GetMetadata().GetUid() { + t.Fatalf("golden source actor = %v, want %s/%s", source, atespace, tmpl.GetMetadata().GetUid()) + } + if got := tag.GetStatus().GetSnapshot().GetFidelity(); got != fidelity { + t.Fatalf("golden fidelity = %v, want %v", got, fidelity) + } + uri, err := resources.ParseSnapshotURI(tagSnapshotURI(tag)) + if err != nil { + t.Fatal(err) + } + if !uri.OwnedBy(resources.TagSnapshotOwner(atespace, tag.GetMetadata().GetUid())) { + t.Fatalf("golden snapshot %s is not owned by its tenant tag", uri) + } + actorRef := &ateapipb.ObjectRef{Atespace: atespace, Name: "clone"} + t.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + _, _ = clients.SubstrateAPI.DeleteActor(cleanupCtx, &ateapipb.DeleteActorRequest{Actor: actorRef, AnyState: true}) + }) + actor, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: atespace, Name: actorRef.Name}, ActorTemplate: templateRef, + }}) + if err != nil { + t.Fatal(err) + } + if got := durableSnapshotURI(actor.GetStatus()); got != uri.String() { + t.Fatalf("clone snapshot = %q, want golden %q", got, uri) + } + if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { + t.Fatal(err) + } + response, err := callActor(t, resources.ActorRefFromObjectRef(actorRef)) + if err != nil { + t.Fatal(err) + } + validateCounterResponse(t, response, "golden restore", 1, 1) + // Delete the borrower before collecting the template's snapshot. + if _, err := clients.SubstrateAPI.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: actorRef, AnyState: true}); err != nil { + t.Fatal(err) + } + if _, err := clients.SubstrateAPI.DeleteActorTemplate(ctx, &ateapipb.DeleteActorTemplateRequest{ActorTemplate: templateRef}); err != nil { + t.Fatal(err) + } + if _, err := clients.SubstrateAPI.GetTag(ctx, &ateapipb.GetTagRequest{Tag: tagRef}); status.Code(err) != codes.NotFound { + t.Fatalf("golden tag after template deletion = %v, want NotFound", err) + } + if _, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{Actor: tagRef}); status.Code(err) != codes.NotFound { + t.Fatalf("golden actor after template deletion = %v, want NotFound", err) + } + t.Logf("golden lifecycle passed: fidelity=%v, snapshot=%s", fidelity, uri) + }) + } +} diff --git a/internal/e2e/suites/demo/termination_test.go b/internal/e2e/suites/demo/termination_test.go index e865ded0cc..326fbafbaf 100644 --- a/internal/e2e/suites/demo/termination_test.go +++ b/internal/e2e/suites/demo/termination_test.go @@ -215,7 +215,8 @@ func TestGracefulWorkerTerminationSuspend(t *testing.T) { // Suspend the actor. t.Logf("Suspending actor %q during termination", actorID) _, suspendErr := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, + Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, }) wantState := ateapipb.ActorState_ACTOR_STATE_SUSPENDED switch status.Code(suspendErr) { diff --git a/internal/e2e/suites/demo/updatetemplate_test.go b/internal/e2e/suites/demo/updatetemplate_test.go index 38332b6557..488792402b 100644 --- a/internal/e2e/suites/demo/updatetemplate_test.go +++ b/internal/e2e/suites/demo/updatetemplate_test.go @@ -53,9 +53,6 @@ func TestUpdateTemplateLifecycle(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - if test.fidelity == ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS && !e2e.IsMicroVM() { - t.Skipf("Skipping %s: only the micro-VM runtime serves this fidelity", test.name) - } t.Parallel() runUpdateTemplateTestCase(t, test.fidelity) }) @@ -134,7 +131,8 @@ func runUpdateTemplateTestCase(t *testing.T, fidelity ateapipb.SnapshotFidelity) // t.Logf("Suspending Actor %q...", actorID) if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, + Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, }); err != nil { t.Fatalf("failed to suspend Actor: %v", err) } @@ -205,8 +203,9 @@ func runUpdateTemplateTestCase(t *testing.T, fidelity ateapipb.SnapshotFidelity) // can only be updated while SUSPENDED) and preserves the in-memory counter // when preferredFidelity is MEMORY. t.Logf("Pausing Actor %q under template B...", actorID) - if _, err := clients.SubstrateAPI.PauseActor(ctx, &ateapipb.PauseActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, + if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, }); err != nil { t.Fatalf("failed to pause Actor under template B: %v", err) } @@ -306,7 +305,8 @@ func runUpdateTemplateTestCase(t *testing.T, fidelity ateapipb.SnapshotFidelity) // snapshot's own scope rather than being forced down to data-only. t.Logf("Suspending Actor %q again...", actorID) if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, + Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, }); err != nil { t.Fatalf("failed to suspend Actor again: %v", err) } diff --git a/internal/e2e/suites/identity/identity_test.go b/internal/e2e/suites/identity/identity_test.go index 8355109d20..aff525db91 100644 --- a/internal/e2e/suites/identity/identity_test.go +++ b/internal/e2e/suites/identity/identity_test.go @@ -17,9 +17,12 @@ package identity import ( "context" "encoding/json" + "errors" "fmt" "io" "net/http" + "net/url" + "slices" "testing" "time" @@ -149,6 +152,14 @@ func TestActorIdentity_AfterRestore_IsOwnID_NotGolden(t *testing.T) { t.Errorf("actor %q and %q both report uid %q — actors are not distinct", id, other, got.UID) } seenUIDs[got.UID] = id + + // Resolved from the /etc/hosts the runtime writes, not DNS: images + // commonly ship an empty one. + for host, want := range map[string]string{"localhost": "127.0.0.1", got.Hostname: "169.254.17.2"} { + if addrs, err := lookup(ctx, rc, id, host); err != nil || !slices.Contains(addrs, want) { + t.Errorf("actor %q: lookup %q = %v, %v; want %s", id, host, addrs, err, want) + } + } } // Live refresh: rotate the pool while both actors run and wait until each @@ -170,6 +181,14 @@ func TestActorIdentity_AfterRestore_IsOwnID_NotGolden(t *testing.T) { rotatedTrust := e2e.AddEgressCA(t, ctx, clients, "ate-e2e-probe-trust-rotated") id := ids[0] ref := &ateapipb.ObjectRef{Atespace: probeNamespace, Name: id} + // The actor may edit its /etc files, as under Docker or kubelet, and the + // edits outlive the suspend like any other rootfs write. + etcFiles := []string{"/etc/hosts", "/etc/resolv.conf"} + for _, path := range etcFiles { + if out, err := probeFile(ctx, rc, id, "/writefile", path); err != nil || out["error"] != "" { + t.Fatalf("actor %q: editing %s: %v %v", id, path, err, out["error"]) + } + } if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: ref}); err != nil { t.Fatalf("SuspendActor %q: %v", id, err) } @@ -192,6 +211,11 @@ func TestActorIdentity_AfterRestore_IsOwnID_NotGolden(t *testing.T) { if wantUID := seenUIDFor(t, seenUIDs, id); got.UID != wantUID { t.Errorf("after suspend/resume: /run/ate/actor-uid = %q, want %q (probe read error: %q)", got.UID, wantUID, got.Error) } + for _, path := range etcFiles { + if out, err := probeFile(ctx, rc, id, "/readfile", path); err != nil || out["content"] != "written by probe" { + t.Errorf("after suspend/resume: %s = %q, %v %v; want the actor's edit", path, out["content"], err, out["error"]) + } + } waitForTrust(t, ctx, rc, id, rotatedTrust, 10*time.Second) // The other actor never cycled: the second rotation must reach it live, @@ -290,6 +314,40 @@ func whoami(t *testing.T, ctx context.Context, rc *e2e.RouterClient, id string) } } +// lookup asks the probe to resolve host from inside the actor. +func lookup(ctx context.Context, rc *e2e.RouterClient, id, host string) ([]string, error) { + resp, err := rc.Get(ctx, resources.ActorRef{Atespace: probeNamespace, Name: id}, "/lookup?host="+url.QueryEscape(host)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + var out struct { + Addrs []string `json:"addrs"` + Error string `json:"error"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return nil, fmt.Errorf("decoding /lookup: %w", err) + } + if out.Error != "" { + return nil, errors.New(out.Error) + } + return out.Addrs, nil +} + +// probeFile runs the probe's /readfile or /writefile on path. +func probeFile(ctx context.Context, rc *e2e.RouterClient, id, op, path string) (map[string]string, error) { + resp, err := rc.Get(ctx, resources.ActorRef{Atespace: probeNamespace, Name: id}, op+"?path="+url.QueryEscape(path)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + var out map[string]string + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return nil, fmt.Errorf("decoding %s: %w", op, err) + } + return out, nil +} + // tryWhoami is whoami returning the error instead of failing the test. func tryWhoami(ctx context.Context, rc *e2e.RouterClient, id string) (whoamiResponse, error) { var out whoamiResponse diff --git a/internal/e2e/suites/metrics/metrics_test.go b/internal/e2e/suites/metrics/metrics_test.go index 590d19fef2..a4b401ebc8 100644 --- a/internal/e2e/suites/metrics/metrics_test.go +++ b/internal/e2e/suites/metrics/metrics_test.go @@ -50,7 +50,7 @@ func TestPlatformMetricsEmitted(t *testing.T) { if err != nil { t.Fatalf("ScrapeCollectorMetrics for the lifecycle baseline: %v", err) } - lifecycleBaseline := e2e.LifecycleEventCounts(baselineScrape) + lifecycleBaseline := e2e.LifecycleEventSeries(baselineScrape) // CreateActor requires the atespace to exist first; ignore AlreadyExists. _, _ = clients.SubstrateAPI.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{ @@ -123,7 +123,7 @@ func TestPlatformMetricsEmitted(t *testing.T) { if err != nil { t.Fatalf("checking route-duration metric: %v", err) } - missingStates = e2e.StatesNotAdvanced(lifecycleBaseline, e2e.LifecycleEventCounts(scrape), wantStates) + missingStates = e2e.StatesNotAdvanced(lifecycleBaseline, e2e.LifecycleEventSeries(scrape), wantStates) lifecycleSeen = len(missingStates) == 0 workerSeen = e2e.CollectorHasService(scrape, "ateworker-gvisor", "ateworker-microvm") // atecontroller bridges controller-runtime's Prometheus registry onto its OTLP diff --git a/internal/e2e/suites/networking/connectegress_test.go b/internal/e2e/suites/networking/connectegress_test.go new file mode 100644 index 0000000000..fe16af9b4c --- /dev/null +++ b/internal/e2e/suites/networking/connectegress_test.go @@ -0,0 +1,137 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package networking + +import ( + "context" + "encoding/json" + "net/http" + "testing" + + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/resources" +) + +// The only CONNECT the egress gateway tunnels is the one atunnel wraps the +// actor's traffic in, authenticated by the actor's certificate and addressed +// to the IP:port the actor dialed. A CONNECT the actor sends itself -- what a +// client configured with a forward proxy sends first, in the clear or inside +// TLS -- arrives as a request inside that tunnel, and the gateway refuses it +// (docs/egress-traffic.md): the request would otherwise be policed +// as its authority's host on the port the actor dialed, while the tunnel went +// to the port the authority names. +// +// Each test sends the CONNECT to two proxies. One is an origin the test +// deploys, which answers any CONNECT 200: the denial the actor sees cannot be +// its doing, so the gateway refused the CONNECT before it reached anything +// that would have accepted it. The other is the gateway's own Service, the +// address a client that knew about the gateway would configure. + +// egressConnectResponse mirrors the egress demo's /connect observation: the +// proxy's answer to the CONNECT. +type egressConnectResponse struct { + StatusCode int `json:"statusCode"` + Body string `json:"body"` + TLS bool `json:"tls"` +} + +// connectTunnelTarget is what the actor asks for a tunnel to. The gateway +// refuses the CONNECT without resolving it, so it need not exist. +const connectTunnelTarget = "example.com:443" + +// egressGatewayAddress is the egress gateway's CONNECT front door, the +// default-egress-gateway-address ate-api-server hands every atelet. +func egressGatewayAddress() string { + return "atenet-egress." + e2e.SystemNamespace() + ".svc:443" +} + +// connectProxySpec is testserver's http subcommand answering CONNECT the way a +// forward proxy does, published on port. +func connectProxySpec(port, tlsPort int) e2e.ServerPod { + return e2e.ServerPod{ + Name: "connect-proxy", + ImportPath: "go.ate.dev/substrate/internal/e2e/fixtures/testserver", + Args: []string{"http", "--accept-connect"}, + Port: port, + TargetPort: 8080, + TLSPort: tlsPort, + } +} + +// TestActorEgressConnect: the actor sends a cleartext CONNECT, as a client +// configured with an http:// proxy does. The proxy listens on 3128, a +// conventional proxy port and one that is neither 80 nor 443, so the refusal +// is shown not to depend on the port the actor dialed. +func TestActorEgressConnect(t *testing.T) { + ctx := t.Context() + _, proxy, _ := prepareProtocolOrigin(t, ctx, connectProxySpec(3128, 0), false) + _, router, actorRef := prepareProtocolActor(t, ctx, "connect") + + t.Run("to a proxy", func(t *testing.T) { + assertActorConnectRefused(t, ctx, router, actorRef, proxy, false) + }) + t.Run("to the gateway", func(t *testing.T) { + assertActorConnectRefused(t, ctx, router, actorRef, egressGatewayAddress(), false) + }) +} + +// TestActorEgressSecureConnect: the actor sends its CONNECT inside TLS, as a +// client configured with an https:// proxy does. The gateway terminates that +// TLS under the actor's https rule, so the CONNECT is read and refused on the +// decrypted leg, and the actor sees the denial over a connection it verified. +func TestActorEgressSecureConnect(t *testing.T) { + ctx := t.Context() + _, proxy, _ := prepareProtocolOrigin(t, ctx, connectProxySpec(80, 443), true) + _, router, actorRef := prepareProtocolActor(t, ctx, "connect-tls") + + t.Run("to a proxy", func(t *testing.T) { + assertActorConnectRefused(t, ctx, router, actorRef, proxy, true) + }) + t.Run("to the gateway", func(t *testing.T) { + assertActorConnectRefused(t, ctx, router, actorRef, egressGatewayAddress(), true) + }) +} + +// assertActorConnectRefused has the actor send proxy a CONNECT, over TLS when +// encrypted, and checks the dataplane's explicit denial. The actor's answer is +// retried until it has one: a failed TLS handshake is reported as a 502 by the +// demo, and the first handshake with a minted leaf can fail until kubelet has +// propagated the gateway's CA to the actor. +func assertActorConnectRefused(t *testing.T, ctx context.Context, router *e2e.RouterClient, actorRef resources.ActorRef, proxy string, encrypted bool) { + t.Helper() + payload, err := json.Marshal(map[string]any{"proxy": proxy, "target": connectTunnelTarget, "tls": encrypted}) + if err != nil { + t.Fatalf("marshaling the CONNECT request: %v", err) + } + status, raw := postThroughEgressActorUntil(t, ctx, router, actorRef, "/connect", payload, reached) + if status != http.StatusOK { + t.Fatalf("the actor got no answer to its CONNECT %s via %s: HTTP %d; body: %s", connectTunnelTarget, proxy, status, raw) + } + var got egressConnectResponse + if err := json.Unmarshal(raw, &got); err != nil { + t.Fatalf("decoding the CONNECT observation: %v (body %q)", err, raw) + } + if got.StatusCode/100 == 2 { + t.Fatalf("CONNECT %s via %s was answered %d: the gateway opened a tunnel for the actor's own CONNECT: %+v", connectTunnelTarget, proxy, got.StatusCode, got) + } + wantStatus, wantBody := e2e.CurrentAtenetDataplane().EgressConnectDeniedResponse() + if got.StatusCode != wantStatus || got.Body != wantBody { + t.Fatalf("CONNECT %s via %s was answered %d %q, want %d %q: %+v", connectTunnelTarget, proxy, got.StatusCode, got.Body, wantStatus, wantBody, got) + } + if got.TLS != encrypted { + t.Errorf("CONNECT %s via %s went over verified TLS = %t, want %t: %+v", connectTunnelTarget, proxy, got.TLS, encrypted, got) + } + t.Logf("the gateway refused the actor's CONNECT %s via %s: %+v", connectTunnelTarget, proxy, got) +} diff --git a/internal/e2e/suites/networking/egresspolicy_test.go b/internal/e2e/suites/networking/egresspolicy_test.go index e3ee1216ce..55eccd663d 100644 --- a/internal/e2e/suites/networking/egresspolicy_test.go +++ b/internal/e2e/suites/networking/egresspolicy_test.go @@ -304,7 +304,7 @@ func TestActorEgressPendingPlaintextHeadersDoNotBlockMITM(t *testing.T) { } } -// TestActorDNSRequiresPolicy: the sandbox's DNS relay refuses the queries of an +// TestActorDNSRequiresPolicy: the sandbox's DNS relay denies the queries of an // actor with no EgressPolicy or for a host outside its policy, and answers them // once a rule in the policy matches the host, without a restart. func TestActorDNSRequiresPolicy(t *testing.T) { @@ -321,10 +321,10 @@ func TestActorDNSRequiresPolicy(t *testing.T) { waitForActorRoute(t, ctx, router, actorRef) status, body := fetchThroughEgressActorUntil(t, ctx, router, actorRef, url, notTransient) - if status != http.StatusBadGateway || !isRefusedLookup(string(body), name) { - t.Fatalf("fetch of %s by an actor with no policy returned HTTP %d, want its lookup refused; body: %s", url, status, body) + if status != http.StatusBadGateway || !isDeniedLookup(string(body), name) { + t.Fatalf("fetch of %s by an actor with no policy returned HTTP %d, want its lookup denied; body: %s", url, status, body) } - t.Logf("lookup was refused as expected: %s", body) + t.Logf("lookup was denied as expected: %s", body) // The relay fetches the policy again within seconds, so the retries below // see it. A wildcard rule covering the target namespace's Services allows @@ -338,81 +338,16 @@ func TestActorDNSRequiresPolicy(t *testing.T) { const unlisted = "unlisted.example.com" status, body = fetchThroughEgressActorUntil(t, ctx, router, actorRef, "http://"+unlisted+"/healthz", notTransient) - if status != http.StatusBadGateway || !isRefusedLookup(string(body), unlisted) { - t.Fatalf("fetch of unlisted host %s returned HTTP %d, want its lookup refused; body: %s", unlisted, status, body) + if status != http.StatusBadGateway || !isDeniedLookup(string(body), unlisted) { + t.Fatalf("fetch of unlisted host %s returned HTTP %d, want its lookup denied; body: %s", unlisted, status, body) } } -// isRefusedLookup reports whether body is the egress demo's report of a lookup -// of name that the resolver refused. Go's resolver reports a REFUSED answer as -// "server misbehaving". -func isRefusedLookup(body, name string) bool { - return strings.Contains(body, "lookup "+name) && strings.Contains(body, "server misbehaving") -} - -// hostnamePolicyActor creates an actor whose policy allows HTTPS to -// example.com, TLS passthrough to example.edu, and cleartext HTTP to -// example.org (so example.org resolves in DNS while still being denied over -// HTTPS on 443), and waits until it is routable. -func hostnamePolicyActor(t *testing.T, ctx context.Context) (*e2e.RouterClient, resources.ActorRef) { - t.Helper() - _, actorName, _ := createAndResumeActorWithEgress(t, ctx, "egress-sni", e2e.EgressFixture(), - e2e.EgressAllowHTTPS("example.com"), - e2e.EgressAllowPassthrough("example.edu"), - e2e.EgressAllowHTTP("example.org"), - ) - router := mustRouterClient(t, ctx) - t.Cleanup(func() { router.Close() }) - actorRef := resources.ActorRef{Atespace: networkingAtespace, Name: actorName} - waitForActorRoute(t, ctx, router, actorRef) - return router, actorRef -} - -func isMitmCert(body string) bool { - // example.* domains use certs signed by Google Trust Services - // If this substring is not present it means dataplane used a minted cert - return !strings.Contains(body, "O=Google Trust Services") -} - -// TestActorEgressHTTPSByHostnameMITM: the gateway terminates the TLS and decides -// each request by name: example.com 200, example.org 403. -func TestActorEgressHTTPSByHostnameMITM(t *testing.T) { - ctx := context.Background() - dataplane := e2e.CurrentAtenetDataplane() - router, actorRef := hostnamePolicyActor(t, ctx) - - status, body := fetchThroughEgressActorUntil(t, ctx, router, actorRef, "https://example.com/", reached) - if status != http.StatusOK { - t.Fatalf("fetch of the allowed host returned HTTP %d, want 200; body: %s", status, body) - } - if !isMitmCert(string(body)) { - t.Fatalf("request did not use minted cert; body: %s", body) - } - status, body = fetchThroughEgressActorUntil(t, ctx, router, actorRef, "https://example.org/", notTransient) - if !dataplane.IsEgressPolicyDenied(status, string(body)) { - t.Fatalf("fetch of a host outside the policy returned HTTP %d, want an egress-policy denial; body: %s", status, body) - } - t.Logf("denied on the decrypted request: %s", body) -} - -// TestActorEgressHTTPSByHostnamePassthrough: the gateway acts as TCP proxy fetching -// allowed SNI. -func TestActorEgressHTTPSByHostnamePassthrough(t *testing.T) { - ctx := context.Background() - dataplane := e2e.CurrentAtenetDataplane() - router, actorRef := hostnamePolicyActor(t, ctx) - - status, body := fetchThroughEgressActorUntil(t, ctx, router, actorRef, "https://example.edu/", reached) - if status != http.StatusOK { - t.Fatalf("fetch of the allowed host returned HTTP %d, want 200; body: %s", status, body) - } - if isMitmCert(string(body)) { - t.Fatalf("request used minted cert; body: %s", body) - } - status, body = fetchThroughEgressActorUntil(t, ctx, router, actorRef, "https://example.org/", notTransient) - if !dataplane.IsEgressPolicyDenied(status, string(body)) { - t.Fatalf("fetch of a host outside the policy returned HTTP %d, want an egress-policy denial; body: %s", status, body) - } +// isDeniedLookup reports whether body is the egress demo's report of a lookup +// of name that the DNS relay denied. The relay answers NXDOMAIN, which Go's +// resolver reports as "no such host". +func isDeniedLookup(body, name string) bool { + return strings.Contains(body, "lookup "+name) && strings.Contains(body, "no such host") } // fetchThroughEgressActorUntil is fetchThroughEgressActor with the caller diff --git a/internal/e2e/suites/security/crossatespace_test.go b/internal/e2e/suites/security/crossatespace_test.go index 5d5e1ca58f..bca1f30b83 100644 --- a/internal/e2e/suites/security/crossatespace_test.go +++ b/internal/e2e/suites/security/crossatespace_test.go @@ -220,7 +220,8 @@ func testTagScopeBoundary(t *testing.T, ctx context.Context, clients *e2e.Client t.Helper() suspended, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: ns.a, Name: source}, + Actor: &ateapipb.ObjectRef{Atespace: ns.a, Name: source}, + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, }) if err != nil { t.Fatalf("SuspendActor %s/%s: %v", ns.a, source, err) diff --git a/internal/e2e/template.go b/internal/e2e/template.go index 102d49ce5e..f76153291c 100644 --- a/internal/e2e/template.go +++ b/internal/e2e/template.go @@ -60,8 +60,8 @@ func CreateSubstrateCounterTemplate(ctx context.Context, t *testing.T, clients * } // CreateSubstrateTemplateFrom creates a per-test WorkerPool CRD plus a -// substrate ActorTemplate copying the resolved runtime (sandbox config, worker -// image, container images, sandbox size) from the installed fixture src. It +// substrate ActorTemplate copying the resolved runtime (worker image, +// container images, sandbox size) from the installed fixture src. It // registers cleanup of the template (which does not ride the k8s namespace GC // the CRD templates did) and blocks until the golden snapshot exists. func CreateSubstrateTemplateFrom(ctx context.Context, t *testing.T, clients *Clients, namespace string, src SubstrateFixture, opts SubstrateTemplateOptions) *ateapipb.ActorTemplate { @@ -108,10 +108,7 @@ func CreateSubstrateTemplateFrom(ctx context.Context, t *testing.T, clients *Cli WorkerSelector: &ateapipb.Selector{MatchLabels: opts.Labels}, Containers: srcTmpl.GetContainers(), // The source's limits size the sandbox, and the API requires them. - Resources: srcTmpl.GetResources(), - // The source carries the sandbox_class/config_name pair for the - // class under test. - SandboxConfig: srcTmpl.GetSandboxConfig(), + Resources: srcTmpl.GetResources(), SnapshotConfig: snapshots, Volumes: srcTmpl.GetVolumes(), } diff --git a/internal/hardware/hardware.go b/internal/hardware/hardware.go index 4511b90707..ff858cf033 100644 --- a/internal/hardware/hardware.go +++ b/internal/hardware/hardware.go @@ -30,10 +30,10 @@ const ( // AttrArchitecture is the CPU architecture ("amd64", "arm64", etc.). AttrArchitecture = "architecture" - // AttrSandboxRuntimeHash identifies the sandbox runtime binaries: a + // AttrSandboxBundleHash identifies the sandbox runtime bundle: a // sha256 over a SandboxConfig version's asset digests and pause image // digest, computed by atelet. - AttrSandboxRuntimeHash = "sandbox_runtime_hash" + AttrSandboxBundleHash = "sandbox_bundle_hash" // AttrCPUFeatures is the set of CPU features `runsc cpu-features` reports // for a gVisor runtime, encoded by EncodeCPUFeatures as a hex bitmask over diff --git a/internal/hardware/hardware_test.go b/internal/hardware/hardware_test.go index ee50f280dc..ace1cf7144 100644 --- a/internal/hardware/hardware_test.go +++ b/internal/hardware/hardware_test.go @@ -65,7 +65,7 @@ func TestMatches(t *testing.T) { amd64GVisorExtraAttr := runtimeWith("gvisor", "v1", &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "amd64"}, &ateapipb.AttributeEntry{Key: AttrCPUFeatures, Value: "7"}, - &ateapipb.AttributeEntry{Key: AttrSandboxRuntimeHash, Value: "sha256:abc"}, + &ateapipb.AttributeEntry{Key: AttrSandboxBundleHash, Value: "sha256:abc"}, ) tests := []struct { @@ -100,7 +100,7 @@ func TestMatchesCompat(t *testing.T) { } amd64 := &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "amd64"} arm64 := &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "arm64"} - hash := &ateapipb.AttributeEntry{Key: AttrSandboxRuntimeHash, Value: "sha256:abc"} + hash := &ateapipb.AttributeEntry{Key: AttrSandboxBundleHash, Value: "sha256:abc"} cpuBits := func(val string) *ateapipb.AttributeEntry { return &ateapipb.AttributeEntry{Key: AttrCPUFeatures, Value: val} } @@ -214,7 +214,7 @@ func TestMatchingRuntimeAndWorkerMatches(t *testing.T) { rt := func(class, arch, hash string) *ateapipb.SandboxRuntime { attrs := []*ateapipb.AttributeEntry{{Key: AttrArchitecture, Value: arch}} if hash != "" { - attrs = append(attrs, &ateapipb.AttributeEntry{Key: "gvisor_asset_hash", Value: hash}) + attrs = append(attrs, &ateapipb.AttributeEntry{Key: AttrSandboxBundleHash, Value: hash}) } return &ateapipb.SandboxRuntime{ SandboxClass: class, diff --git a/internal/installdefaults/installdefaults.go b/internal/installdefaults/installdefaults.go index 3ebe646191..fa8f7098d7 100644 --- a/internal/installdefaults/installdefaults.go +++ b/internal/installdefaults/installdefaults.go @@ -50,6 +50,7 @@ const ( RouterServiceAccount = "atenet-router" EgressServiceAccount = "atenet-egress" ControllerServiceAccount = "ate-controller" + APIServerServiceAccount = "ate-api-server" // WorkerServiceAccount is what worker pods run as when their WorkerPool // names none; atecontroller creates it in the pool's namespace. @@ -117,3 +118,9 @@ func EgressSPIFFEID(namespace string) string { func ControllerSPIFFEID(namespace string) string { return SPIFFEID(namespace, ControllerServiceAccount) } + +// APIServerSPIFFEID returns the SPIFFE ID the ate-api-server pod presents when +// it runs in namespace. Its objectstore-plugin sidecar shares it. +func APIServerSPIFFEID(namespace string) string { + return SPIFFEID(namespace, APIServerServiceAccount) +} diff --git a/internal/installdefaults/installdefaults_test.go b/internal/installdefaults/installdefaults_test.go index e15cc7f128..4c29998cff 100644 --- a/internal/installdefaults/installdefaults_test.go +++ b/internal/installdefaults/installdefaults_test.go @@ -79,6 +79,15 @@ func TestControllerSPIFFEID(t *testing.T) { } } +func TestAPIServerSPIFFEID(t *testing.T) { + // Matches the --api-server-sidecar-spiffe-id default ate-api-server ships + // with, the identity its pod presents on the loopback. + const want = "spiffe://cluster.local/ns/ate-system/sa/ate-api-server" + if got := APIServerSPIFFEID(SystemNamespace); got != want { + t.Errorf("APIServerSPIFFEID(%q) = %q, want %q", SystemNamespace, got, want) + } +} + func TestActorJWTIssuer(t *testing.T) { tests := []struct { name string diff --git a/internal/ocispec/gvisor.go b/internal/ocispec/gvisor.go index cde27bccc8..c532a0c67f 100644 --- a/internal/ocispec/gvisor.go +++ b/internal/ocispec/gvisor.go @@ -16,7 +16,6 @@ package ocispec import ( "path/filepath" - "slices" "github.com/opencontainers/runtime-spec/specs-go" "go.ate.dev/substrate/internal/proto/ateworkerpb" @@ -28,9 +27,6 @@ import ( // name is drawn from, so no actor container can collide with it. const PauseContainer = "_pause" -// resolvConf is the sandbox resolver path and default bind source. -const resolvConf = "/etc/resolv.conf" - // GVisorOptions describes the gVisor-specific context of one actor container. type GVisorOptions struct { ActorUID string @@ -38,8 +34,6 @@ type GVisorOptions struct { ContainerName string // DurableVolumes are declared on the sandbox (pause) spec only. DurableVolumes []string - // ResolvConf is the bind source for /etc/resolv.conf; empty uses the pod's file. - ResolvConf string // Size sizes the container's cgroup leaf. Only gVisor applies it; a micro-VM // container's limits come from its own declared resources (see sizing). Size sizing.SandboxSize @@ -50,8 +44,8 @@ func GVisorCgroupLeaf(actorUID, containerName string) string { return actorUID + "-" + containerName } -// ShapeGVisor adds runsc CRI annotations, durable-dir mount hints, host -// resolv.conf, and per-container cgroups to the spec. It is idempotent. +// ShapeGVisor adds runsc CRI annotations, durable-dir mount hints, and +// per-container cgroups to the spec. It is idempotent. func ShapeGVisor(spec *specs.Spec, o GVisorOptions) { if spec.Annotations == nil { spec.Annotations = make(map[string]string) @@ -69,24 +63,6 @@ func ShapeGVisor(spec *specs.Spec, o GVisorOptions) { spec.Annotations["io.kubernetes.cri.sandbox-id"] = PauseContainer } - // Insert resolv.conf before any volume bind mounts. - if !slices.ContainsFunc(spec.Mounts, func(m specs.Mount) bool { return m.Destination == resolvConf }) { - i := slices.IndexFunc(spec.Mounts, func(m specs.Mount) bool { return m.Type == "bind" }) - if i < 0 { - i = len(spec.Mounts) - } - source := o.ResolvConf - if source == "" { - source = resolvConf - } - spec.Mounts = slices.Insert(spec.Mounts, i, specs.Mount{ - Destination: resolvConf, - Type: "bind", - Source: source, - Options: []string{"ro"}, - }) - } - if spec.Linux == nil { spec.Linux = &specs.Linux{} } diff --git a/internal/ocispec/gvisor_test.go b/internal/ocispec/gvisor_test.go index 84d0ec04e8..6d0ff33566 100644 --- a/internal/ocispec/gvisor_test.go +++ b/internal/ocispec/gvisor_test.go @@ -53,34 +53,3 @@ func TestGVisorCgroupLeafMatchesTheShapedPath(t *testing.T) { t.Errorf("shaped cgroupsPath = %q, want %q", spec.Linux.CgroupsPath, want) } } - -func TestShapeGVisorBindsTheNamedResolvConf(t *testing.T) { - for _, tc := range []struct { - name string - resolvConf string - wantSource string - }{ - {name: "default is the worker pod's", wantSource: "/etc/resolv.conf"}, - { - name: "a sandbox may name its own", - resolvConf: "/var/lib/ateworker-gvisor/actors/a/resolv.conf", - wantSource: "/var/lib/ateworker-gvisor/actors/a/resolv.conf", - }, - } { - t.Run(tc.name, func(t *testing.T) { - spec := &specs.Spec{} - ShapeGVisor(spec, GVisorOptions{ - ActorUID: "a", ContainerName: "app", ResolvConf: tc.resolvConf, - }) - var got string - for _, m := range spec.Mounts { - if m.Destination == "/etc/resolv.conf" { - got = m.Source - } - } - if got != tc.wantSource { - t.Errorf("resolv.conf bound from %q, want %q", got, tc.wantSource) - } - }) - } -} diff --git a/internal/ocispec/ocispec.go b/internal/ocispec/ocispec.go index 5a2d491e10..2557d43594 100644 --- a/internal/ocispec/ocispec.go +++ b/internal/ocispec/ocispec.go @@ -32,8 +32,8 @@ import ( // specFile is the OCI spec file name within a bundle. const specFile = "config.json" -// hostname is the UTS hostname for actor containers. -const hostname = "actor" +// Hostname is the UTS hostname for actor containers. +const Hostname = "actor" // Options describes one actor container. type Options struct { @@ -112,7 +112,7 @@ func Build(o Options) *specs.Spec { Path: "rootfs", Readonly: false, }, - Hostname: hostname, + Hostname: Hostname, Mounts: []specs.Mount{ { Destination: "/proc", diff --git a/internal/ocispec/parity_test.go b/internal/ocispec/parity_test.go index 5e6d5d30f6..2f7f9d9d92 100644 --- a/internal/ocispec/parity_test.go +++ b/internal/ocispec/parity_test.go @@ -16,7 +16,6 @@ package ocispec import ( "encoding/json" - "slices" "strings" "testing" @@ -138,18 +137,6 @@ func TestShapeMicroVM_UnstagedSourceIsAnError(t *testing.T) { } } -// ShapeGVisor inserts resolv.conf ahead of the volume bind mounts. -func TestShapeGVisor_ResolvConfPrecedesVolumes(t *testing.T) { - spec := Build(parityOptions) - ShapeGVisor(spec, GVisorOptions{ActorUID: testActorUID, ContainerName: "app", Size: paritySize}) - at := func(dest string) int { - return slices.IndexFunc(spec.Mounts, func(m specs.Mount) bool { return m.Destination == dest }) - } - if got, want := at(resolvConf), at("/var/data"); got > want { - t.Errorf("%s is at index %d, after the first volume bind at %d", resolvConf, got, want) - } -} - // Shaping a spec twice does not accumulate mounts. func TestShapeGVisor_Idempotent(t *testing.T) { spec := Build(parityOptions) diff --git a/internal/oidcdiscovery/issuer.go b/internal/oidcdiscovery/issuer.go deleted file mode 100644 index 4652f72f78..0000000000 --- a/internal/oidcdiscovery/issuer.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package oidcdiscovery implements the issuer side of OpenID Connect Discovery -// for actor JWTs. -package oidcdiscovery - -import ( - "fmt" - "net/url" - "strings" -) - -// ValidateIssuer returns an error unless issuer is usable as an OpenID issuer -// identifier: an https URL with a host and no user info, query, or fragment. A -// path is allowed. -func ValidateIssuer(issuer string) error { - u, err := url.Parse(issuer) - if err != nil { - return fmt.Errorf("invalid issuer: %w", err) - } - switch { - case u.Scheme != "https": - return fmt.Errorf("issuer %q must use the https scheme", issuer) - case u.Hostname() == "": - return fmt.Errorf("issuer %q has no host", issuer) - case u.User != nil: - return fmt.Errorf("issuer %q must not contain user info", issuer) - case strings.Contains(issuer, "?"): - return fmt.Errorf("issuer %q must not contain a query", issuer) - case strings.Contains(issuer, "#"): - return fmt.Errorf("issuer %q must not contain a fragment", issuer) - } - return nil -} diff --git a/internal/oidcdiscovery/issuer_test.go b/internal/oidcdiscovery/issuer_test.go deleted file mode 100644 index 7d23047c97..0000000000 --- a/internal/oidcdiscovery/issuer_test.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package oidcdiscovery - -import "testing" - -func TestValidateIssuer(t *testing.T) { - tests := []struct { - name string - issuer string - wantErr bool - }{ - {name: "in-cluster default", issuer: "https://idp.ate-system.svc"}, - {name: "public host", issuer: "https://idp.example.com"}, - {name: "path", issuer: "https://idp.example.com/clusters/prod"}, - {name: "port", issuer: "https://idp.example.com:8443"}, - {name: "trailing slash", issuer: "https://idp.example.com/"}, - {name: "path with trailing slash", issuer: "https://idp.example.com/prod/"}, - - {name: "empty", issuer: "", wantErr: true}, - {name: "http", issuer: "http://idp.example.com", wantErr: true}, - {name: "no scheme", issuer: "idp.example.com", wantErr: true}, - {name: "no host", issuer: "https:///prod", wantErr: true}, - {name: "port without host", issuer: "https://:443", wantErr: true}, - {name: "opaque", issuer: "https:idp.example.com", wantErr: true}, - {name: "user info", issuer: "https://user:pass@idp.example.com", wantErr: true}, - {name: "query", issuer: "https://idp.example.com?cluster=prod", wantErr: true}, - {name: "empty query", issuer: "https://idp.example.com?", wantErr: true}, - {name: "fragment", issuer: "https://idp.example.com#prod", wantErr: true}, - {name: "empty fragment", issuer: "https://idp.example.com#", wantErr: true}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := ValidateIssuer(tt.issuer) - if tt.wantErr && err == nil { - t.Errorf("ValidateIssuer(%q) returned nil, want error", tt.issuer) - } - if !tt.wantErr && err != nil { - t.Errorf("ValidateIssuer(%q) returned error: %v", tt.issuer, err) - } - }) - } -} diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index 91554e0af9..5f085380aa 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -100,8 +100,7 @@ const ( // (currently DurableDir-typed volumes). Memory and the rest of rootfs are // excluded. SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES SnapshotFidelity = 1 - // Volumes plus the root filesystem changes made since boot. Served by the - // micro-VM runtime only; other runtimes reject requests carrying it. + // Volumes plus the root filesystem changes made since boot. SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS SnapshotFidelity = 2 // Capture process memory plus the full filesystem delta on top of the OCI // image (including any attached DurableDir volumes). @@ -156,20 +155,25 @@ type UploadMode int32 const ( UploadMode_UPLOAD_MODE_UNSPECIFIED UploadMode = 0 - // Keep the snapshot locally and upload it on atelet's schedule: at most - // for atelet's upload-delay after the oldest snapshot of the actor - // that is not yet durable was taken. Snapshots superseded before then - // are never uploaded. + // Keep the snapshot locally and upload it on atelet's schedule: about + // atelet's upload-delay after the oldest snapshot of the actor that is not + // yet durable was taken, or later while an upload is in progress or + // failing. Snapshots superseded before then are never uploaded. // - // Once the actor has stayed suspended on this node for - // atelet's detach-delay, atelet detaches it (deletes all the local snapshots). + // Once the actor has stayed suspended on this node for atelet's + // detach-delay, atelet detaches it: it uploads the latest local snapshot + // unless that is durable already, then deletes all the local snapshots. // - // TODO: Temporarily the deferred upload and detach are unimplemented. Only a local - // copy is taken. - UploadMode_UPLOAD_MODE_DEFERRED UploadMode = 1 - // Upload the snapshot synchronously before Checkpoint returns, while keeping the local copy. + // atelet reports the actor's state with + // WorkerService.ReportActorSnapshotState once such an upload makes a newer + // snapshot durable, and once such a detach has deleted the local snapshots. // - // TODO: Temporarily this deletes the local snapshot after the upload. + // atelet never uploads or detaches a snapshot it cannot make durable, such + // as one without `snapshot.object`. + UploadMode_UPLOAD_MODE_DEFERRED UploadMode = 1 + // Upload the snapshot synchronously before Checkpoint returns, and keep a + // local copy if atelet can: failing to store it does not fail the + // Checkpoint. UploadMode_UPLOAD_MODE_IMMEDIATE UploadMode = 2 // Keep this in sync with the maximums on fields of this type. ) @@ -366,7 +370,7 @@ type VersionedSandboxCompat struct { // +k8s:maxLength=64 SchemaVersion string `protobuf:"bytes,1,opt,name=schema_version,json=schemaVersion,proto3" json:"schema_version,omitempty"` // Key-value compatibility attributes defined by this schema version, covering - // both runtime assets (e.g. "sandbox_runtime_hash") and effective guest + // both runtime assets (e.g. "sandbox_bundle_hash") and effective guest // hardware features after masking (e.g. "architecture", "cpu_features"). // Matched against a snapshot under the same schema_version: every attribute // on the snapshot must be present on the worker with an equal value (except @@ -433,7 +437,7 @@ func (x *VersionedSandboxCompat) GetAttributes() []*AttributeEntry { // AttributeEntry is a single key-value compatibility attribute. type AttributeEntry struct { state protoimpl.MessageState `protogen:"open.v1"` - // Attribute name (e.g. "architecture", "cpu_features", "sandbox_runtime_hash"). + // Attribute name (e.g. "architecture", "cpu_features", "sandbox_bundle_hash"). // // +k8s:required // +k8s:maxLength=128 @@ -1504,9 +1508,18 @@ type RunRequest struct { CpuMilli int64 `protobuf:"varint,10,opt,name=cpu_milli,json=cpuMilli,proto3" json:"cpu_milli,omitempty"` // CPU limit in millicores (1000 = one core). // +k8s:optional // +k8s:minimum=0 - MemoryBytes int64 `protobuf:"varint,11,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` // Memory limit in bytes. - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + MemoryBytes int64 `protobuf:"varint,11,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` // Memory limit in bytes. + // The runtime the assigned worker runs this actor under, from + // WorkerAssignment.sandbox_runtime. + // + // TODO: atelet still runs with sandbox_assets; a follow-up resolves the + // SandboxConfig version from this field, makes it required, and removes + // sandbox_assets. + // + // +k8s:optional + SandboxRuntime *SandboxRuntime `protobuf:"bytes,12,opt,name=sandbox_runtime,json=sandboxRuntime,proto3" json:"sandbox_runtime,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RunRequest) Reset() { @@ -1616,6 +1629,13 @@ func (x *RunRequest) GetMemoryBytes() int64 { return 0 } +func (x *RunRequest) GetSandboxRuntime() *SandboxRuntime { + if x != nil { + return x.SandboxRuntime + } + return nil +} + // EgressGateway configures tunneled egress for one actor activation. type EgressGateway struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1787,7 +1807,7 @@ func (x *ArchAssets) GetFiles() map[string]*AssetFile { // an actor runs in: a sandbox class, the pause image holding its namespaces, // and binaries keyed first by architecture (GOARCH) and then by asset name. // atelet's backend code interprets the asset names (gVisor expects "gvisor", -// the release tarball; legacy "runsc", a bare binary, is still accepted). +// the release tarball). type SandboxAssets struct { state protoimpl.MessageState `protogen:"open.v1"` // +k8s:required @@ -3137,8 +3157,6 @@ type Snapshot struct { // for the actor, or for its newest under another snapshot_uuid: the // caller's view is stale. // - // TODO: Temporarily, atelet does not compare it with the snapshots it holds. - // // +k8s:required // +k8s:minimum=1 Generation int32 `protobuf:"varint,2,opt,name=generation,proto3" json:"generation,omitempty"` @@ -3296,26 +3314,27 @@ func (x *SnapshotReport) GetDurableCompleteTime() *timestamppb.Timestamp { // Returned by Checkpoint, PromoteSnapshot, DetachActor. type ActorSnapshotState struct { state protoimpl.MessageState `protogen:"open.v1"` - // The snapshot atelet restores the actor from locally. Unset when the - // latest checkpoint kept no local copy, and once a detach has made the - // snapshot durable and begun deleting it. Either way, latest_durable is - // then the actor's latest snapshot. + // The snapshot atelet restores the actor from locally. Unset when an + // UPLOAD_MODE_IMMEDIATE checkpoint could not keep a local copy, and once a + // detach has made the snapshot durable and begun deleting it. Either way, + // latest_durable is then the actor's latest snapshot. LatestLocal *SnapshotReport `protobuf:"bytes,1,opt,name=latest_local,json=latestLocal,proto3" json:"latest_local,omitempty"` - // The snapshot an in-flight upload is reading, if any. If it is - // latest_local, its local copy stays after the upload. If it is an older - // snapshot, whose upload began before latest_local was taken, atelet - // deletes its local copy once the upload ends. + // The snapshot a background upload has begun making durable, if any. Once + // begun, atelet keeps uploading it across checkpoints and restarts until + // it succeeds, fails and a newer local snapshot replaces it, or a newer + // snapshot becomes durable. If it is latest_local, its local copy stays + // after the upload. If it is an older snapshot, atelet deletes its local + // copy once the upload ends or is replaced. Uploading *SnapshotReport `protobuf:"bytes,2,opt,name=uploading,proto3" json:"uploading,omitempty"` // The newest snapshot atelet knows is durable, whether or not the control // plane has recorded it: one atelet uploaded since it started, or one a // PromoteSnapshot found already uploaded. Unset if there is none, which says // nothing about older durable snapshots. LatestDurable *SnapshotReport `protobuf:"bytes,3,opt,name=latest_durable,json=latestDurable,proto3" json:"latest_durable,omitempty"` - // When atelet next plans to trigger a background upload, which will upload - // whichever snapshot is latest_local at the time of triggering. Unset when - // there is no latest_local, or it is already durable or being uploaded. - // Advisory: a promote or detach can upload it sooner, and an older - // snapshot's upload can delay it. + // When atelet next plans to trigger a background upload, of `uploading` if + // it is waiting to resume or retry, or of `latest_local` otherwise. Unset + // when neither needs an upload. Advisory: a promote or detach can upload + // it sooner, and an upload or promote in progress can delay it. UploadDueTime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=upload_due_time,json=uploadDueTime,proto3" json:"upload_due_time,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -3574,9 +3593,14 @@ func (x *CheckpointRequest) GetUploadConfig() *UploadConfig { type CheckpointResponse struct { state protoimpl.MessageState `protogen:"open.v1"` // The actor's state once the checkpoint is taken: latest_local is the new - // snapshot, unless UPLOAD_MODE_IMMEDIATE kept no local copy. The control - // plane can drop its entries for any local-only generation not named here: - // atelet has deleted it, and will never upload it. + // snapshot, unless UPLOAD_MODE_IMMEDIATE could not keep a local copy. + // atelet has deleted any local snapshot not named here, or deletes it once + // uploading ends. + // + // TODO: Add a snapshot garbage collection system for orphaned objects in + // object storage. atelet reports a background upload only when it records + // a newer durable snapshot, so an upload that a newer durable snapshot + // superseded while it ran leaves objects ateapi never learned were durable. State *ActorSnapshotState `protobuf:"bytes,1,opt,name=state,proto3" json:"state,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -3674,12 +3698,21 @@ type RestoreRequest struct { // +k8s:minimum=0 MemoryBytes int64 `protobuf:"varint,15,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` // Memory limit in bytes. // The sandbox binaries and pause image to restore with, resolved from the - // ActorTemplate's SandboxConfig. + // assigned worker pool's SandboxConfig. // // +k8s:required SandboxAssets *SandboxAssets `protobuf:"bytes,16,opt,name=sandbox_assets,json=sandboxAssets,proto3" json:"sandbox_assets,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // The runtime the assigned worker restores this actor under, from + // WorkerAssignment.sandbox_runtime. + // + // TODO: atelet still restores with sandbox_assets; a follow-up resolves the + // SandboxConfig version from this field, makes it required, and removes + // sandbox_assets. + // + // +k8s:optional + SandboxRuntime *SandboxRuntime `protobuf:"bytes,17,opt,name=sandbox_runtime,json=sandboxRuntime,proto3" json:"sandbox_runtime,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RestoreRequest) Reset() { @@ -3796,6 +3829,13 @@ func (x *RestoreRequest) GetSandboxAssets() *SandboxAssets { return nil } +func (x *RestoreRequest) GetSandboxRuntime() *SandboxRuntime { + if x != nil { + return x.SandboxRuntime + } + return nil +} + type RestoreResponse struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -3834,11 +3874,8 @@ func (*RestoreResponse) Descriptor() ([]byte, []int) { type PromoteSnapshotRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - // Golden actors are never paused, so the golden atespace is rejected. - // // +k8s:required // +k8s:format=k8s-short-name - // +k8s:customValidation # not the golden atespace Atespace string `protobuf:"bytes,1,opt,name=atespace,proto3" json:"atespace,omitempty"` // +k8s:required // +k8s:format=k8s-short-name @@ -4168,7 +4205,7 @@ const file_atelet_proto_rawDesc = "" + "\x17actor_template_atespace\x18\x05 \x01(\tR\x15actorTemplateAtespace\x12.\n" + "\x13actor_template_name\x18\x06 \x01(\tR\x11actorTemplateName\x12(\n" + "\x04spec\x18\a \x01(\v2\x14.atelet.WorkloadSpecR\x04spec\"\x13\n" + - "\x11TerminateResponse\"\xf0\x03\n" + + "\x11TerminateResponse\"\xb1\x04\n" + "\n" + "RunRequest\x12$\n" + "\x0eworker_pod_uid\x18\x01 \x01(\tR\fworkerPodUid\x12\x1a\n" + @@ -4183,7 +4220,8 @@ const file_atelet_proto_rawDesc = "" + "\x0eegress_gateway\x18\t \x01(\v2\x15.atelet.EgressGatewayH\x00R\regressGateway\x88\x01\x01\x12\x1b\n" + "\tcpu_milli\x18\n" + " \x01(\x03R\bcpuMilli\x12!\n" + - "\fmemory_bytes\x18\v \x01(\x03R\vmemoryBytesB\x11\n" + + "\fmemory_bytes\x18\v \x01(\x03R\vmemoryBytes\x12?\n" + + "\x0fsandbox_runtime\x18\f \x01(\v2\x16.atelet.SandboxRuntimeR\x0esandboxRuntimeB\x11\n" + "\x0f_egress_gateway\")\n" + "\rEgressGateway\x12\x18\n" + "\aaddress\x18\x01 \x01(\tR\aaddress\":\n" + @@ -4311,7 +4349,7 @@ const file_atelet_proto_rawDesc = "" + "\bsnapshot\x18\b \x01(\v2\x10.atelet.SnapshotR\bsnapshot\x129\n" + "\rupload_config\x18\t \x01(\v2\x14.atelet.UploadConfigR\fuploadConfig\"F\n" + "\x12CheckpointResponse\x120\n" + - "\x05state\x18\x01 \x01(\v2\x1a.atelet.ActorSnapshotStateR\x05state\"\xa2\x04\n" + + "\x05state\x18\x01 \x01(\v2\x1a.atelet.ActorSnapshotStateR\x05state\"\xe3\x04\n" + "\x0eRestoreRequest\x12$\n" + "\x0eworker_pod_uid\x18\x01 \x01(\tR\fworkerPodUid\x12\x1a\n" + "\batespace\x18\x02 \x01(\tR\batespace\x12\x1d\n" + @@ -4325,7 +4363,8 @@ const file_atelet_proto_rawDesc = "" + "\x0eegress_gateway\x18\r \x01(\v2\x15.atelet.EgressGatewayH\x00R\regressGateway\x88\x01\x01\x12\x1b\n" + "\tcpu_milli\x18\x0e \x01(\x03R\bcpuMilli\x12!\n" + "\fmemory_bytes\x18\x0f \x01(\x03R\vmemoryBytes\x12<\n" + - "\x0esandbox_assets\x18\x10 \x01(\v2\x15.atelet.SandboxAssetsR\rsandboxAssetsB\x11\n" + + "\x0esandbox_assets\x18\x10 \x01(\v2\x15.atelet.SandboxAssetsR\rsandboxAssets\x12?\n" + + "\x0fsandbox_runtime\x18\x11 \x01(\v2\x16.atelet.SandboxRuntimeR\x0esandboxRuntimeB\x11\n" + "\x0f_egress_gateway\"\x11\n" + "\x0fRestoreResponse\"\x86\x02\n" + "\x16PromoteSnapshotRequest\x12\x1a\n" + @@ -4475,76 +4514,78 @@ var file_atelet_proto_depIdxs = []int32{ 29, // 15: atelet.RunRequest.spec:type_name -> atelet.WorkloadSpec 28, // 16: atelet.RunRequest.sandbox_assets:type_name -> atelet.SandboxAssets 25, // 17: atelet.RunRequest.egress_gateway:type_name -> atelet.EgressGateway - 61, // 18: atelet.ArchAssets.files:type_name -> atelet.ArchAssets.FilesEntry - 62, // 19: atelet.SandboxAssets.assets:type_name -> atelet.SandboxAssets.AssetsEntry - 40, // 20: atelet.WorkloadSpec.containers:type_name -> atelet.Container - 38, // 21: atelet.WorkloadSpec.volumes:type_name -> atelet.Volume - 63, // 22: atelet.ExternalVolumeSource.volume_context:type_name -> atelet.ExternalVolumeSource.VolumeContextEntry - 64, // 23: atelet.ExternalVolumeSource.publish_context:type_name -> atelet.ExternalVolumeSource.PublishContextEntry - 0, // 24: atelet.ActorMetadataItem.field:type_name -> atelet.ActorMetadataField - 33, // 25: atelet.ActorMetadataDataSource.items:type_name -> atelet.ActorMetadataItem - 34, // 26: atelet.SystemInfoDataSource.actor_metadata:type_name -> atelet.ActorMetadataDataSource - 35, // 27: atelet.SystemInfoDataSource.trust_bundle:type_name -> atelet.TrustBundleDataSource - 36, // 28: atelet.SystemInfoVolumeSource.data_sources:type_name -> atelet.SystemInfoDataSource - 30, // 29: atelet.Volume.durable_dir:type_name -> atelet.DurableDirVolumeSource - 31, // 30: atelet.Volume.external:type_name -> atelet.ExternalVolumeSource - 37, // 31: atelet.Volume.system_info:type_name -> atelet.SystemInfoVolumeSource - 32, // 32: atelet.Volume.image:type_name -> atelet.ImageVolumeSource - 44, // 33: atelet.Container.env:type_name -> atelet.EnvVar - 45, // 34: atelet.Container.wakeup_probe:type_name -> atelet.WakeupProbe - 39, // 35: atelet.Container.volume_mounts:type_name -> atelet.VolumeMount - 41, // 36: atelet.Container.security_context:type_name -> atelet.SecurityContext - 43, // 37: atelet.Container.resources:type_name -> atelet.ResourceLimits - 42, // 38: atelet.SecurityContext.capabilities:type_name -> atelet.Capabilities - 46, // 39: atelet.WakeupProbe.http_get:type_name -> atelet.HttpGetAction - 1, // 40: atelet.Snapshot.fidelity:type_name -> atelet.SnapshotFidelity - 48, // 41: atelet.Snapshot.object:type_name -> atelet.ObjectStorage - 49, // 42: atelet.SnapshotReport.snapshot:type_name -> atelet.Snapshot - 65, // 43: atelet.SnapshotReport.local_commit_time:type_name -> google.protobuf.Timestamp - 65, // 44: atelet.SnapshotReport.durable_complete_time:type_name -> google.protobuf.Timestamp - 50, // 45: atelet.ActorSnapshotState.latest_local:type_name -> atelet.SnapshotReport - 50, // 46: atelet.ActorSnapshotState.uploading:type_name -> atelet.SnapshotReport - 50, // 47: atelet.ActorSnapshotState.latest_durable:type_name -> atelet.SnapshotReport - 65, // 48: atelet.ActorSnapshotState.upload_due_time:type_name -> google.protobuf.Timestamp - 2, // 49: atelet.UploadConfig.mode:type_name -> atelet.UploadMode - 29, // 50: atelet.CheckpointRequest.spec:type_name -> atelet.WorkloadSpec - 49, // 51: atelet.CheckpointRequest.snapshot:type_name -> atelet.Snapshot - 52, // 52: atelet.CheckpointRequest.upload_config:type_name -> atelet.UploadConfig - 51, // 53: atelet.CheckpointResponse.state:type_name -> atelet.ActorSnapshotState - 29, // 54: atelet.RestoreRequest.spec:type_name -> atelet.WorkloadSpec - 49, // 55: atelet.RestoreRequest.snapshot:type_name -> atelet.Snapshot - 25, // 56: atelet.RestoreRequest.egress_gateway:type_name -> atelet.EgressGateway - 28, // 57: atelet.RestoreRequest.sandbox_assets:type_name -> atelet.SandboxAssets - 49, // 58: atelet.PromoteSnapshotRequest.snapshot:type_name -> atelet.Snapshot - 51, // 59: atelet.PromoteSnapshotResponse.state:type_name -> atelet.ActorSnapshotState - 51, // 60: atelet.DetachActorResponse.state:type_name -> atelet.ActorSnapshotState - 26, // 61: atelet.ArchAssets.FilesEntry.value:type_name -> atelet.AssetFile - 27, // 62: atelet.SandboxAssets.AssetsEntry.value:type_name -> atelet.ArchAssets - 20, // 63: atelet.WorkerSupport.MintActorCertificate:input_type -> atelet.MintActorCertificateRequest - 3, // 64: atelet.WorkerSupport.RegisterWorker:input_type -> atelet.RegisterWorkerRequest - 11, // 65: atelet.WorkerSupport.RequestActorSuspend:input_type -> atelet.RequestActorSuspendRequest - 13, // 66: atelet.WorkerSupport.GetActorEgressPolicy:input_type -> atelet.GetActorEgressPolicyRequest - 24, // 67: atelet.Atelet.Run:input_type -> atelet.RunRequest - 53, // 68: atelet.Atelet.Checkpoint:input_type -> atelet.CheckpointRequest - 55, // 69: atelet.Atelet.Restore:input_type -> atelet.RestoreRequest - 57, // 70: atelet.Atelet.PromoteSnapshot:input_type -> atelet.PromoteSnapshotRequest - 59, // 71: atelet.Atelet.DetachActor:input_type -> atelet.DetachActorRequest - 22, // 72: atelet.Atelet.Terminate:input_type -> atelet.TerminateRequest - 21, // 73: atelet.WorkerSupport.MintActorCertificate:output_type -> atelet.MintActorCertificateResponse - 10, // 74: atelet.WorkerSupport.RegisterWorker:output_type -> atelet.RegisterWorkerResponse - 12, // 75: atelet.WorkerSupport.RequestActorSuspend:output_type -> atelet.RequestActorSuspendResponse - 14, // 76: atelet.WorkerSupport.GetActorEgressPolicy:output_type -> atelet.GetActorEgressPolicyResponse - 47, // 77: atelet.Atelet.Run:output_type -> atelet.RunResponse - 54, // 78: atelet.Atelet.Checkpoint:output_type -> atelet.CheckpointResponse - 56, // 79: atelet.Atelet.Restore:output_type -> atelet.RestoreResponse - 58, // 80: atelet.Atelet.PromoteSnapshot:output_type -> atelet.PromoteSnapshotResponse - 60, // 81: atelet.Atelet.DetachActor:output_type -> atelet.DetachActorResponse - 23, // 82: atelet.Atelet.Terminate:output_type -> atelet.TerminateResponse - 73, // [73:83] is the sub-list for method output_type - 63, // [63:73] is the sub-list for method input_type - 63, // [63:63] is the sub-list for extension type_name - 63, // [63:63] is the sub-list for extension extendee - 0, // [0:63] is the sub-list for field type_name + 4, // 18: atelet.RunRequest.sandbox_runtime:type_name -> atelet.SandboxRuntime + 61, // 19: atelet.ArchAssets.files:type_name -> atelet.ArchAssets.FilesEntry + 62, // 20: atelet.SandboxAssets.assets:type_name -> atelet.SandboxAssets.AssetsEntry + 40, // 21: atelet.WorkloadSpec.containers:type_name -> atelet.Container + 38, // 22: atelet.WorkloadSpec.volumes:type_name -> atelet.Volume + 63, // 23: atelet.ExternalVolumeSource.volume_context:type_name -> atelet.ExternalVolumeSource.VolumeContextEntry + 64, // 24: atelet.ExternalVolumeSource.publish_context:type_name -> atelet.ExternalVolumeSource.PublishContextEntry + 0, // 25: atelet.ActorMetadataItem.field:type_name -> atelet.ActorMetadataField + 33, // 26: atelet.ActorMetadataDataSource.items:type_name -> atelet.ActorMetadataItem + 34, // 27: atelet.SystemInfoDataSource.actor_metadata:type_name -> atelet.ActorMetadataDataSource + 35, // 28: atelet.SystemInfoDataSource.trust_bundle:type_name -> atelet.TrustBundleDataSource + 36, // 29: atelet.SystemInfoVolumeSource.data_sources:type_name -> atelet.SystemInfoDataSource + 30, // 30: atelet.Volume.durable_dir:type_name -> atelet.DurableDirVolumeSource + 31, // 31: atelet.Volume.external:type_name -> atelet.ExternalVolumeSource + 37, // 32: atelet.Volume.system_info:type_name -> atelet.SystemInfoVolumeSource + 32, // 33: atelet.Volume.image:type_name -> atelet.ImageVolumeSource + 44, // 34: atelet.Container.env:type_name -> atelet.EnvVar + 45, // 35: atelet.Container.wakeup_probe:type_name -> atelet.WakeupProbe + 39, // 36: atelet.Container.volume_mounts:type_name -> atelet.VolumeMount + 41, // 37: atelet.Container.security_context:type_name -> atelet.SecurityContext + 43, // 38: atelet.Container.resources:type_name -> atelet.ResourceLimits + 42, // 39: atelet.SecurityContext.capabilities:type_name -> atelet.Capabilities + 46, // 40: atelet.WakeupProbe.http_get:type_name -> atelet.HttpGetAction + 1, // 41: atelet.Snapshot.fidelity:type_name -> atelet.SnapshotFidelity + 48, // 42: atelet.Snapshot.object:type_name -> atelet.ObjectStorage + 49, // 43: atelet.SnapshotReport.snapshot:type_name -> atelet.Snapshot + 65, // 44: atelet.SnapshotReport.local_commit_time:type_name -> google.protobuf.Timestamp + 65, // 45: atelet.SnapshotReport.durable_complete_time:type_name -> google.protobuf.Timestamp + 50, // 46: atelet.ActorSnapshotState.latest_local:type_name -> atelet.SnapshotReport + 50, // 47: atelet.ActorSnapshotState.uploading:type_name -> atelet.SnapshotReport + 50, // 48: atelet.ActorSnapshotState.latest_durable:type_name -> atelet.SnapshotReport + 65, // 49: atelet.ActorSnapshotState.upload_due_time:type_name -> google.protobuf.Timestamp + 2, // 50: atelet.UploadConfig.mode:type_name -> atelet.UploadMode + 29, // 51: atelet.CheckpointRequest.spec:type_name -> atelet.WorkloadSpec + 49, // 52: atelet.CheckpointRequest.snapshot:type_name -> atelet.Snapshot + 52, // 53: atelet.CheckpointRequest.upload_config:type_name -> atelet.UploadConfig + 51, // 54: atelet.CheckpointResponse.state:type_name -> atelet.ActorSnapshotState + 29, // 55: atelet.RestoreRequest.spec:type_name -> atelet.WorkloadSpec + 49, // 56: atelet.RestoreRequest.snapshot:type_name -> atelet.Snapshot + 25, // 57: atelet.RestoreRequest.egress_gateway:type_name -> atelet.EgressGateway + 28, // 58: atelet.RestoreRequest.sandbox_assets:type_name -> atelet.SandboxAssets + 4, // 59: atelet.RestoreRequest.sandbox_runtime:type_name -> atelet.SandboxRuntime + 49, // 60: atelet.PromoteSnapshotRequest.snapshot:type_name -> atelet.Snapshot + 51, // 61: atelet.PromoteSnapshotResponse.state:type_name -> atelet.ActorSnapshotState + 51, // 62: atelet.DetachActorResponse.state:type_name -> atelet.ActorSnapshotState + 26, // 63: atelet.ArchAssets.FilesEntry.value:type_name -> atelet.AssetFile + 27, // 64: atelet.SandboxAssets.AssetsEntry.value:type_name -> atelet.ArchAssets + 20, // 65: atelet.WorkerSupport.MintActorCertificate:input_type -> atelet.MintActorCertificateRequest + 3, // 66: atelet.WorkerSupport.RegisterWorker:input_type -> atelet.RegisterWorkerRequest + 11, // 67: atelet.WorkerSupport.RequestActorSuspend:input_type -> atelet.RequestActorSuspendRequest + 13, // 68: atelet.WorkerSupport.GetActorEgressPolicy:input_type -> atelet.GetActorEgressPolicyRequest + 24, // 69: atelet.Atelet.Run:input_type -> atelet.RunRequest + 53, // 70: atelet.Atelet.Checkpoint:input_type -> atelet.CheckpointRequest + 55, // 71: atelet.Atelet.Restore:input_type -> atelet.RestoreRequest + 57, // 72: atelet.Atelet.PromoteSnapshot:input_type -> atelet.PromoteSnapshotRequest + 59, // 73: atelet.Atelet.DetachActor:input_type -> atelet.DetachActorRequest + 22, // 74: atelet.Atelet.Terminate:input_type -> atelet.TerminateRequest + 21, // 75: atelet.WorkerSupport.MintActorCertificate:output_type -> atelet.MintActorCertificateResponse + 10, // 76: atelet.WorkerSupport.RegisterWorker:output_type -> atelet.RegisterWorkerResponse + 12, // 77: atelet.WorkerSupport.RequestActorSuspend:output_type -> atelet.RequestActorSuspendResponse + 14, // 78: atelet.WorkerSupport.GetActorEgressPolicy:output_type -> atelet.GetActorEgressPolicyResponse + 47, // 79: atelet.Atelet.Run:output_type -> atelet.RunResponse + 54, // 80: atelet.Atelet.Checkpoint:output_type -> atelet.CheckpointResponse + 56, // 81: atelet.Atelet.Restore:output_type -> atelet.RestoreResponse + 58, // 82: atelet.Atelet.PromoteSnapshot:output_type -> atelet.PromoteSnapshotResponse + 60, // 83: atelet.Atelet.DetachActor:output_type -> atelet.DetachActorResponse + 23, // 84: atelet.Atelet.Terminate:output_type -> atelet.TerminateResponse + 75, // [75:85] is the sub-list for method output_type + 65, // [65:75] is the sub-list for method input_type + 65, // [65:65] is the sub-list for extension type_name + 65, // [65:65] is the sub-list for extension extendee + 0, // [0:65] is the sub-list for field type_name } func init() { file_atelet_proto_init() } diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index dca175468d..c00ee06159 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -110,7 +110,7 @@ message VersionedSandboxCompat { string schema_version = 1; // Key-value compatibility attributes defined by this schema version, covering - // both runtime assets (e.g. "sandbox_runtime_hash") and effective guest + // both runtime assets (e.g. "sandbox_bundle_hash") and effective guest // hardware features after masking (e.g. "architecture", "cpu_features"). // Matched against a snapshot under the same schema_version: every attribute // on the snapshot must be present on the worker with an equal value (except @@ -130,7 +130,7 @@ message VersionedSandboxCompat { // AttributeEntry is a single key-value compatibility attribute. message AttributeEntry { - // Attribute name (e.g. "architecture", "cpu_features", "sandbox_runtime_hash"). + // Attribute name (e.g. "architecture", "cpu_features", "sandbox_bundle_hash"). // // +k8s:required // +k8s:maxLength=128 @@ -320,11 +320,10 @@ service Atelet { // Checkpoint saves the workload's state as a new snapshot of the actor and // returns the worker to a blank state. upload_config says when the snapshot - // becomes durable. Once the worker has captured the workload, and before the new - // snapshot is uploaded or stored locally, atelet deletes the actor's older - // local snapshots, except one still being uploaded, which it deletes when - // that upload ends. A Checkpoint that then fails to upload or store the new - // snapshot leaves the actor with no local snapshot. + // becomes durable. Once the new snapshot is uploaded or stored locally, + // atelet deletes the actor's older local snapshots, except one still being + // uploaded, which it deletes after that upload ends. A Checkpoint that fails + // before then leaves the actor's previous local snapshot in place. rpc Checkpoint(CheckpointRequest) returns (CheckpointResponse) {} // Restore restores a workload from a snapshot onto a worker. @@ -339,18 +338,20 @@ service Atelet { // Idempotent: once the snapshot is durable, atelet answers from the // manifest at `snapshot.object`, and fails if there is none. // - // TODO: Temporarily, this deletes all of the actor's local snapshots after - // the upload and does not check whether the actor is running. Its only - // caller, the suspend of a paused actor, relies on the deletion. + // TODO: Temporarily, this does not check whether the actor is running. rpc PromoteSnapshot(PromoteSnapshotRequest) returns (PromoteSnapshotResponse) {} // DetachActor makes the actor's latest local snapshot durable, then deletes // the actor's local snapshots on this node, so it can resume on any node. - // atelet also detaches a suspended actor on its own after its detach delay. + // atelet also detaches a suspended actor on its own after its detach delay, + // and reports that with WorkerService.ReportActorSnapshotState; it does not + // report a DetachActor call, whose caller reads the state from the response. // Waits for an operation already in progress on the actor, such as an // upload or a detach. Does nothing when atelet holds no local snapshot for // the actor. Returns ABORTED while a Run, Restore, or Checkpoint of the - // actor is in progress or the actor runs on this node. + // actor is in progress or the actor runs on this node, and + // FAILED_PRECONDITION when atelet cannot make the latest local snapshot + // durable, such as when it has no `snapshot.object`. // // TODO: Temporarily, atelet returns UNIMPLEMENTED. rpc DetachActor(DetachActorRequest) returns (DetachActorResponse) {} @@ -456,6 +457,16 @@ message RunRequest { // +k8s:optional // +k8s:minimum=0 int64 memory_bytes = 11; // Memory limit in bytes. + + // The runtime the assigned worker runs this actor under, from + // WorkerAssignment.sandbox_runtime. + // + // TODO: atelet still runs with sandbox_assets; a follow-up resolves the + // SandboxConfig version from this field, makes it required, and removes + // sandbox_assets. + // + // +k8s:optional + SandboxRuntime sandbox_runtime = 12; } // EgressGateway configures tunneled egress for one actor activation. @@ -503,7 +514,7 @@ message ArchAssets { // an actor runs in: a sandbox class, the pause image holding its namespaces, // and binaries keyed first by architecture (GOARCH) and then by asset name. // atelet's backend code interprets the asset names (gVisor expects "gvisor", -// the release tarball; legacy "runsc", a bare binary, is still accepted). +// the release tarball). message SandboxAssets { // +k8s:required // +k8s:maxLength=63 # guardrail; the class names are a few characters @@ -858,8 +869,7 @@ enum SnapshotFidelity { // (currently DurableDir-typed volumes). Memory and the rest of rootfs are // excluded. SNAPSHOT_FIDELITY_VOLUMES = 1; - // Volumes plus the root filesystem changes made since boot. Served by the - // micro-VM runtime only; other runtimes reject requests carrying it. + // Volumes plus the root filesystem changes made since boot. SNAPSHOT_FIDELITY_ROOTFS = 2; // Capture process memory plus the full filesystem delta on top of the OCI // image (including any attached DurableDir volumes). @@ -893,8 +903,6 @@ message Snapshot { // for the actor, or for its newest under another snapshot_uuid: the // caller's view is stale. // - // TODO: Temporarily, atelet does not compare it with the snapshots it holds. - // // +k8s:required // +k8s:minimum=1 int32 generation = 2; @@ -941,16 +949,18 @@ message SnapshotReport { // ActorSnapshotState is what atelet holds for one actor on this node. // Returned by Checkpoint, PromoteSnapshot, DetachActor. message ActorSnapshotState { - // The snapshot atelet restores the actor from locally. Unset when the - // latest checkpoint kept no local copy, and once a detach has made the - // snapshot durable and begun deleting it. Either way, latest_durable is - // then the actor's latest snapshot. + // The snapshot atelet restores the actor from locally. Unset when an + // UPLOAD_MODE_IMMEDIATE checkpoint could not keep a local copy, and once a + // detach has made the snapshot durable and begun deleting it. Either way, + // latest_durable is then the actor's latest snapshot. SnapshotReport latest_local = 1; - // The snapshot an in-flight upload is reading, if any. If it is - // latest_local, its local copy stays after the upload. If it is an older - // snapshot, whose upload began before latest_local was taken, atelet - // deletes its local copy once the upload ends. + // The snapshot a background upload has begun making durable, if any. Once + // begun, atelet keeps uploading it across checkpoints and restarts until + // it succeeds, fails and a newer local snapshot replaces it, or a newer + // snapshot becomes durable. If it is latest_local, its local copy stays + // after the upload. If it is an older snapshot, atelet deletes its local + // copy once the upload ends or is replaced. SnapshotReport uploading = 2; // The newest snapshot atelet knows is durable, whether or not the control @@ -959,11 +969,10 @@ message ActorSnapshotState { // nothing about older durable snapshots. SnapshotReport latest_durable = 3; - // When atelet next plans to trigger a background upload, which will upload - // whichever snapshot is latest_local at the time of triggering. Unset when - // there is no latest_local, or it is already durable or being uploaded. - // Advisory: a promote or detach can upload it sooner, and an older - // snapshot's upload can delay it. + // When atelet next plans to trigger a background upload, of `uploading` if + // it is waiting to resume or retry, or of `latest_local` otherwise. Unset + // when neither needs an upload. Advisory: a promote or detach can upload + // it sooner, and an upload or promote in progress can delay it. google.protobuf.Timestamp upload_due_time = 4; } @@ -979,21 +988,26 @@ message UploadConfig { enum UploadMode { UPLOAD_MODE_UNSPECIFIED = 0; - // Keep the snapshot locally and upload it on atelet's schedule: at most - // for atelet's upload-delay after the oldest snapshot of the actor - // that is not yet durable was taken. Snapshots superseded before then - // are never uploaded. + // Keep the snapshot locally and upload it on atelet's schedule: about + // atelet's upload-delay after the oldest snapshot of the actor that is not + // yet durable was taken, or later while an upload is in progress or + // failing. Snapshots superseded before then are never uploaded. + // + // Once the actor has stayed suspended on this node for atelet's + // detach-delay, atelet detaches it: it uploads the latest local snapshot + // unless that is durable already, then deletes all the local snapshots. // - // Once the actor has stayed suspended on this node for - // atelet's detach-delay, atelet detaches it (deletes all the local snapshots). + // atelet reports the actor's state with + // WorkerService.ReportActorSnapshotState once such an upload makes a newer + // snapshot durable, and once such a detach has deleted the local snapshots. // - // TODO: Temporarily the deferred upload and detach are unimplemented. Only a local - // copy is taken. + // atelet never uploads or detaches a snapshot it cannot make durable, such + // as one without `snapshot.object`. UPLOAD_MODE_DEFERRED = 1; - // Upload the snapshot synchronously before Checkpoint returns, while keeping the local copy. - // - // TODO: Temporarily this deletes the local snapshot after the upload. + // Upload the snapshot synchronously before Checkpoint returns, and keep a + // local copy if atelet can: failing to store it does not fail the + // Checkpoint. UPLOAD_MODE_IMMEDIATE = 2; // Keep this in sync with the maximums on fields of this type. } @@ -1056,9 +1070,14 @@ message CheckpointRequest { message CheckpointResponse { // The actor's state once the checkpoint is taken: latest_local is the new - // snapshot, unless UPLOAD_MODE_IMMEDIATE kept no local copy. The control - // plane can drop its entries for any local-only generation not named here: - // atelet has deleted it, and will never upload it. + // snapshot, unless UPLOAD_MODE_IMMEDIATE could not keep a local copy. + // atelet has deleted any local snapshot not named here, or deletes it once + // uploading ends. + // + // TODO: Add a snapshot garbage collection system for orphaned objects in + // object storage. atelet reports a background upload only when it records + // a newer durable snapshot, so an upload that a newer durable snapshot + // superseded while it ran leaves objects ateapi never learned were durable. ActorSnapshotState state = 1; } @@ -1127,21 +1146,28 @@ message RestoreRequest { int64 memory_bytes = 15; // Memory limit in bytes. // The sandbox binaries and pause image to restore with, resolved from the - // ActorTemplate's SandboxConfig. + // assigned worker pool's SandboxConfig. // // +k8s:required SandboxAssets sandbox_assets = 16; + + // The runtime the assigned worker restores this actor under, from + // WorkerAssignment.sandbox_runtime. + // + // TODO: atelet still restores with sandbox_assets; a follow-up resolves the + // SandboxConfig version from this field, makes it required, and removes + // sandbox_assets. + // + // +k8s:optional + SandboxRuntime sandbox_runtime = 17; } message RestoreResponse { } message PromoteSnapshotRequest { - // Golden actors are never paused, so the golden atespace is rejected. - // // +k8s:required // +k8s:format=k8s-short-name - // +k8s:customValidation # not the golden atespace string atespace = 1; // +k8s:required diff --git a/internal/proto/ateletpb/atelet_grpc.pb.go b/internal/proto/ateletpb/atelet_grpc.pb.go index 2282a4651b..5a35d73bbe 100644 --- a/internal/proto/ateletpb/atelet_grpc.pb.go +++ b/internal/proto/ateletpb/atelet_grpc.pb.go @@ -319,11 +319,10 @@ type AteletClient interface { Run(ctx context.Context, in *RunRequest, opts ...grpc.CallOption) (*RunResponse, error) // Checkpoint saves the workload's state as a new snapshot of the actor and // returns the worker to a blank state. upload_config says when the snapshot - // becomes durable. Once the worker has captured the workload, and before the new - // snapshot is uploaded or stored locally, atelet deletes the actor's older - // local snapshots, except one still being uploaded, which it deletes when - // that upload ends. A Checkpoint that then fails to upload or store the new - // snapshot leaves the actor with no local snapshot. + // becomes durable. Once the new snapshot is uploaded or stored locally, + // atelet deletes the actor's older local snapshots, except one still being + // uploaded, which it deletes after that upload ends. A Checkpoint that fails + // before then leaves the actor's previous local snapshot in place. Checkpoint(ctx context.Context, in *CheckpointRequest, opts ...grpc.CallOption) (*CheckpointResponse, error) // Restore restores a workload from a snapshot onto a worker. Restore(ctx context.Context, in *RestoreRequest, opts ...grpc.CallOption) (*RestoreResponse, error) @@ -336,17 +335,19 @@ type AteletClient interface { // Idempotent: once the snapshot is durable, atelet answers from the // manifest at `snapshot.object`, and fails if there is none. // - // TODO: Temporarily, this deletes all of the actor's local snapshots after - // the upload and does not check whether the actor is running. Its only - // caller, the suspend of a paused actor, relies on the deletion. + // TODO: Temporarily, this does not check whether the actor is running. PromoteSnapshot(ctx context.Context, in *PromoteSnapshotRequest, opts ...grpc.CallOption) (*PromoteSnapshotResponse, error) // DetachActor makes the actor's latest local snapshot durable, then deletes // the actor's local snapshots on this node, so it can resume on any node. - // atelet also detaches a suspended actor on its own after its detach delay. + // atelet also detaches a suspended actor on its own after its detach delay, + // and reports that with WorkerService.ReportActorSnapshotState; it does not + // report a DetachActor call, whose caller reads the state from the response. // Waits for an operation already in progress on the actor, such as an // upload or a detach. Does nothing when atelet holds no local snapshot for // the actor. Returns ABORTED while a Run, Restore, or Checkpoint of the - // actor is in progress or the actor runs on this node. + // actor is in progress or the actor runs on this node, and + // FAILED_PRECONDITION when atelet cannot make the latest local snapshot + // durable, such as when it has no `snapshot.object`. // // TODO: Temporarily, atelet returns UNIMPLEMENTED. DetachActor(ctx context.Context, in *DetachActorRequest, opts ...grpc.CallOption) (*DetachActorResponse, error) @@ -437,11 +438,10 @@ type AteletServer interface { Run(context.Context, *RunRequest) (*RunResponse, error) // Checkpoint saves the workload's state as a new snapshot of the actor and // returns the worker to a blank state. upload_config says when the snapshot - // becomes durable. Once the worker has captured the workload, and before the new - // snapshot is uploaded or stored locally, atelet deletes the actor's older - // local snapshots, except one still being uploaded, which it deletes when - // that upload ends. A Checkpoint that then fails to upload or store the new - // snapshot leaves the actor with no local snapshot. + // becomes durable. Once the new snapshot is uploaded or stored locally, + // atelet deletes the actor's older local snapshots, except one still being + // uploaded, which it deletes after that upload ends. A Checkpoint that fails + // before then leaves the actor's previous local snapshot in place. Checkpoint(context.Context, *CheckpointRequest) (*CheckpointResponse, error) // Restore restores a workload from a snapshot onto a worker. Restore(context.Context, *RestoreRequest) (*RestoreResponse, error) @@ -454,17 +454,19 @@ type AteletServer interface { // Idempotent: once the snapshot is durable, atelet answers from the // manifest at `snapshot.object`, and fails if there is none. // - // TODO: Temporarily, this deletes all of the actor's local snapshots after - // the upload and does not check whether the actor is running. Its only - // caller, the suspend of a paused actor, relies on the deletion. + // TODO: Temporarily, this does not check whether the actor is running. PromoteSnapshot(context.Context, *PromoteSnapshotRequest) (*PromoteSnapshotResponse, error) // DetachActor makes the actor's latest local snapshot durable, then deletes // the actor's local snapshots on this node, so it can resume on any node. - // atelet also detaches a suspended actor on its own after its detach delay. + // atelet also detaches a suspended actor on its own after its detach delay, + // and reports that with WorkerService.ReportActorSnapshotState; it does not + // report a DetachActor call, whose caller reads the state from the response. // Waits for an operation already in progress on the actor, such as an // upload or a detach. Does nothing when atelet holds no local snapshot for // the actor. Returns ABORTED while a Run, Restore, or Checkpoint of the - // actor is in progress or the actor runs on this node. + // actor is in progress or the actor runs on this node, and + // FAILED_PRECONDITION when atelet cannot make the latest local snapshot + // durable, such as when it has no `snapshot.object`. // // TODO: Temporarily, atelet returns UNIMPLEMENTED. DetachActor(context.Context, *DetachActorRequest) (*DetachActorResponse, error) diff --git a/internal/proto/ateworkerpb/ateworker.pb.go b/internal/proto/ateworkerpb/ateworker.pb.go index 2c12a7afed..fdc172232b 100644 --- a/internal/proto/ateworkerpb/ateworker.pb.go +++ b/internal/proto/ateworkerpb/ateworker.pb.go @@ -46,12 +46,11 @@ const ( // (currently DurableDir-typed volumes). Memory and the rest of rootfs are // excluded. SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES SnapshotFidelity = 1 - // Volumes plus the root filesystem changes made since boot. Served by the - // micro-VM runtime only; other runtimes reject requests carrying it. + // Volumes plus the root filesystem changes made since boot. SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS SnapshotFidelity = 2 // Capture process memory plus the full filesystem delta on top of the OCI // image (including any attached DurableDir volumes). - SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY SnapshotFidelity = 3 + SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY SnapshotFidelity = 3 // Keep this in sync with the maximums on fields of this type. ) // Enum value maps for SnapshotFidelity. @@ -161,27 +160,53 @@ func (StatsSource) EnumDescriptor() ([]byte, []int) { // it shares with the worker. The worker takes them from the RPC instead of deriving // them from the actor UID. // Every directory is under root_dir. +// +// +k8s:customValidation # each directory is an absolute, clean path type ActorDirs struct { state protoimpl.MessageState `protogen:"open.v1"` // The actor's own directory. + // + // +k8s:required + // +k8s:maxLength=4096 # PATH_MAX RootDir string `protobuf:"bytes,1,opt,name=root_dir,json=rootDir,proto3" json:"root_dir,omitempty"` // Holds one OCI bundle per container, named after the container. + // + // +k8s:required + // +k8s:maxLength=4096 # PATH_MAX OciBundleDir string `protobuf:"bytes,2,opt,name=oci_bundle_dir,json=ociBundleDir,proto3" json:"oci_bundle_dir,omitempty"` // Where CheckpointWorkload writes the snapshot files. + // + // +k8s:required + // +k8s:maxLength=4096 # PATH_MAX CheckpointDir string `protobuf:"bytes,3,opt,name=checkpoint_dir,json=checkpointDir,proto3" json:"checkpoint_dir,omitempty"` // Where the snapshot files RestoreWorkload reads have been downloaded to. + // + // +k8s:required + // +k8s:maxLength=4096 # PATH_MAX RestoreDir string `protobuf:"bytes,4,opt,name=restore_dir,json=restoreDir,proto3" json:"restore_dir,omitempty"` // Holds one subdirectory per durable-dir volume. + // + // +k8s:required + // +k8s:maxLength=4096 # PATH_MAX DurableDirVolumeMountsDir string `protobuf:"bytes,5,opt,name=durable_dir_volume_mounts_dir,json=durableDirVolumeMountsDir,proto3" json:"durable_dir_volume_mounts_dir,omitempty"` // Holds one subdirectory per system-info volume. + // + // +k8s:required + // +k8s:maxLength=4096 # PATH_MAX SystemInfoVolumeRootsDir string `protobuf:"bytes,6,opt,name=system_info_volume_roots_dir,json=systemInfoVolumeRootsDir,proto3" json:"system_info_volume_roots_dir,omitempty"` // Holds one mount point per external volume. + // + // +k8s:required + // +k8s:maxLength=4096 # PATH_MAX ExternalVolumeMountsDir string `protobuf:"bytes,7,opt,name=external_volume_mounts_dir,json=externalVolumeMountsDir,proto3" json:"external_volume_mounts_dir,omitempty"` // Holds the sandbox runtime assets staged for this actor (runsc, or the // micro-VM kernel, VMM and guest image), hard-linked from atelet's asset // cache. The only location a sandbox runtime may run a binary from; the // worker derives the same path from actor_uid and rejects a request // naming any other. + // + // +k8s:required + // +k8s:maxLength=4096 # PATH_MAX SandboxAssetsDir string `protobuf:"bytes,8,opt,name=sandbox_assets_dir,json=sandboxAssetsDir,proto3" json:"sandbox_assets_dir,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -274,17 +299,35 @@ func (x *ActorDirs) GetSandboxAssetsDir() string { } type TerminateWorkloadRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - Atespace string `protobuf:"bytes,1,opt,name=atespace,proto3" json:"atespace,omitempty"` - ActorName string `protobuf:"bytes,2,opt,name=actor_name,json=actorName,proto3" json:"actor_name,omitempty"` - ActorUid string `protobuf:"bytes,3,opt,name=actor_uid,json=actorUid,proto3" json:"actor_uid,omitempty"` - ActorTemplateAtespace string `protobuf:"bytes,4,opt,name=actor_template_atespace,json=actorTemplateAtespace,proto3" json:"actor_template_atespace,omitempty"` - ActorTemplateName string `protobuf:"bytes,5,opt,name=actor_template_name,json=actorTemplateName,proto3" json:"actor_template_name,omitempty"` - RunscPath string `protobuf:"bytes,6,opt,name=runsc_path,json=runscPath,proto3" json:"runsc_path,omitempty"` - Spec *WorkloadSpec `protobuf:"bytes,7,opt,name=spec,proto3" json:"spec,omitempty"` - ActorDirs *ActorDirs `protobuf:"bytes,8,opt,name=actor_dirs,json=actorDirs,proto3" json:"actor_dirs,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + // +k8s:required + // +k8s:format=k8s-short-name + Atespace string `protobuf:"bytes,1,opt,name=atespace,proto3" json:"atespace,omitempty"` + // +k8s:required + // +k8s:format=k8s-short-name + ActorName string `protobuf:"bytes,2,opt,name=actor_name,json=actorName,proto3" json:"actor_name,omitempty"` + // +k8s:required + // +k8s:format=k8s-uuid + ActorUid string `protobuf:"bytes,3,opt,name=actor_uid,json=actorUid,proto3" json:"actor_uid,omitempty"` + // +k8s:optional + // +k8s:format=k8s-short-name + ActorTemplateAtespace string `protobuf:"bytes,4,opt,name=actor_template_atespace,json=actorTemplateAtespace,proto3" json:"actor_template_atespace,omitempty"` + // +k8s:optional + // +k8s:format=k8s-short-name + ActorTemplateName string `protobuf:"bytes,5,opt,name=actor_template_name,json=actorTemplateName,proto3" json:"actor_template_name,omitempty"` + // Empty for runtimes that do not use runsc. + // + // +k8s:optional + // +k8s:maxLength=4096 # PATH_MAX; a host path, not a contract + RunscPath string `protobuf:"bytes,6,opt,name=runsc_path,json=runscPath,proto3" json:"runsc_path,omitempty"` + // Names the containers to stop. + // + // +k8s:required + Spec *WorkloadSpec `protobuf:"bytes,7,opt,name=spec,proto3" json:"spec,omitempty"` + // +k8s:required + ActorDirs *ActorDirs `protobuf:"bytes,8,opt,name=actor_dirs,json=actorDirs,proto3" json:"actor_dirs,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *TerminateWorkloadRequest) Reset() { @@ -410,27 +453,58 @@ func (*TerminateWorkloadResponse) Descriptor() ([]byte, []int) { } type RunWorkloadRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - Atespace string `protobuf:"bytes,1,opt,name=atespace,proto3" json:"atespace,omitempty"` - ActorName string `protobuf:"bytes,2,opt,name=actor_name,json=actorName,proto3" json:"actor_name,omitempty"` - ActorUid string `protobuf:"bytes,3,opt,name=actor_uid,json=actorUid,proto3" json:"actor_uid,omitempty"` - ActorTemplateAtespace string `protobuf:"bytes,4,opt,name=actor_template_atespace,json=actorTemplateAtespace,proto3" json:"actor_template_atespace,omitempty"` - ActorTemplateName string `protobuf:"bytes,5,opt,name=actor_template_name,json=actorTemplateName,proto3" json:"actor_template_name,omitempty"` - RunscPath string `protobuf:"bytes,6,opt,name=runsc_path,json=runscPath,proto3" json:"runsc_path,omitempty"` - Spec *WorkloadSpec `protobuf:"bytes,7,opt,name=spec,proto3" json:"spec,omitempty"` - // runtime_asset_paths maps a runtime asset name (e.g. "cloud-hypervisor", - // "virtiofsd", "kata-kernel", "kata-image") - // to the local on-disk path atelet fetched it to (content-addressed, like - // runsc_path). Empty for the gVisor runtime, which uses runsc_path. + state protoimpl.MessageState `protogen:"open.v1"` + // +k8s:required + // +k8s:format=k8s-short-name + Atespace string `protobuf:"bytes,1,opt,name=atespace,proto3" json:"atespace,omitempty"` + // +k8s:required + // +k8s:format=k8s-short-name + ActorName string `protobuf:"bytes,2,opt,name=actor_name,json=actorName,proto3" json:"actor_name,omitempty"` + // +k8s:required + // +k8s:format=k8s-uuid + ActorUid string `protobuf:"bytes,3,opt,name=actor_uid,json=actorUid,proto3" json:"actor_uid,omitempty"` + // +k8s:optional + // +k8s:format=k8s-short-name + ActorTemplateAtespace string `protobuf:"bytes,4,opt,name=actor_template_atespace,json=actorTemplateAtespace,proto3" json:"actor_template_atespace,omitempty"` + // +k8s:optional + // +k8s:format=k8s-short-name + ActorTemplateName string `protobuf:"bytes,5,opt,name=actor_template_name,json=actorTemplateName,proto3" json:"actor_template_name,omitempty"` + // Empty for runtimes that do not use runsc. + // + // +k8s:optional + // +k8s:maxLength=4096 # PATH_MAX; a host path, not a contract + RunscPath string `protobuf:"bytes,6,opt,name=runsc_path,json=runscPath,proto3" json:"runsc_path,omitempty"` + // +k8s:required + // +k8s:subfield(containers)=+k8s:required # a container to start; only Terminate's fallback spec may have none + Spec *WorkloadSpec `protobuf:"bytes,7,opt,name=spec,proto3" json:"spec,omitempty"` + // runtime_asset_paths maps a runtime asset name to the local on-disk path + // atelet fetched it to (content-addressed, like runsc_path). The names are + // the runtime's own: "gvisor" for gVisor, "cloud-hypervisor", + // "virtiofsd", "kata-kernel", and "kata-image" for the micro-VM. Empty for + // the gVisor runtime, which uses runsc_path. + // + // +k8s:optional + // +k8s:maxProperties=16 # matches atelet.ArchAssets.files + // +k8s:eachKey=+k8s:maxLength=64 # matches atelet.ArchAssets.files + // +k8s:eachVal=+k8s:maxLength=4096 # PATH_MAX; a host path, not a contract RuntimeAssetPaths map[string]string `protobuf:"bytes,8,rep,name=runtime_asset_paths,json=runtimeAssetPaths,proto3" json:"runtime_asset_paths,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // When absent the actor has no egress: its TCP is captured and refused. + // + // +k8s:optional EgressGateway *EgressGateway `protobuf:"bytes,10,opt,name=egress_gateway,json=egressGateway,proto3,oneof" json:"egress_gateway,omitempty"` // The actor's declared size, from the ActorTemplate's resource limits. The worker // sizes the sandbox to these (cgroup caps via the OCI spec, and for the // micro-VM the VM's vCPU count and memory). Zero means "unset": keep the // runtime default (unlimited for gVisor, the worker's own default for the micro-VM). - CpuMilli int64 `protobuf:"varint,11,opt,name=cpu_milli,json=cpuMilli,proto3" json:"cpu_milli,omitempty"` // CPU limit in millicores (1000 = one core). - MemoryBytes int64 `protobuf:"varint,12,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` // Memory limit in bytes. + // + // +k8s:optional + // +k8s:minimum=1 + // +k8s:maximum=999999 # matches atelet.RunRequest.cpu_milli, the control plane caps cpu limits strictly below 1000 cores + CpuMilli int64 `protobuf:"varint,11,opt,name=cpu_milli,json=cpuMilli,proto3" json:"cpu_milli,omitempty"` // CPU limit in millicores (1000 = one core). + // +k8s:optional + // +k8s:minimum=1 + MemoryBytes int64 `protobuf:"varint,12,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` // Memory limit in bytes. + // +k8s:required ActorDirs *ActorDirs `protobuf:"bytes,13,opt,name=actor_dirs,json=actorDirs,proto3" json:"actor_dirs,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -555,6 +629,10 @@ type EgressGateway struct { state protoimpl.MessageState `protogen:"open.v1"` // address is required and identifies the remote gateway as an IP address or // DNS name followed by its port. + // + // +k8s:required + // +k8s:maxLength=261 # a bracketed or 253-character host, ':' and a 5-digit port + // +k8s:customValidation # host:port shape Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -599,8 +677,15 @@ func (x *EgressGateway) GetAddress() string { // WorkloadSpec parallels Pod, but with far fewer configurable fields. type WorkloadSpec struct { - state protoimpl.MessageState `protogen:"open.v1"` - Containers []*Container `protobuf:"bytes,1,rep,name=containers,proto3" json:"containers,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + // Empty on the fallback spec atelet's Terminate sends for an actor whose + // template is gone, as on atelet's own WorkloadSpec. + // + // +k8s:optional + // +k8s:maxItems=10 # matches ateapi.ActorTemplate.containers + // +k8s:listType=map + // +k8s:listMapKey=name + Containers []*Container `protobuf:"bytes,1,rep,name=containers,proto3" json:"containers,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -642,25 +727,61 @@ func (x *WorkloadSpec) GetContainers() []*Container { return nil } +// A container's mounts split into four typed lists. Each list's key keeps +// only that list's paths unique, so the rules that span the lists are a hook. +// +// +k8s:customValidation # mount paths are unique and do not nest across the four lists type Container struct { - state protoimpl.MessageState `protogen:"open.v1"` - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - WakeupProbe *WakeupProbe `protobuf:"bytes,2,opt,name=wakeup_probe,json=wakeupProbe,proto3" json:"wakeup_probe,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + // The sandbox's own container, ocispec.PauseContainer, is not a short + // name, so the format keeps a workload container from taking its bundle. + // + // +k8s:required + // +k8s:format=k8s-short-name + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // +k8s:optional + WakeupProbe *WakeupProbe `protobuf:"bytes,2,opt,name=wakeup_probe,json=wakeupProbe,proto3" json:"wakeup_probe,omitempty"` // durable_dir_volume_mounts are the durable-dir volumes this container // mounts, if any. + // + // +k8s:optional + // +k8s:maxItems=32 # matches ateapi.Container.volume_mounts + // +k8s:listType=map + // +k8s:listMapKey=mount_path DurableDirVolumeMounts []*VolumeMount `protobuf:"bytes,3,rep,name=durable_dir_volume_mounts,json=durableDirVolumeMounts,proto3" json:"durable_dir_volume_mounts,omitempty"` // external_volume_mounts are the external volumes this container mounts, if // any. + // + // +k8s:optional + // +k8s:maxItems=32 # matches ateapi.Container.volume_mounts + // +k8s:listType=map + // +k8s:listMapKey=mount_path ExternalVolumeMounts []*VolumeMount `protobuf:"bytes,4,rep,name=external_volume_mounts,json=externalVolumeMounts,proto3" json:"external_volume_mounts,omitempty"` // system_info_volume_mounts are the system-info volumes this container // mounts, if any. Contents are generated by atelet on the host on every // Run/Restore and never captured into snapshots. The container sees them // read-only. + // + // +k8s:optional + // +k8s:maxItems=32 # matches ateapi.Container.volume_mounts + // +k8s:listType=map + // +k8s:listMapKey=mount_path SystemInfoVolumeMounts []*VolumeMount `protobuf:"bytes,5,rep,name=system_info_volume_mounts,json=systemInfoVolumeMounts,proto3" json:"system_info_volume_mounts,omitempty"` // image_volume_mounts are the image volumes this container mounts, if any. + // + // +k8s:optional + // +k8s:maxItems=32 # matches ateapi.Container.volume_mounts + // +k8s:listType=map + // +k8s:listMapKey=mount_path ImageVolumeMounts []*VolumeMount `protobuf:"bytes,6,rep,name=image_volume_mounts,json=imageVolumeMounts,proto3" json:"image_volume_mounts,omitempty"` // container_spec is what the worker builds the container's OCI spec from, - // besides its volume mounts. Unset on RPCs that start no process. + // besides its volume mounts. Unset on RPCs that start no process. atelet + // derives it from template fields it validated (argv resolved against the + // image's ENTRYPOINT and CMD, env merged with the image's ENV), so the + // template's bounds do not apply to it and it is not validated here. + // + // +k8s:optional + // +k8s:opaqueType # derived by atelet, see above ContainerSpec *ContainerSpec `protobuf:"bytes,7,opt,name=container_spec,json=containerSpec,proto3" json:"container_spec,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -881,8 +1002,15 @@ type VolumeMount struct { // name is the name the ActorTemplate gave the volume. For durable-dir and // system-info volumes it selects the per-volume directory atelet prepared // for the actor on the host. + // + // +k8s:required + // +k8s:format=k8s-short-name Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // mount_path is where the container sees the volume. + // + // +k8s:required + // +k8s:maxLength=4096 + // +k8s:customValidation # clean-absolute-path shape; no regex/pattern tag exists MountPath string `protobuf:"bytes,2,opt,name=mount_path,json=mountPath,proto3" json:"mount_path,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -935,9 +1063,14 @@ func (x *VolumeMount) GetMountPath() string { // WakeupProbe describes how to check that a container is ready to serve. // Only HTTP is supported today. type WakeupProbe struct { - state protoimpl.MessageState `protogen:"open.v1"` - HttpGet *HttpGetAction `protobuf:"bytes,1,opt,name=http_get,json=httpGet,proto3" json:"http_get,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + // +k8s:required + HttpGet *HttpGetAction `protobuf:"bytes,1,opt,name=http_get,json=httpGet,proto3" json:"http_get,omitempty"` // How long to keep polling before giving up and failing the actor start. + // + // +k8s:required + // +k8s:minimum=1 + // +k8s:maximum=3600 # matches ateapi.WakeupProbe.timeout_seconds TimeoutSeconds int32 `protobuf:"varint,2,opt,name=timeout_seconds,json=timeoutSeconds,proto3" json:"timeout_seconds,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -991,8 +1124,16 @@ func (x *WakeupProbe) GetTimeoutSeconds() int32 { type HttpGetAction struct { state protoimpl.MessageState `protogen:"open.v1"` // Path to access on the HTTP server. + // + // +k8s:required + // +k8s:maxLength=1024 # matches ateapi.HttpGetAction.path + // +k8s:customValidation # RFC 3986 path shape; no regex/pattern tag exists Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` // TCP port to connect to (1..65535). + // + // +k8s:required + // +k8s:minimum=1 + // +k8s:maximum=65535 Port int32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -1079,31 +1220,37 @@ func (*RunWorkloadResponse) Descriptor() ([]byte, []int) { } type CheckpointWorkloadRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - Atespace string `protobuf:"bytes,1,opt,name=atespace,proto3" json:"atespace,omitempty"` - ActorName string `protobuf:"bytes,2,opt,name=actor_name,json=actorName,proto3" json:"actor_name,omitempty"` - ActorUid string `protobuf:"bytes,3,opt,name=actor_uid,json=actorUid,proto3" json:"actor_uid,omitempty"` - ActorTemplateAtespace string `protobuf:"bytes,4,opt,name=actor_template_atespace,json=actorTemplateAtespace,proto3" json:"actor_template_atespace,omitempty"` - ActorTemplateName string `protobuf:"bytes,5,opt,name=actor_template_name,json=actorTemplateName,proto3" json:"actor_template_name,omitempty"` - RunscPath string `protobuf:"bytes,6,opt,name=runsc_path,json=runscPath,proto3" json:"runsc_path,omitempty"` - Spec *WorkloadSpec `protobuf:"bytes,7,opt,name=spec,proto3" json:"spec,omitempty"` - // The object storage URI of the snapshot to write. Object names are appended - // to it, so it addresses the snapshot as a whole rather than any one object. - // - // The structure of the checkpoint should generally be treated as opaque. For - // gVisor, the checkpoint consists of a checkpoint.img file that contains the - // memory, sentry state, and filesystem deltas. + state protoimpl.MessageState `protogen:"open.v1"` + // +k8s:required + // +k8s:format=k8s-short-name + Atespace string `protobuf:"bytes,1,opt,name=atespace,proto3" json:"atespace,omitempty"` + // +k8s:required + // +k8s:format=k8s-short-name + ActorName string `protobuf:"bytes,2,opt,name=actor_name,json=actorName,proto3" json:"actor_name,omitempty"` + // +k8s:required + // +k8s:format=k8s-uuid + ActorUid string `protobuf:"bytes,3,opt,name=actor_uid,json=actorUid,proto3" json:"actor_uid,omitempty"` + // +k8s:optional + // +k8s:format=k8s-short-name + ActorTemplateAtespace string `protobuf:"bytes,4,opt,name=actor_template_atespace,json=actorTemplateAtespace,proto3" json:"actor_template_atespace,omitempty"` + // +k8s:optional + // +k8s:format=k8s-short-name + ActorTemplateName string `protobuf:"bytes,5,opt,name=actor_template_name,json=actorTemplateName,proto3" json:"actor_template_name,omitempty"` + // Empty for runtimes that do not use runsc. // - // The control plane sends an ActorSnapshot's snapshot_uri, which sits under - // the prefix of the resource that owns the snapshot. For example: - // "gs://bucket/root/atespaces/team-a/actors//snapshots/5678". - SnapshotUri string `protobuf:"bytes,8,opt,name=snapshot_uri,json=snapshotUri,proto3" json:"snapshot_uri,omitempty"` - // runtime_asset_paths maps a runtime asset name to the local on-disk path - // atelet fetched it to (see RunWorkloadRequest). Empty for gVisor. - RuntimeAssetPaths map[string]string `protobuf:"bytes,9,rep,name=runtime_asset_paths,json=runtimeAssetPaths,proto3" json:"runtime_asset_paths,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // +k8s:optional + // +k8s:maxLength=4096 # PATH_MAX; a host path, not a contract + RunscPath string `protobuf:"bytes,6,opt,name=runsc_path,json=runscPath,proto3" json:"runsc_path,omitempty"` + // +k8s:required + Spec *WorkloadSpec `protobuf:"bytes,7,opt,name=spec,proto3" json:"spec,omitempty"` // Fidelity the checkpoint captures. - Fidelity SnapshotFidelity `protobuf:"varint,10,opt,name=fidelity,proto3,enum=ateworker.SnapshotFidelity" json:"fidelity,omitempty"` - ActorDirs *ActorDirs `protobuf:"bytes,11,opt,name=actor_dirs,json=actorDirs,proto3" json:"actor_dirs,omitempty"` + // + // +k8s:required + // +k8s:minimum=1 + // +k8s:maximum=3 # keep this in sync with the SnapshotFidelity enum + Fidelity SnapshotFidelity `protobuf:"varint,10,opt,name=fidelity,proto3,enum=ateworker.SnapshotFidelity" json:"fidelity,omitempty"` + // +k8s:required + ActorDirs *ActorDirs `protobuf:"bytes,11,opt,name=actor_dirs,json=actorDirs,proto3" json:"actor_dirs,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1187,20 +1334,6 @@ func (x *CheckpointWorkloadRequest) GetSpec() *WorkloadSpec { return nil } -func (x *CheckpointWorkloadRequest) GetSnapshotUri() string { - if x != nil { - return x.SnapshotUri - } - return "" -} - -func (x *CheckpointWorkloadRequest) GetRuntimeAssetPaths() map[string]string { - if x != nil { - return x.RuntimeAssetPaths - } - return nil -} - func (x *CheckpointWorkloadRequest) GetFidelity() SnapshotFidelity { if x != nil { return x.Fidelity @@ -1273,33 +1406,66 @@ func (x *CheckpointWorkloadResponse) GetDataSnapshotFiles() []string { } type RestoreWorkloadRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - Atespace string `protobuf:"bytes,1,opt,name=atespace,proto3" json:"atespace,omitempty"` - ActorName string `protobuf:"bytes,2,opt,name=actor_name,json=actorName,proto3" json:"actor_name,omitempty"` - ActorUid string `protobuf:"bytes,3,opt,name=actor_uid,json=actorUid,proto3" json:"actor_uid,omitempty"` - ActorTemplateAtespace string `protobuf:"bytes,4,opt,name=actor_template_atespace,json=actorTemplateAtespace,proto3" json:"actor_template_atespace,omitempty"` - ActorTemplateName string `protobuf:"bytes,5,opt,name=actor_template_name,json=actorTemplateName,proto3" json:"actor_template_name,omitempty"` - RunscPath string `protobuf:"bytes,6,opt,name=runsc_path,json=runscPath,proto3" json:"runsc_path,omitempty"` - Spec *WorkloadSpec `protobuf:"bytes,7,opt,name=spec,proto3" json:"spec,omitempty"` - // The object storage URI of the snapshot to restore. Object names are - // appended to it; it addresses the snapshot, not any one object. - SnapshotUri string `protobuf:"bytes,8,opt,name=snapshot_uri,json=snapshotUri,proto3" json:"snapshot_uri,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + // +k8s:required + // +k8s:format=k8s-short-name + Atespace string `protobuf:"bytes,1,opt,name=atespace,proto3" json:"atespace,omitempty"` + // +k8s:required + // +k8s:format=k8s-short-name + ActorName string `protobuf:"bytes,2,opt,name=actor_name,json=actorName,proto3" json:"actor_name,omitempty"` + // +k8s:required + // +k8s:format=k8s-uuid + ActorUid string `protobuf:"bytes,3,opt,name=actor_uid,json=actorUid,proto3" json:"actor_uid,omitempty"` + // +k8s:optional + // +k8s:format=k8s-short-name + ActorTemplateAtespace string `protobuf:"bytes,4,opt,name=actor_template_atespace,json=actorTemplateAtespace,proto3" json:"actor_template_atespace,omitempty"` + // +k8s:optional + // +k8s:format=k8s-short-name + ActorTemplateName string `protobuf:"bytes,5,opt,name=actor_template_name,json=actorTemplateName,proto3" json:"actor_template_name,omitempty"` + // Empty for runtimes that do not use runsc. + // + // +k8s:optional + // +k8s:maxLength=4096 # PATH_MAX; a host path, not a contract + RunscPath string `protobuf:"bytes,6,opt,name=runsc_path,json=runscPath,proto3" json:"runsc_path,omitempty"` + // +k8s:required + // +k8s:subfield(containers)=+k8s:required # a container to start; only Terminate's fallback spec may have none + Spec *WorkloadSpec `protobuf:"bytes,7,opt,name=spec,proto3" json:"spec,omitempty"` // runtime_asset_paths maps a runtime asset name to the local on-disk path // atelet fetched it to (see RunWorkloadRequest). Empty for gVisor. + // + // +k8s:optional + // +k8s:maxProperties=16 # matches atelet.ArchAssets.files + // +k8s:eachKey=+k8s:maxLength=64 # matches atelet.ArchAssets.files + // +k8s:eachVal=+k8s:maxLength=4096 # PATH_MAX; a host path, not a contract RuntimeAssetPaths map[string]string `protobuf:"bytes,9,rep,name=runtime_asset_paths,json=runtimeAssetPaths,proto3" json:"runtime_asset_paths,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // Fidelity to restore from the snapshot. + // + // +k8s:required + // +k8s:minimum=1 + // +k8s:maximum=3 # keep this in sync with the SnapshotFidelity enum Fidelity SnapshotFidelity `protobuf:"varint,10,opt,name=fidelity,proto3,enum=ateworker.SnapshotFidelity" json:"fidelity,omitempty"` // When absent the actor has no egress: its TCP is captured and refused. + // + // +k8s:optional EgressGateway *EgressGateway `protobuf:"bytes,12,opt,name=egress_gateway,json=egressGateway,proto3,oneof" json:"egress_gateway,omitempty"` // The actor's declared size, from the ActorTemplate's resource limits. Used to // (re)size the sandbox on a VOLUMES restore (fresh guest container). On a // MEMORY micro-VM restore the size baked into the snapshot is authoritative and // these are ignored. Zero means "unset": keep the runtime default. - CpuMilli int64 `protobuf:"varint,14,opt,name=cpu_milli,json=cpuMilli,proto3" json:"cpu_milli,omitempty"` // CPU limit in millicores (1000 = one core). - MemoryBytes int64 `protobuf:"varint,15,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` // Memory limit in bytes. - ActorDirs *ActorDirs `protobuf:"bytes,16,opt,name=actor_dirs,json=actorDirs,proto3" json:"actor_dirs,omitempty"` + // + // +k8s:optional + // +k8s:minimum=1 + // +k8s:maximum=999999 # matches atelet.RestoreRequest.cpu_milli, the control plane caps cpu limits strictly below 1000 cores + CpuMilli int64 `protobuf:"varint,14,opt,name=cpu_milli,json=cpuMilli,proto3" json:"cpu_milli,omitempty"` // CPU limit in millicores (1000 = one core). + // +k8s:optional + // +k8s:minimum=1 + MemoryBytes int64 `protobuf:"varint,15,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` // Memory limit in bytes. + // +k8s:required + ActorDirs *ActorDirs `protobuf:"bytes,16,opt,name=actor_dirs,json=actorDirs,proto3" json:"actor_dirs,omitempty"` // actor_dirs.restore_dir is a preserved local snapshot, not a scratch copy: // the worker must not delete or modify files in it. + // + // +k8s:optional PreserveRestoreDir bool `protobuf:"varint,17,opt,name=preserve_restore_dir,json=preserveRestoreDir,proto3" json:"preserve_restore_dir,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -1384,13 +1550,6 @@ func (x *RestoreWorkloadRequest) GetSpec() *WorkloadSpec { return nil } -func (x *RestoreWorkloadRequest) GetSnapshotUri() string { - if x != nil { - return x.SnapshotUri - } - return "" -} - func (x *RestoreWorkloadRequest) GetRuntimeAssetPaths() map[string]string { if x != nil { return x.RuntimeAssetPaths @@ -1482,6 +1641,9 @@ type GetWorkloadStatsRequest struct { // between the caller's view of the world and this call, so the worker answers // NOT_FOUND on a mismatch rather than reporting a different actor's numbers // under the requested actor's identity. + // + // +k8s:required + // +k8s:format=k8s-uuid ActorUid string `protobuf:"bytes,1,opt,name=actor_uid,json=actorUid,proto3" json:"actor_uid,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -1912,7 +2074,7 @@ const file_ateworker_proto_rawDesc = "" + "\rHttpGetAction\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x12\n" + "\x04port\x18\x02 \x01(\x05R\x04port\"\x15\n" + - "\x13RunWorkloadResponse\"\xeb\x04\n" + + "\x13RunWorkloadResponse\"\x95\x03\n" + "\x19CheckpointWorkloadRequest\x12\x1a\n" + "\batespace\x18\x01 \x01(\tR\batespace\x12\x1d\n" + "\n" + @@ -1922,19 +2084,14 @@ const file_ateworker_proto_rawDesc = "" + "\x13actor_template_name\x18\x05 \x01(\tR\x11actorTemplateName\x12\x1d\n" + "\n" + "runsc_path\x18\x06 \x01(\tR\trunscPath\x12+\n" + - "\x04spec\x18\a \x01(\v2\x17.ateworker.WorkloadSpecR\x04spec\x12!\n" + - "\fsnapshot_uri\x18\b \x01(\tR\vsnapshotUri\x12k\n" + - "\x13runtime_asset_paths\x18\t \x03(\v2;.ateworker.CheckpointWorkloadRequest.RuntimeAssetPathsEntryR\x11runtimeAssetPaths\x127\n" + + "\x04spec\x18\a \x01(\v2\x17.ateworker.WorkloadSpecR\x04spec\x127\n" + "\bfidelity\x18\n" + " \x01(\x0e2\x1b.ateworker.SnapshotFidelityR\bfidelity\x123\n" + "\n" + - "actor_dirs\x18\v \x01(\v2\x14.ateworker.ActorDirsR\tactorDirs\x1aD\n" + - "\x16RuntimeAssetPathsEntry\x12\x10\n" + - "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"s\n" + + "actor_dirs\x18\v \x01(\v2\x14.ateworker.ActorDirsR\tactorDirs\"s\n" + "\x1aCheckpointWorkloadResponse\x12%\n" + "\x0esnapshot_files\x18\x01 \x03(\tR\rsnapshotFiles\x12.\n" + - "\x13data_snapshot_files\x18\x02 \x03(\tR\x11dataSnapshotFiles\"\xb0\x06\n" + + "\x13data_snapshot_files\x18\x02 \x03(\tR\x11dataSnapshotFiles\"\x8d\x06\n" + "\x16RestoreWorkloadRequest\x12\x1a\n" + "\batespace\x18\x01 \x01(\tR\batespace\x12\x1d\n" + "\n" + @@ -1944,8 +2101,7 @@ const file_ateworker_proto_rawDesc = "" + "\x13actor_template_name\x18\x05 \x01(\tR\x11actorTemplateName\x12\x1d\n" + "\n" + "runsc_path\x18\x06 \x01(\tR\trunscPath\x12+\n" + - "\x04spec\x18\a \x01(\v2\x17.ateworker.WorkloadSpecR\x04spec\x12!\n" + - "\fsnapshot_uri\x18\b \x01(\tR\vsnapshotUri\x12h\n" + + "\x04spec\x18\a \x01(\v2\x17.ateworker.WorkloadSpecR\x04spec\x12h\n" + "\x13runtime_asset_paths\x18\t \x03(\v28.ateworker.RestoreWorkloadRequest.RuntimeAssetPathsEntryR\x11runtimeAssetPaths\x127\n" + "\bfidelity\x18\n" + " \x01(\x0e2\x1b.ateworker.SnapshotFidelityR\bfidelity\x12D\n" + @@ -2013,7 +2169,7 @@ func file_ateworker_proto_rawDescGZIP() []byte { } var file_ateworker_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_ateworker_proto_msgTypes = make([]protoimpl.MessageInfo, 25) +var file_ateworker_proto_msgTypes = make([]protoimpl.MessageInfo, 24) var file_ateworker_proto_goTypes = []any{ (SnapshotFidelity)(0), // 0: ateworker.SnapshotFidelity (StatsSource)(0), // 1: ateworker.StatsSource @@ -2040,8 +2196,7 @@ var file_ateworker_proto_goTypes = []any{ (*GetActiveWorkloadStatsRequest)(nil), // 22: ateworker.GetActiveWorkloadStatsRequest (*GetActiveWorkloadStatsResponse)(nil), // 23: ateworker.GetActiveWorkloadStatsResponse nil, // 24: ateworker.RunWorkloadRequest.RuntimeAssetPathsEntry - nil, // 25: ateworker.CheckpointWorkloadRequest.RuntimeAssetPathsEntry - nil, // 26: ateworker.RestoreWorkloadRequest.RuntimeAssetPathsEntry + nil, // 25: ateworker.RestoreWorkloadRequest.RuntimeAssetPathsEntry } var file_ateworker_proto_depIdxs = []int32{ 7, // 0: ateworker.TerminateWorkloadRequest.spec:type_name -> ateworker.WorkloadSpec @@ -2060,34 +2215,33 @@ var file_ateworker_proto_depIdxs = []int32{ 10, // 13: ateworker.ContainerSpec.resources:type_name -> ateworker.ResourceLimits 13, // 14: ateworker.WakeupProbe.http_get:type_name -> ateworker.HttpGetAction 7, // 15: ateworker.CheckpointWorkloadRequest.spec:type_name -> ateworker.WorkloadSpec - 25, // 16: ateworker.CheckpointWorkloadRequest.runtime_asset_paths:type_name -> ateworker.CheckpointWorkloadRequest.RuntimeAssetPathsEntry - 0, // 17: ateworker.CheckpointWorkloadRequest.fidelity:type_name -> ateworker.SnapshotFidelity - 2, // 18: ateworker.CheckpointWorkloadRequest.actor_dirs:type_name -> ateworker.ActorDirs - 7, // 19: ateworker.RestoreWorkloadRequest.spec:type_name -> ateworker.WorkloadSpec - 26, // 20: ateworker.RestoreWorkloadRequest.runtime_asset_paths:type_name -> ateworker.RestoreWorkloadRequest.RuntimeAssetPathsEntry - 0, // 21: ateworker.RestoreWorkloadRequest.fidelity:type_name -> ateworker.SnapshotFidelity - 6, // 22: ateworker.RestoreWorkloadRequest.egress_gateway:type_name -> ateworker.EgressGateway - 2, // 23: ateworker.RestoreWorkloadRequest.actor_dirs:type_name -> ateworker.ActorDirs - 1, // 24: ateworker.WorkloadStatsSample.source:type_name -> ateworker.StatsSource - 20, // 25: ateworker.GetWorkloadStatsResponse.sample:type_name -> ateworker.WorkloadStatsSample - 20, // 26: ateworker.GetActiveWorkloadStatsResponse.samples:type_name -> ateworker.WorkloadStatsSample - 5, // 27: ateworker.Worker.RunWorkload:input_type -> ateworker.RunWorkloadRequest - 15, // 28: ateworker.Worker.CheckpointWorkload:input_type -> ateworker.CheckpointWorkloadRequest - 17, // 29: ateworker.Worker.RestoreWorkload:input_type -> ateworker.RestoreWorkloadRequest - 19, // 30: ateworker.Worker.GetWorkloadStats:input_type -> ateworker.GetWorkloadStatsRequest - 22, // 31: ateworker.Worker.GetActiveWorkloadStats:input_type -> ateworker.GetActiveWorkloadStatsRequest - 3, // 32: ateworker.Worker.TerminateWorkload:input_type -> ateworker.TerminateWorkloadRequest - 14, // 33: ateworker.Worker.RunWorkload:output_type -> ateworker.RunWorkloadResponse - 16, // 34: ateworker.Worker.CheckpointWorkload:output_type -> ateworker.CheckpointWorkloadResponse - 18, // 35: ateworker.Worker.RestoreWorkload:output_type -> ateworker.RestoreWorkloadResponse - 21, // 36: ateworker.Worker.GetWorkloadStats:output_type -> ateworker.GetWorkloadStatsResponse - 23, // 37: ateworker.Worker.GetActiveWorkloadStats:output_type -> ateworker.GetActiveWorkloadStatsResponse - 4, // 38: ateworker.Worker.TerminateWorkload:output_type -> ateworker.TerminateWorkloadResponse - 33, // [33:39] is the sub-list for method output_type - 27, // [27:33] is the sub-list for method input_type - 27, // [27:27] is the sub-list for extension type_name - 27, // [27:27] is the sub-list for extension extendee - 0, // [0:27] is the sub-list for field type_name + 0, // 16: ateworker.CheckpointWorkloadRequest.fidelity:type_name -> ateworker.SnapshotFidelity + 2, // 17: ateworker.CheckpointWorkloadRequest.actor_dirs:type_name -> ateworker.ActorDirs + 7, // 18: ateworker.RestoreWorkloadRequest.spec:type_name -> ateworker.WorkloadSpec + 25, // 19: ateworker.RestoreWorkloadRequest.runtime_asset_paths:type_name -> ateworker.RestoreWorkloadRequest.RuntimeAssetPathsEntry + 0, // 20: ateworker.RestoreWorkloadRequest.fidelity:type_name -> ateworker.SnapshotFidelity + 6, // 21: ateworker.RestoreWorkloadRequest.egress_gateway:type_name -> ateworker.EgressGateway + 2, // 22: ateworker.RestoreWorkloadRequest.actor_dirs:type_name -> ateworker.ActorDirs + 1, // 23: ateworker.WorkloadStatsSample.source:type_name -> ateworker.StatsSource + 20, // 24: ateworker.GetWorkloadStatsResponse.sample:type_name -> ateworker.WorkloadStatsSample + 20, // 25: ateworker.GetActiveWorkloadStatsResponse.samples:type_name -> ateworker.WorkloadStatsSample + 5, // 26: ateworker.Worker.RunWorkload:input_type -> ateworker.RunWorkloadRequest + 15, // 27: ateworker.Worker.CheckpointWorkload:input_type -> ateworker.CheckpointWorkloadRequest + 17, // 28: ateworker.Worker.RestoreWorkload:input_type -> ateworker.RestoreWorkloadRequest + 19, // 29: ateworker.Worker.GetWorkloadStats:input_type -> ateworker.GetWorkloadStatsRequest + 22, // 30: ateworker.Worker.GetActiveWorkloadStats:input_type -> ateworker.GetActiveWorkloadStatsRequest + 3, // 31: ateworker.Worker.TerminateWorkload:input_type -> ateworker.TerminateWorkloadRequest + 14, // 32: ateworker.Worker.RunWorkload:output_type -> ateworker.RunWorkloadResponse + 16, // 33: ateworker.Worker.CheckpointWorkload:output_type -> ateworker.CheckpointWorkloadResponse + 18, // 34: ateworker.Worker.RestoreWorkload:output_type -> ateworker.RestoreWorkloadResponse + 21, // 35: ateworker.Worker.GetWorkloadStats:output_type -> ateworker.GetWorkloadStatsResponse + 23, // 36: ateworker.Worker.GetActiveWorkloadStats:output_type -> ateworker.GetActiveWorkloadStatsResponse + 4, // 37: ateworker.Worker.TerminateWorkload:output_type -> ateworker.TerminateWorkloadResponse + 32, // [32:38] is the sub-list for method output_type + 26, // [26:32] is the sub-list for method input_type + 26, // [26:26] is the sub-list for extension type_name + 26, // [26:26] is the sub-list for extension extendee + 0, // [0:26] is the sub-list for field type_name } func init() { file_ateworker_proto_init() } @@ -2103,7 +2257,7 @@ func file_ateworker_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_ateworker_proto_rawDesc), len(file_ateworker_proto_rawDesc)), NumEnums: 2, - NumMessages: 25, + NumMessages: 24, NumExtensions: 0, NumServices: 1, }, diff --git a/internal/proto/ateworkerpb/ateworker.proto b/internal/proto/ateworkerpb/ateworker.proto index 3a51bc76d0..4e07f820f7 100644 --- a/internal/proto/ateworkerpb/ateworker.proto +++ b/internal/proto/ateworkerpb/ateworker.proto @@ -109,37 +109,81 @@ service Worker { // it shares with the worker. The worker takes them from the RPC instead of deriving // them from the actor UID. // Every directory is under root_dir. +// +// +k8s:customValidation # each directory is an absolute, clean path message ActorDirs { // The actor's own directory. + // + // +k8s:required + // +k8s:maxLength=4096 # PATH_MAX string root_dir = 1; // Holds one OCI bundle per container, named after the container. + // + // +k8s:required + // +k8s:maxLength=4096 # PATH_MAX string oci_bundle_dir = 2; // Where CheckpointWorkload writes the snapshot files. + // + // +k8s:required + // +k8s:maxLength=4096 # PATH_MAX string checkpoint_dir = 3; // Where the snapshot files RestoreWorkload reads have been downloaded to. + // + // +k8s:required + // +k8s:maxLength=4096 # PATH_MAX string restore_dir = 4; // Holds one subdirectory per durable-dir volume. + // + // +k8s:required + // +k8s:maxLength=4096 # PATH_MAX string durable_dir_volume_mounts_dir = 5; // Holds one subdirectory per system-info volume. + // + // +k8s:required + // +k8s:maxLength=4096 # PATH_MAX string system_info_volume_roots_dir = 6; // Holds one mount point per external volume. + // + // +k8s:required + // +k8s:maxLength=4096 # PATH_MAX string external_volume_mounts_dir = 7; // Holds the sandbox runtime assets staged for this actor (runsc, or the // micro-VM kernel, VMM and guest image), hard-linked from atelet's asset // cache. The only location a sandbox runtime may run a binary from; the // worker derives the same path from actor_uid and rejects a request // naming any other. + // + // +k8s:required + // +k8s:maxLength=4096 # PATH_MAX string sandbox_assets_dir = 8; } message TerminateWorkloadRequest { + // +k8s:required + // +k8s:format=k8s-short-name string atespace = 1; + // +k8s:required + // +k8s:format=k8s-short-name string actor_name = 2; + // +k8s:required + // +k8s:format=k8s-uuid string actor_uid = 3; + // +k8s:optional + // +k8s:format=k8s-short-name string actor_template_atespace = 4; + // +k8s:optional + // +k8s:format=k8s-short-name string actor_template_name = 5; + // Empty for runtimes that do not use runsc. + // + // +k8s:optional + // +k8s:maxLength=4096 # PATH_MAX; a host path, not a contract string runsc_path = 6; + // Names the containers to stop. + // + // +k8s:required WorkloadSpec spec = 7; + // +k8s:required ActorDirs actor_dirs = 8; } @@ -147,31 +191,62 @@ message TerminateWorkloadResponse { } message RunWorkloadRequest { + // +k8s:required + // +k8s:format=k8s-short-name string atespace = 1; + // +k8s:required + // +k8s:format=k8s-short-name string actor_name = 2; + // +k8s:required + // +k8s:format=k8s-uuid string actor_uid = 3; + // +k8s:optional + // +k8s:format=k8s-short-name string actor_template_atespace = 4; + // +k8s:optional + // +k8s:format=k8s-short-name string actor_template_name = 5; + // Empty for runtimes that do not use runsc. + // + // +k8s:optional + // +k8s:maxLength=4096 # PATH_MAX; a host path, not a contract string runsc_path = 6; + // +k8s:required + // +k8s:subfield(containers)=+k8s:required # a container to start; only Terminate's fallback spec may have none WorkloadSpec spec = 7; - // runtime_asset_paths maps a runtime asset name (e.g. "cloud-hypervisor", - // "virtiofsd", "kata-kernel", "kata-image") - // to the local on-disk path atelet fetched it to (content-addressed, like - // runsc_path). Empty for the gVisor runtime, which uses runsc_path. + // runtime_asset_paths maps a runtime asset name to the local on-disk path + // atelet fetched it to (content-addressed, like runsc_path). The names are + // the runtime's own: "gvisor" for gVisor, "cloud-hypervisor", + // "virtiofsd", "kata-kernel", and "kata-image" for the micro-VM. Empty for + // the gVisor runtime, which uses runsc_path. + // + // +k8s:optional + // +k8s:maxProperties=16 # matches atelet.ArchAssets.files + // +k8s:eachKey=+k8s:maxLength=64 # matches atelet.ArchAssets.files + // +k8s:eachVal=+k8s:maxLength=4096 # PATH_MAX; a host path, not a contract map runtime_asset_paths = 8; // When absent the actor has no egress: its TCP is captured and refused. + // + // +k8s:optional optional EgressGateway egress_gateway = 10; // The actor's declared size, from the ActorTemplate's resource limits. The worker // sizes the sandbox to these (cgroup caps via the OCI spec, and for the // micro-VM the VM's vCPU count and memory). Zero means "unset": keep the // runtime default (unlimited for gVisor, the worker's own default for the micro-VM). - int64 cpu_milli = 11; // CPU limit in millicores (1000 = one core). + // + // +k8s:optional + // +k8s:minimum=1 + // +k8s:maximum=999999 # matches atelet.RunRequest.cpu_milli, the control plane caps cpu limits strictly below 1000 cores + int64 cpu_milli = 11; // CPU limit in millicores (1000 = one core). + // +k8s:optional + // +k8s:minimum=1 int64 memory_bytes = 12; // Memory limit in bytes. + // +k8s:required ActorDirs actor_dirs = 13; } @@ -179,37 +254,84 @@ message RunWorkloadRequest { message EgressGateway { // address is required and identifies the remote gateway as an IP address or // DNS name followed by its port. + // + // +k8s:required + // +k8s:maxLength=261 # a bracketed or 253-character host, ':' and a 5-digit port + // +k8s:customValidation # host:port shape string address = 1; } // WorkloadSpec parallels Pod, but with far fewer configurable fields. message WorkloadSpec { + // Empty on the fallback spec atelet's Terminate sends for an actor whose + // template is gone, as on atelet's own WorkloadSpec. + // + // +k8s:optional + // +k8s:maxItems=10 # matches ateapi.ActorTemplate.containers + // +k8s:listType=map + // +k8s:listMapKey=name repeated Container containers = 1; } +// A container's mounts split into four typed lists. Each list's key keeps +// only that list's paths unique, so the rules that span the lists are a hook. +// +// +k8s:customValidation # mount paths are unique and do not nest across the four lists message Container { + // The sandbox's own container, ocispec.PauseContainer, is not a short + // name, so the format keeps a workload container from taking its bundle. + // + // +k8s:required + // +k8s:format=k8s-short-name string name = 1; + // +k8s:optional WakeupProbe wakeup_probe = 2; // durable_dir_volume_mounts are the durable-dir volumes this container // mounts, if any. + // + // +k8s:optional + // +k8s:maxItems=32 # matches ateapi.Container.volume_mounts + // +k8s:listType=map + // +k8s:listMapKey=mount_path repeated VolumeMount durable_dir_volume_mounts = 3; // external_volume_mounts are the external volumes this container mounts, if // any. + // + // +k8s:optional + // +k8s:maxItems=32 # matches ateapi.Container.volume_mounts + // +k8s:listType=map + // +k8s:listMapKey=mount_path repeated VolumeMount external_volume_mounts = 4; // system_info_volume_mounts are the system-info volumes this container // mounts, if any. Contents are generated by atelet on the host on every // Run/Restore and never captured into snapshots. The container sees them // read-only. + // + // +k8s:optional + // +k8s:maxItems=32 # matches ateapi.Container.volume_mounts + // +k8s:listType=map + // +k8s:listMapKey=mount_path repeated VolumeMount system_info_volume_mounts = 5; // image_volume_mounts are the image volumes this container mounts, if any. + // + // +k8s:optional + // +k8s:maxItems=32 # matches ateapi.Container.volume_mounts + // +k8s:listType=map + // +k8s:listMapKey=mount_path repeated VolumeMount image_volume_mounts = 6; // container_spec is what the worker builds the container's OCI spec from, - // besides its volume mounts. Unset on RPCs that start no process. + // besides its volume mounts. Unset on RPCs that start no process. atelet + // derives it from template fields it validated (argv resolved against the + // image's ENTRYPOINT and CMD, env merged with the image's ENV), so the + // template's bounds do not apply to it and it is not validated here. + // + // +k8s:optional + // +k8s:opaqueType # derived by atelet, see above ContainerSpec container_spec = 7; } @@ -243,24 +365,46 @@ message VolumeMount { // name is the name the ActorTemplate gave the volume. For durable-dir and // system-info volumes it selects the per-volume directory atelet prepared // for the actor on the host. + // + // +k8s:required + // +k8s:format=k8s-short-name string name = 1; // mount_path is where the container sees the volume. + // + // +k8s:required + // +k8s:maxLength=4096 + // +k8s:customValidation # clean-absolute-path shape; no regex/pattern tag exists string mount_path = 2; } // WakeupProbe describes how to check that a container is ready to serve. // Only HTTP is supported today. message WakeupProbe { + // +k8s:required HttpGetAction http_get = 1; + // How long to keep polling before giving up and failing the actor start. + // + // +k8s:required + // +k8s:minimum=1 + // +k8s:maximum=3600 # matches ateapi.WakeupProbe.timeout_seconds int32 timeout_seconds = 2; } // HttpGetAction performs an HTTP GET against the container. message HttpGetAction { // Path to access on the HTTP server. + // + // +k8s:required + // +k8s:maxLength=1024 # matches ateapi.HttpGetAction.path + // +k8s:customValidation # RFC 3986 path shape; no regex/pattern tag exists string path = 1; + // TCP port to connect to (1..65535). + // + // +k8s:required + // +k8s:minimum=1 + // +k8s:maximum=65535 int32 port = 2; } @@ -276,43 +420,47 @@ enum SnapshotFidelity { // (currently DurableDir-typed volumes). Memory and the rest of rootfs are // excluded. SNAPSHOT_FIDELITY_VOLUMES = 1; - // Volumes plus the root filesystem changes made since boot. Served by the - // micro-VM runtime only; other runtimes reject requests carrying it. + // Volumes plus the root filesystem changes made since boot. SNAPSHOT_FIDELITY_ROOTFS = 2; // Capture process memory plus the full filesystem delta on top of the OCI // image (including any attached DurableDir volumes). SNAPSHOT_FIDELITY_MEMORY = 3; + // Keep this in sync with the maximums on fields of this type. } message CheckpointWorkloadRequest { + // +k8s:required + // +k8s:format=k8s-short-name string atespace = 1; + // +k8s:required + // +k8s:format=k8s-short-name string actor_name = 2; + // +k8s:required + // +k8s:format=k8s-uuid string actor_uid = 3; + // +k8s:optional + // +k8s:format=k8s-short-name string actor_template_atespace = 4; + // +k8s:optional + // +k8s:format=k8s-short-name string actor_template_name = 5; + // Empty for runtimes that do not use runsc. + // + // +k8s:optional + // +k8s:maxLength=4096 # PATH_MAX; a host path, not a contract string runsc_path = 6; + // +k8s:required WorkloadSpec spec = 7; - // The object storage URI of the snapshot to write. Object names are appended - // to it, so it addresses the snapshot as a whole rather than any one object. - // - // The structure of the checkpoint should generally be treated as opaque. For - // gVisor, the checkpoint consists of a checkpoint.img file that contains the - // memory, sentry state, and filesystem deltas. - // - // The control plane sends an ActorSnapshot's snapshot_uri, which sits under - // the prefix of the resource that owns the snapshot. For example: - // "gs://bucket/root/atespaces/team-a/actors//snapshots/5678". - string snapshot_uri = 8; - - // runtime_asset_paths maps a runtime asset name to the local on-disk path - // atelet fetched it to (see RunWorkloadRequest). Empty for gVisor. - map runtime_asset_paths = 9; - // Fidelity the checkpoint captures. + // + // +k8s:required + // +k8s:minimum=1 + // +k8s:maximum=3 # keep this in sync with the SnapshotFidelity enum SnapshotFidelity fidelity = 10; + // +k8s:required ActorDirs actor_dirs = 11; } @@ -328,40 +476,72 @@ message CheckpointWorkloadResponse { } message RestoreWorkloadRequest { + // +k8s:required + // +k8s:format=k8s-short-name string atespace = 1; + // +k8s:required + // +k8s:format=k8s-short-name string actor_name = 2; + // +k8s:required + // +k8s:format=k8s-uuid string actor_uid = 3; + // +k8s:optional + // +k8s:format=k8s-short-name string actor_template_atespace = 4; + // +k8s:optional + // +k8s:format=k8s-short-name string actor_template_name = 5; + // Empty for runtimes that do not use runsc. + // + // +k8s:optional + // +k8s:maxLength=4096 # PATH_MAX; a host path, not a contract string runsc_path = 6; + // +k8s:required + // +k8s:subfield(containers)=+k8s:required # a container to start; only Terminate's fallback spec may have none WorkloadSpec spec = 7; - // The object storage URI of the snapshot to restore. Object names are - // appended to it; it addresses the snapshot, not any one object. - string snapshot_uri = 8; - // runtime_asset_paths maps a runtime asset name to the local on-disk path // atelet fetched it to (see RunWorkloadRequest). Empty for gVisor. + // + // +k8s:optional + // +k8s:maxProperties=16 # matches atelet.ArchAssets.files + // +k8s:eachKey=+k8s:maxLength=64 # matches atelet.ArchAssets.files + // +k8s:eachVal=+k8s:maxLength=4096 # PATH_MAX; a host path, not a contract map runtime_asset_paths = 9; // Fidelity to restore from the snapshot. + // + // +k8s:required + // +k8s:minimum=1 + // +k8s:maximum=3 # keep this in sync with the SnapshotFidelity enum SnapshotFidelity fidelity = 10; // When absent the actor has no egress: its TCP is captured and refused. + // + // +k8s:optional optional EgressGateway egress_gateway = 12; // The actor's declared size, from the ActorTemplate's resource limits. Used to // (re)size the sandbox on a VOLUMES restore (fresh guest container). On a // MEMORY micro-VM restore the size baked into the snapshot is authoritative and // these are ignored. Zero means "unset": keep the runtime default. - int64 cpu_milli = 14; // CPU limit in millicores (1000 = one core). + // + // +k8s:optional + // +k8s:minimum=1 + // +k8s:maximum=999999 # matches atelet.RestoreRequest.cpu_milli, the control plane caps cpu limits strictly below 1000 cores + int64 cpu_milli = 14; // CPU limit in millicores (1000 = one core). + // +k8s:optional + // +k8s:minimum=1 int64 memory_bytes = 15; // Memory limit in bytes. + // +k8s:required ActorDirs actor_dirs = 16; // actor_dirs.restore_dir is a preserved local snapshot, not a scratch copy: // the worker must not delete or modify files in it. + // + // +k8s:optional bool preserve_restore_dir = 17; } @@ -373,6 +553,9 @@ message GetWorkloadStatsRequest { // between the caller's view of the world and this call, so the worker answers // NOT_FOUND on a mismatch rather than reporting a different actor's numbers // under the requested actor's identity. + // + // +k8s:required + // +k8s:format=k8s-uuid string actor_uid = 1; } diff --git a/internal/resources/actor.go b/internal/resources/actor.go index 13682a2d40..cf2413565a 100644 --- a/internal/resources/actor.go +++ b/internal/resources/actor.go @@ -18,7 +18,4 @@ const ( // ResourceNameRegexPattern is the regular expression pattern for a valid // Substrate resource name. ResourceNameRegexPattern = `[a-z0-9]([-a-z0-9]*[a-z0-9])?` - // GoldenActorAtespace is the reserved system atespace that per-template golden - // actors live in. - GoldenActorAtespace = "ate-golden" ) diff --git a/internal/resources/snapshotfiles.go b/internal/resources/snapshotfiles.go index fcacb2cdda..c5604d9799 100644 --- a/internal/resources/snapshotfiles.go +++ b/internal/resources/snapshotfiles.go @@ -20,19 +20,23 @@ import ( ) // FSCheckpointSubdir is the subdirectory under the checkpoint directory where -// filesystem checkpoint files are stored. +// durable-dir filesystem checkpoint files are stored. const FSCheckpointSubdir = "fs" +// RootfsCheckpointSubdir is the subdirectory under the checkpoint directory +// where container rootfs filesystem checkpoint files are stored. +const RootfsCheckpointSubdir = "rootfs" + // ValidateSnapshotFileNames requires each name to be a distinct plain file name -// in the checkpoint directory or its "fs" subdirectory. Actual file access must -// still use os.Root so symlinks cannot escape that directory. +// in the checkpoint directory or its "fs" or "rootfs" subdirectories. Actual +// file access must still use os.Root so symlinks cannot escape that directory. func ValidateSnapshotFileNames(files []string) error { seen := make(map[string]bool, len(files)) for i, name := range files { dir := filepath.Dir(name) base := filepath.Base(name) switch { - case !filepath.IsLocal(name) || name != filepath.Clean(name) || base == "." || (dir != "." && dir != FSCheckpointSubdir): + case !filepath.IsLocal(name) || name != filepath.Clean(name) || base == "." || (dir != "." && dir != FSCheckpointSubdir && dir != RootfsCheckpointSubdir): return fmt.Errorf("snapshotFiles[%d] %q is not a file name in the checkpoint directory", i, name) case seen[name]: return fmt.Errorf("snapshotFiles[%d] %q is duplicated", i, name) diff --git a/internal/resources/snapshotfiles_test.go b/internal/resources/snapshotfiles_test.go index e5667f6429..768699da95 100644 --- a/internal/resources/snapshotfiles_test.go +++ b/internal/resources/snapshotfiles_test.go @@ -25,12 +25,14 @@ func TestValidateSnapshotFileNames(t *testing.T) { {name: "empty list", files: nil}, {name: "plain names", files: []string{"manifest.json", "memory.img", "durable-dir.tar"}}, {name: "fs subdir names", files: []string{"checkpoint.img", "fs/fscheckpoint.pb", "fs/multitar.img", "fs/pages.img", "fs/pages_meta.img"}}, + {name: "rootfs subdir names", files: []string{"checkpoint.img", "rootfs/fscheckpoint.pb", "rootfs/multitar.img", "rootfs/pages.img", "rootfs/pages_meta.img"}}, {name: "empty name", files: []string{""}, wantErr: true}, {name: "dot", files: []string{"."}, wantErr: true}, {name: "dot dot", files: []string{".."}, wantErr: true}, {name: "absolute", files: []string{"/etc/passwd"}, wantErr: true}, {name: "nested arbitrary subdir", files: []string{"a/b"}, wantErr: true}, {name: "deeply nested under fs", files: []string{"fs/a/b"}, wantErr: true}, + {name: "deeply nested under rootfs", files: []string{"rootfs/a/b"}, wantErr: true}, {name: "escaping", files: []string{"../x"}, wantErr: true}, {name: "duplicate", files: []string{"a", "b", "a"}, wantErr: true}, } { diff --git a/internal/resources/validate.go b/internal/resources/validate.go index 98c1e04fb8..3298cdd8b2 100644 --- a/internal/resources/validate.go +++ b/internal/resources/validate.go @@ -52,8 +52,11 @@ func ToGRPCStatusError(errs field.ErrorList) error { } // ToAPIError turns validation errors into the InvalidArgument error an RPC -// handler responds with. Callers check len(errs) > 0 first. +// handler responds with, or nil if errs is empty. func ToAPIError(errs field.ErrorList) error { + if len(errs) == 0 { + return nil + } return apierror.InvalidArgument("%v", errs.ToAggregate()) } @@ -107,11 +110,12 @@ func ValidateWorkerPodUID(workerPodUID string) error { return nil } -// ValidateActorDirs checks that every directory atelet passes to worker is -// set, absolute and clean. +// ValidateActorDirs checks that every directory atelet passes to the worker +// is absolute and clean. Presence is enforced by the tags on the message; it +// backs the ateworkerpb.ActorDirs custom validation hook. func ValidateActorDirs(actorDirs *ateworkerpb.ActorDirs, fldPath *field.Path) field.ErrorList { if actorDirs == nil { - return field.ErrorList{field.Required(fldPath, "")} + return nil } var errs field.ErrorList for _, actorDir := range []struct{ name, path string }{ @@ -158,7 +162,7 @@ func ValidateSandboxClass(fldPath *field.Path, value string) field.ErrorList { func validateAbsDir(dir string, fldPath *field.Path) field.ErrorList { if dir == "" { - return field.ErrorList{field.Required(fldPath, "")} + return nil // required is enforced by tags } if !filepath.IsAbs(dir) || filepath.Clean(dir) != dir { return field.ErrorList{field.Invalid(fldPath, dir, "must be an absolute, clean path")} @@ -425,6 +429,12 @@ func ValidateMountPath(fldPath *field.Path, p string) field.ErrorList { // fldPath[i].mount_path. Identical paths are left to the list-key uniqueness // check. func ValidateNestedMountPaths(fldPath *field.Path, paths []string) field.ErrorList { + return ValidateNestedMountPathsAt(paths, func(i int) *field.Path { return fldPath.Index(i).Child("mount_path") }) +} + +// ValidateNestedMountPathsAt is ValidateNestedMountPaths for paths gathered +// from more than one list: at(i) is where paths[i] is reported. +func ValidateNestedMountPathsAt(paths []string, at func(i int) *field.Path) field.ErrorList { var errs field.ErrorList for i, path := range paths { if path == "" { @@ -435,7 +445,7 @@ func ValidateNestedMountPaths(fldPath *field.Path, paths []string) field.ErrorLi continue } if strings.HasPrefix(path, prior+"/") || strings.HasPrefix(prior, path+"/") { - errs = append(errs, field.Invalid(fldPath.Index(i).Child("mount_path"), path, + errs = append(errs, field.Invalid(at(i), path, fmt.Sprintf("must not nest under or over another mount (%q)", prior))) } } diff --git a/internal/resources/validate_test.go b/internal/resources/validate_test.go index 41b8029a98..676106d8e0 100644 --- a/internal/resources/validate_test.go +++ b/internal/resources/validate_test.go @@ -51,6 +51,26 @@ func TestToAPIError(t *testing.T) { } } +func TestToAPIErrorEmpty(t *testing.T) { + for _, tc := range []struct { + name string + errs field.ErrorList + }{ + { + name: "nil", + errs: nil, + }, + { + name: "empty", + errs: field.ErrorList{}, + }, + } { + if err := ToAPIError(tc.errs); err != nil { + t.Errorf("%s: ToAPIError = %v, want nil", tc.name, err) + } + } +} + func TestDeepEqual(t *testing.T) { // Proto messages carry internal state that reflect.DeepEqual would // compare; proto.Equal compares only field values. @@ -368,11 +388,12 @@ func TestValidateActorDirs(t *testing.T) { wantField string }{ {"valid", func(actorDirs *ateworkerpb.ActorDirs) *ateworkerpb.ActorDirs { return actorDirs }, ""}, - {"nil", func(*ateworkerpb.ActorDirs) *ateworkerpb.ActorDirs { return nil }, "actor_dirs"}, + // Presence is the tags' job: nil and empty are not this check's to report. + {"nil", func(*ateworkerpb.ActorDirs) *ateworkerpb.ActorDirs { return nil }, ""}, {"missing dir", func(actorDirs *ateworkerpb.ActorDirs) *ateworkerpb.ActorDirs { actorDirs.RestoreDir = "" return actorDirs - }, "actor_dirs.restore_dir"}, + }, ""}, {"relative dir", func(actorDirs *ateworkerpb.ActorDirs) *ateworkerpb.ActorDirs { actorDirs.OciBundleDir = "bundles" return actorDirs @@ -610,6 +631,20 @@ func TestValidateNestedMountPaths(t *testing.T) { } } +// TestValidateNestedMountPathsAt pins that each finding lands on the field +// the caller maps its index to, which is how a caller spanning several lists +// points at the right one. +func TestValidateNestedMountPathsAt(t *testing.T) { + fields := []*field.Path{ + field.NewPath("external_volume_mounts").Index(0).Child("mount_path"), + field.NewPath("image_volume_mounts").Index(0).Child("mount_path"), + field.NewPath("image_volume_mounts").Index(1).Child("mount_path"), + } + got := ValidateNestedMountPathsAt([]string{"/a", "/b", "/a/c"}, func(i int) *field.Path { return fields[i] }) + want := field.ErrorList{field.Invalid(fields[2], nil, "")} + field.ErrorMatcher{}.ByType().ByField().Test(t, want, got) +} + func TestValidateCSIMap(t *testing.T) { path := field.NewPath("publish_context") many := make(map[string]string, 100) diff --git a/internal/sandboxruntimes/sandboxruntimes.go b/internal/sandboxruntimes/sandboxruntimes.go index 34c6ca8c6b..57d7797189 100644 --- a/internal/sandboxruntimes/sandboxruntimes.go +++ b/internal/sandboxruntimes/sandboxruntimes.go @@ -39,7 +39,7 @@ type File struct { RestorableRuntimes []Attributes `json:"restorableRuntimes,omitempty"` } -// Attributes maps a compatibility attribute key, e.g. "sandbox_runtime_hash", +// Attributes maps a compatibility attribute key, e.g. "sandbox_bundle_hash", // to its value. type Attributes map[string]string diff --git a/internal/sandboxruntimes/sandboxruntimes_test.go b/internal/sandboxruntimes/sandboxruntimes_test.go index 0562caa464..03fedadbf7 100644 --- a/internal/sandboxruntimes/sandboxruntimes_test.go +++ b/internal/sandboxruntimes/sandboxruntimes_test.go @@ -28,8 +28,8 @@ import ( func testRuntimes(defaultHash string) *File { return &File{ - DefaultRuntime: Attributes{"sandbox_runtime_hash": defaultHash}, - RestorableRuntimes: []Attributes{{"sandbox_runtime_hash": "sha256:41ab"}}, + DefaultRuntime: Attributes{"sandbox_bundle_hash": defaultHash}, + RestorableRuntimes: []Attributes{{"sandbox_bundle_hash": "sha256:41ab"}}, } } @@ -144,14 +144,14 @@ func TestRead(t *testing.T) { wantErr: true, }, { name: "unknown field", - contents: `{"defaultRuntime": {"sandbox_runtime_hash": "sha256:7c2e"}, "nope": 1}`, + contents: `{"defaultRuntime": {"sandbox_bundle_hash": "sha256:7c2e"}, "nope": 1}`, wantErr: true, }, { name: "valid file", - contents: `{"defaultRuntime": {"sandbox_runtime_hash": "sha256:7c2e"}, "restorableRuntimes": [{"sandbox_runtime_hash": "sha256:41ab"}]}`, + contents: `{"defaultRuntime": {"sandbox_bundle_hash": "sha256:7c2e"}, "restorableRuntimes": [{"sandbox_bundle_hash": "sha256:41ab"}]}`, want: &File{ - DefaultRuntime: Attributes{"sandbox_runtime_hash": "sha256:7c2e"}, - RestorableRuntimes: []Attributes{{"sandbox_runtime_hash": "sha256:41ab"}}, + DefaultRuntime: Attributes{"sandbox_bundle_hash": "sha256:7c2e"}, + RestorableRuntimes: []Attributes{{"sandbox_bundle_hash": "sha256:41ab"}}, }, }} for _, tt := range tests { diff --git a/internal/serverboot/serverboot.go b/internal/serverboot/serverboot.go index 9b9779078c..f261760e4d 100644 --- a/internal/serverboot/serverboot.go +++ b/internal/serverboot/serverboot.go @@ -47,6 +47,7 @@ import ( "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/instrumentation" sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.40.0" @@ -262,10 +263,11 @@ func InitMetrics(ctx context.Context, serviceName string) (*sdkmetric.MeterProvi // InitMetricsBridged is for a binary whose scraped endpoint serves a registry // it does not own (atecontroller: controller-runtime's), so every metric // reaches the backend exactly once on whichever path is active. With the OTLP -// export, reg is bridged onto the push and the OTel instruments stay off it, -// since the bridge would otherwise push them a second time. With prometheus -// alone, nothing is pushed and the OTel instruments register on reg, so its -// endpoint serves both. +// export, reg is bridged onto the push and the OTel instruments are pushed +// natively. With the scraped endpoint on too, the instruments also register on +// reg, and the bridge skips them so the push does not carry them twice. With +// prometheus alone, nothing is pushed and the instruments register on reg, so +// its endpoint serves both. // // servePull reports whether OTEL_METRICS_EXPORTER leaves the scraped endpoint // on, which the caller must then serve from reg: unset, prometheus, or a list @@ -282,28 +284,74 @@ func InitMetricsBridged(ctx context.Context, serviceName string, reg interface { } exporters := metricsExporters(ctx, true) servePull = exporters.Has(exporterPrometheus) - switch { - case exporters.Has(ExporterOTLP): - producer := prombridge.NewMetricProducer(prombridge.WithGatherer(reg)) - if wrapProducer != nil { - producer = wrapProducer(producer) + push := exporters.Has(ExporterOTLP) + + var producers []sdkmetric.Producer + var readers []sdkmetric.Reader + var bridge sdkmetric.Producer + if push { + bridge = prombridge.NewMetricProducer(prombridge.WithGatherer(reg)) + } + if servePull { + registerer := promclient.Registerer(reg) + var own *promclient.Registry + if push { + // own holds only the instruments, so the bridge can tell which of + // reg's families are not its to push. reg serves own as well. + own = promclient.NewRegistry() + registerer = own } - mp, err = newMeterProvider(ctx, serviceName, true, false, nil, []sdkmetric.Producer{producer}) - case servePull: - var promReader *prometheus.Exporter - if promReader, err = prometheus.New(prometheus.WithRegisterer(reg)); err != nil { + promReader, err := prometheus.New(prometheus.WithRegisterer(registerer)) + if err != nil { return nil, false, fmt.Errorf("create Prometheus metric exporter: %w", err) } - mp, err = newMeterProvider(ctx, serviceName, false, false, nil, nil, promReader) - default: - mp, err = newMeterProvider(ctx, serviceName, false, false, nil, nil) + if own != nil { + if err := reg.Register(own); err != nil { + return nil, false, fmt.Errorf("serve the OTel instruments from the registry: %w", err) + } + bridge = skipFamilies{inner: bridge, skip: own} + } + readers = append(readers, promReader) } + if push { + if wrapProducer != nil { + bridge = wrapProducer(bridge) + } + producers = append(producers, bridge) + } + mp, err = newMeterProvider(ctx, serviceName, push, false, nil, producers, readers...) if err != nil { return nil, false, err } return mp, servePull, nil } +// skipFamilies drops the metrics of inner named like a family skip gathers. A +// failed gather drops inner's batch rather than risk pushing a metric twice, +// and goes to the OTel error handler so the other metrics still push. +type skipFamilies struct { + inner sdkmetric.Producer + skip promclient.Gatherer +} + +func (p skipFamilies) Produce(ctx context.Context) ([]metricdata.ScopeMetrics, error) { + scopes, err := p.inner.Produce(ctx) + // Gathered after inner, so skip covers every family inner saw. + families, gerr := p.skip.Gather() + if gerr != nil { + otel.Handle(fmt.Errorf("gather the families to skip: %w", gerr)) + return nil, err + } + skip := make(map[string]bool, len(families)) + for _, f := range families { + skip[f.GetName()] = true + } + for i := range scopes { + scopes[i].Metrics = slices.DeleteFunc(scopes[i].Metrics, func(m metricdata.Metrics) bool { return skip[m.Name] }) + } + return scopes, err +} + // InitMetricsPushOnlyVia is InitMetrics without the Prometheus reader, for a // binary that runs no metrics HTTP server of its own (worker): a pull reader // would collect into a registry nothing serves. conn is the metrics counterpart diff --git a/internal/serverboot/serverboot_test.go b/internal/serverboot/serverboot_test.go index 81c93e6702..33f3e48509 100644 --- a/internal/serverboot/serverboot_test.go +++ b/internal/serverboot/serverboot_test.go @@ -605,35 +605,100 @@ func bridgedRegistry(t *testing.T) *prometheus.Registry { return reg } -// With the OTLP export, whether selected or the default, the bridged family and -// the OTel instrument are each pushed once. The instrument must stay off the -// bridged registry: there the bridge would push it a second time, under its -// Prometheus name. +// With the OTLP export, the bridged family and the OTel instrument are each +// pushed once. When the scraped endpoint is on too, the instrument is served +// from the bridged registry as well, and the bridge must not push it again +// under its Prometheus name. func TestInitMetricsBridgedPushesEachMetricOnce(t *testing.T) { - for _, tt := range []struct{ name, value string }{ - {"unset", ""}, - {"otlp", "otlp"}, - {"list", "otlp,prometheus"}, + for _, tt := range []struct { + name, value string + served bool + }{ + {"unset", "", true}, + {"otlp", "otlp", false}, + {"list", "otlp,prometheus", true}, } { t.Run(tt.name, func(t *testing.T) { collector := startMetricsCollector(t, tt.value) reg := bridgedRegistry(t) - mp, _, err := InitMetricsBridged(t.Context(), "test-bridged-otlp", reg, nil) + mp, servePull, err := InitMetricsBridged(t.Context(), "test-bridged-otlp", reg, nil) if err != nil { t.Fatalf("InitMetricsBridged: %v", err) } + if servePull != tt.served { + t.Errorf("servePull = %v, want %v", servePull, tt.served) + } addOne(t, mp, "ate.test.bridged.count") - if gatheredNames(t, reg)["ate_test_bridged_count_total"] { - t.Error("the OTel instrument is registered on the bridged registry") + names := gatheredNames(t, reg) + if names["ate_test_bridged_count_total"] != tt.served { + t.Errorf("the bridged registry serves the OTel instrument = %v, want %v", !tt.served, tt.served) + } + if !names["test_bridged_family"] { + t.Error("the bridged registry lost its own family") } pushed := collector.flush(t, mp) - if pushed["ate.test.bridged.count"] != 1 || pushed["test_bridged_family"] != 1 || pushed["ate_test_bridged_count_total"] != 0 { - t.Errorf("want ate.test.bridged.count and test_bridged_family pushed once each and nothing else for them, got %v", pushed) + if pushed["ate.test.bridged.count"] != 1 || pushed["test_bridged_family"] != 1 { + t.Errorf("want ate.test.bridged.count and test_bridged_family pushed once each, got %v", pushed) + } + if pushed["ate_test_bridged_count_total"] != 0 || pushed["target_info"] != 0 { + t.Errorf("the bridge pushed what the OTel reader already covers: %v", pushed) } }) } } +// A skip list that cannot be gathered drops the batch without failing the +// collection, so nothing is pushed twice and the other metrics still push. +// inner's own error is still returned. +func TestSkipFamiliesDropsBatchOnGatherError(t *testing.T) { + innerErr := errors.New("inner failed") + for _, tt := range []struct { + name string + innerErr error + }{ + {"no inner error", nil}, + {"inner error", innerErr}, + } { + t.Run(tt.name, func(t *testing.T) { + inner := producerFunc(func(context.Context) ([]metricdata.ScopeMetrics, error) { + return []metricdata.ScopeMetrics{{Metrics: []metricdata.Metrics{{Name: "kept"}}}}, tt.innerErr + }) + failing := prometheus.NewRegistry() + failing.MustRegister(failingCollector{}) + scopes, err := skipFamilies{inner: inner, skip: failing}.Produce(t.Context()) + if scopes != nil || err != tt.innerErr { + t.Errorf("Produce = %v, %v; want no metrics and %v", scopes, err, tt.innerErr) + } + }) + } +} + +// A family that reaches the skip registry while inner produces is still +// dropped, so the bridge never pushes it under its Prometheus name. +func TestSkipFamiliesCoversFamiliesRegisteredDuringProduce(t *testing.T) { + skip := prometheus.NewRegistry() + inner := producerFunc(func(context.Context) ([]metricdata.ScopeMetrics, error) { + skip.MustRegister(prometheus.NewCounter(prometheus.CounterOpts{Name: "late_total", Help: "late"})) + return []metricdata.ScopeMetrics{{Metrics: []metricdata.Metrics{{Name: "late_total"}, {Name: "kept"}}}}, nil + }) + scopes, err := skipFamilies{inner: inner, skip: skip}.Produce(t.Context()) + if err != nil { + t.Fatalf("Produce: %v", err) + } + if len(scopes) != 1 || len(scopes[0].Metrics) != 1 || scopes[0].Metrics[0].Name != "kept" { + t.Errorf("Produce = %v, want only kept", scopes) + } +} + +// failingCollector is an unchecked collector whose every Gather fails. +type failingCollector struct{} + +func (failingCollector) Describe(chan<- *prometheus.Desc) {} + +func (failingCollector) Collect(ch chan<- prometheus.Metric) { + ch <- prometheus.NewInvalidMetric(prometheus.NewDesc("failing", "always invalid", nil, nil), errors.New("boom")) +} + // wrapProducer sits between the bridge and the OTLP push. func TestInitMetricsBridgedWrapsPushedProducer(t *testing.T) { collector := startMetricsCollector(t, "otlp") diff --git a/internal/volume/csi/plugin_test.go b/internal/volume/csi/plugin_test.go index 908e971c1b..8da7583864 100644 --- a/internal/volume/csi/plugin_test.go +++ b/internal/volume/csi/plugin_test.go @@ -280,6 +280,12 @@ func TestPlugin_AttachVolume(t *testing.T) { } driver.controllerPublishVolumeFunc = func(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { + if req.GetNodeId() != "node-1" { + t.Errorf("ControllerPublishVolume received NodeId %q, want %q", req.GetNodeId(), "node-1") + } + if req.GetVolumeId() != "test-vol" { + t.Errorf("ControllerPublishVolume received VolumeId %q, want %q", req.GetVolumeId(), "test-vol") + } return &csi.ControllerPublishVolumeResponse{ PublishContext: map[string]string{"devicePath": "/dev/xvdba"}, }, nil @@ -295,6 +301,9 @@ func TestPlugin_AttachVolume(t *testing.T) { // Test Unimplemented warning bypass driver.controllerPublishVolumeFunc = func(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { + if req.GetNodeId() != "node-1" { + t.Errorf("ControllerPublishVolume received NodeId %q, want %q", req.GetNodeId(), "node-1") + } return nil, status.Error(codes.Unimplemented, "unimplemented") } resp, err = plugin.AttachVolume(ctx, volume.AttachVolumeRequest{VolumeID: "test-vol", Node: "node-1"}) @@ -321,19 +330,40 @@ func TestPlugin_DetachVolume(t *testing.T) { ctx := context.Background() plugin.InitControllerCapabilities(ctx) + var unpublishCalled int + driver.controllerUnpublishVolumeFunc = func(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) { + unpublishCalled++ + if req.GetNodeId() != "node-1" { + t.Errorf("ControllerUnpublishVolume received NodeId %q, want %q", req.GetNodeId(), "node-1") + } + if req.GetVolumeId() != "test-vol" { + t.Errorf("ControllerUnpublishVolume received VolumeId %q, want %q", req.GetVolumeId(), "test-vol") + } + return &csi.ControllerUnpublishVolumeResponse{}, nil + } err = plugin.DetachVolume(ctx, "test-vol", "node-1") if err != nil { t.Fatalf("DetachVolume failed: %v", err) } + if unpublishCalled != 1 { + t.Errorf("expected 1 call to ControllerUnpublishVolume, got %d", unpublishCalled) + } // Test Unimplemented warning bypass driver.controllerUnpublishVolumeFunc = func(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) { + unpublishCalled++ + if req.GetNodeId() != "node-1" { + t.Errorf("ControllerUnpublishVolume received NodeId %q, want %q", req.GetNodeId(), "node-1") + } return nil, status.Error(codes.Unimplemented, "unimplemented") } err = plugin.DetachVolume(ctx, "test-vol", "node-1") if err != nil { t.Errorf("DetachVolume should have ignored Unimplemented error, got: %v", err) } + if unpublishCalled != 2 { + t.Errorf("expected 2 calls to ControllerUnpublishVolume, got %d", unpublishCalled) + } } func TestPlugin_MountVolume(t *testing.T) { diff --git a/internal/volume/mock.go b/internal/volume/mock.go index ad8e24758f..8fad98b3ee 100644 --- a/internal/volume/mock.go +++ b/internal/volume/mock.go @@ -74,11 +74,16 @@ func (p *MockVolumePlugin) DeleteVolume(ctx context.Context, volumeID string) er return nil } -// AttachVolume simulates volume attachment to a node. The mock driver needs no -// attachment metadata, so the response carries no publish context. +// AttachVolume simulates volume attachment to a node. It returns simulated +// publish context metadata reflecting the target node and volume. func (p *MockVolumePlugin) AttachVolume(ctx context.Context, req AttachVolumeRequest) (AttachVolumeResponse, error) { slog.InfoContext(ctx, "MockVolumePlugin.AttachVolume", slog.String("volumeID", req.VolumeID), slog.String("node", req.Node)) - return AttachVolumeResponse{}, nil + return AttachVolumeResponse{ + PublishContext: map[string]string{ + "mock.substrate.io/attached-node": req.Node, + "mock.substrate.io/device-path": "/dev/mock-" + req.VolumeID, + }, + }, nil } // DetachVolume simulates volume detachment from a node. diff --git a/manifests/ate-install/ate-idp-server.yaml b/manifests/ate-install/ate-idp-server.yaml new file mode 100644 index 0000000000..a78641a2ba --- /dev/null +++ b/manifests/ate-install/ate-idp-server.yaml @@ -0,0 +1,117 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ate-idp-server serves the OpenID discovery document and key set relying +# parties use to verify actor JWTs. It needs no Kubernetes API access. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ate-idp-server + namespace: ate-system +automountServiceAccountToken: false +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ate-idp-server + namespace: ate-system +spec: + replicas: 2 + strategy: + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + selector: + matchLabels: + app: ate-idp-server + template: + metadata: + labels: + app: ate-idp-server + spec: + serviceAccountName: ate-idp-server + containers: + - name: ate-idp-server + image: ko://go.ate.dev/substrate/cmd/ate-idp-server + args: + - --actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json + - --server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem + ports: + - name: https + containerPort: 8443 + readinessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + periodSeconds: 5 + livenessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + periodSeconds: 10 + securityContext: + capabilities: + drop: + - ALL + volumeMounts: + # The serving certificate, with a DNS SAN for each Service that + # selects the pod: idp.ate-system.svc. + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + - name: actor-id-jwt-pool + mountPath: /run/actor-id-jwt-pool + volumes: + - name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + # The pool ate-api-server signs with. + - name: actor-id-jwt-pool + projected: + sources: + - secret: + name: actor-id-jwt-pool + items: + - key: pool + path: pool.json +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: ate-idp-server + namespace: ate-system +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ate-idp-server +--- +apiVersion: v1 +kind: Service +metadata: + name: idp + namespace: ate-system +spec: + selector: + app: ate-idp-server + ports: + - name: https + port: 443 + targetPort: https + appProtocol: https diff --git a/manifests/ate-install/ate-system-networkpolicies.yaml b/manifests/ate-install/ate-system-networkpolicies.yaml index 69e88d6028..159f4fd947 100644 --- a/manifests/ate-install/ate-system-networkpolicies.yaml +++ b/manifests/ate-install/ate-system-networkpolicies.yaml @@ -155,6 +155,26 @@ spec: - protocol: TCP port: connect-tls --- +# ate-idp-server serves the actor JWT discovery document and key set. Relying +# parties fetch them from pods in any namespace and, through an Ingress or a +# load balancer, from outside the cluster. Both documents are public, so the +# port admits any source. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: ate-idp-server + namespace: ate-system +spec: + podSelector: + matchLabels: + app: ate-idp-server + policyTypes: + - Ingress + ingress: + - ports: + - protocol: TCP + port: https +--- # The egress gateway is reached by atunnel in worker pods, which the # WorkerPool controller labels ate.dev/worker-pool, in whatever namespace the # WorkerPool lives in. diff --git a/manifests/ate-install/atelet.yaml b/manifests/ate-install/atelet.yaml index 7ff7371dd3..e10e601042 100644 --- a/manifests/ate-install/atelet.yaml +++ b/manifests/ate-install/atelet.yaml @@ -141,6 +141,16 @@ spec: - key: ate.dev/sandboxClass operator: Exists effect: NoSchedule + # GKE taints the nodes it creates for a ComputeClass with + # cloud.google.com/compute-class=:NoSchedule and adds the matching + # toleration to pods that select the class, so a WorkerPool whose + # template.nodeSelector selects a ComputeClass runs its workers on those + # nodes. + # TODO(roycaihw): move this GKE-specific toleration into the GKE install + # overlay once the new installer lands (#2760). + - key: cloud.google.com/compute-class + operator: Exists + effect: NoSchedule # The node object-store plugin moves external snapshots between this # node and object storage and fetches sandbox assets for atelet, over a # Unix socket in a pod-local emptyDir. @@ -271,8 +281,8 @@ spec: name: ate-otel-config ports: - name: grpc - containerPort: 8085 - hostPort: 8085 + containerPort: 21998 + hostPort: 21998 - name: prometheus containerPort: 9090 protocol: TCP diff --git a/manifests/ate-install/base/kustomization.yaml b/manifests/ate-install/base/kustomization.yaml index caf8b58b21..e1519c964a 100644 --- a/manifests/ate-install/base/kustomization.yaml +++ b/manifests/ate-install/base/kustomization.yaml @@ -29,6 +29,7 @@ kind: Kustomization # either. resources: - ../ate-api-server.yaml + - ../ate-idp-server.yaml - ../ate-controller.yaml - ../atelet.yaml - ../atenet-router.yaml diff --git a/manifests/ate-install/components/cordon-control-plane/kustomization.yaml b/manifests/ate-install/components/cordon-control-plane/kustomization.yaml index 86c1f14507..edc70647dd 100644 --- a/manifests/ate-install/components/cordon-control-plane/kustomization.yaml +++ b/manifests/ate-install/components/cordon-control-plane/kustomization.yaml @@ -42,7 +42,7 @@ kind: Component patches: - target: kind: Deployment - name: ate-api-server|ate-controller|atenet-router|atenet-egress|podcertificate-controller|k8s-credential-provider|fake-workersync + name: ate-api-server|ate-idp-server|ate-controller|atenet-router|atenet-egress|podcertificate-controller|k8s-credential-provider|fake-workersync patch: |- apiVersion: apps/v1 kind: Deployment diff --git a/manifests/ate-install/components/fake-data-plane/kustomization.yaml b/manifests/ate-install/components/fake-data-plane/kustomization.yaml index dcd5f803eb..9039c816c1 100644 --- a/manifests/ate-install/components/fake-data-plane/kustomization.yaml +++ b/manifests/ate-install/components/fake-data-plane/kustomization.yaml @@ -71,7 +71,7 @@ patches: path: /spec/template/spec/containers/0/ports value: - name: grpc - containerPort: 8085 + containerPort: 21998 # fake-workersync's capacity reports, relayed to ate-api-server under # this pod's atelet identity; only fake-workersync's is admitted. - name: relay diff --git a/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml b/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml index 74caf8b3c7..cdb8a6512d 100644 --- a/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml +++ b/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml @@ -45,8 +45,9 @@ spec: openAPIV3Schema: description: |- SandboxConfig is cluster-scoped configuration describing the versions of the - sandbox binaries for a sandbox runtime family. It is referenced by an - ActorTemplate's sandbox_config.config_name (required) and decouples sandbox + sandbox binaries for a sandbox runtime family. It is referenced by a + WorkerPool's spec.sandboxClasses[].configRef (or resolved via the + sandboxconfig.ate.dev/is-class-default annotation) and decouples sandbox binary selection from the workload definition. properties: apiVersion: @@ -79,9 +80,9 @@ spec: sandboxClass: default: gvisor description: |- - sandboxClass is the sandbox runtime family this config applies to. An - ActorTemplate only uses SandboxConfigs whose SandboxClass matches its - sandbox_config.sandbox_class. + sandboxClass is the sandbox runtime family this config applies to. A + WorkerPool only uses SandboxConfigs whose sandboxClass matches the entry + in its spec.sandboxClasses. enum: - gvisor - microvm @@ -129,11 +130,10 @@ spec: architecture (GOARCH, e.g. "amd64", "arm64") and then by asset name. The asset names are interpreted by the sandbox backend: gVisor expects a "gvisor" asset (the release's gvisor.tar.zstd, which atelet extracts so - the gvisor-bin/ helpers sit next to runsc; a legacy bare-binary "runsc" - asset is still accepted); a micro-VM backend expects several (e.g. - "cloud-hypervisor", "kata-kernel", "kata-image"). The schema is - intentionally generic; per-class requirements are enforced by a - ValidatingAdmissionPolicy. + the gvisor-bin/ helpers sit next to runsc); a micro-VM backend expects + several (e.g. "cloud-hypervisor", "kata-kernel", "kata-image"). The + schema is intentionally generic; per-class requirements are enforced by + a ValidatingAdmissionPolicy. type: object name: description: |- diff --git a/manifests/ate-install/generated/ate.dev_workerpools.yaml b/manifests/ate-install/generated/ate.dev_workerpools.yaml index a46df13c49..568b7c2fae 100644 --- a/manifests/ate-install/generated/ate.dev_workerpools.yaml +++ b/manifests/ate-install/generated/ate.dev_workerpools.yaml @@ -108,9 +108,7 @@ spec: name selects the sandbox runtime family, which drives the worker pod shape (KVM/vhost device mounts and node placement). The concrete binary is still selected by WorkerImage. The sandbox binaries themselves come - from the SandboxConfig each ActorTemplate names (required). - - See Also: TODOs in ActorTemplate SandboxClass + from the SandboxConfig referenced by ConfigRef (or the class default). enum: - gvisor - microvm diff --git a/manifests/ate-install/kind/kustomization.yaml b/manifests/ate-install/kind/kustomization.yaml index ff65fefdd9..c757f2362f 100644 --- a/manifests/ate-install/kind/kustomization.yaml +++ b/manifests/ate-install/kind/kustomization.yaml @@ -28,6 +28,7 @@ kind: Kustomization # WORKERS_PER_SIGNER override. resources: - ./ate-api-server + - ../ate-idp-server.yaml - ../ate-controller.yaml - ./atelet - ../atenet-router.yaml diff --git a/manifests/ate-install/sandboxconfig-gvisor.yaml b/manifests/ate-install/sandboxconfig-gvisor.yaml index a4f6b843df..ab8fd3742c 100644 --- a/manifests/ate-install/sandboxconfig-gvisor.yaml +++ b/manifests/ate-install/sandboxconfig-gvisor.yaml @@ -13,12 +13,12 @@ # limitations under the License. # Cluster-wide SandboxConfig for the gVisor (runsc) sandbox class, shipped with -# the platform so gVisor ActorTemplates have a config to name via -# sandboxConfig.configName. atelet fetches the gVisor release tarball +# the platform as the class default so gVisor WorkerPools resolve it unless +# they pin a specific configRef. atelet fetches the gVisor release tarball # (gvisor.tar.zstd: runsc plus the gvisor-bin/ helpers runsc requires next to # it) matching the worker node's architecture and extracts it locally. To pin a # different release, add a version below and point defaultVersion at it, or -# create another SandboxConfig and name it from the ActorTemplate. +# create another SandboxConfig and reference it from the WorkerPool. apiVersion: ate.dev/v1alpha1 kind: SandboxConfig metadata: @@ -27,12 +27,13 @@ metadata: sandboxconfig.ate.dev/is-class-default: "true" spec: sandboxClass: gvisor - defaultVersion: v3 + defaultVersion: v4 versions: - # gVisor nightly 2026-10-10, for the split filesystem checkpoint - # (runsc fscheckpoint, -fs-checkpoint-paths and -fs-restore-image-path) that - # ateworker-gvisor uses to save and restore durable-dir volumes. - - name: v3 + # gVisor nightly 2026-10-17, the first to include google/gvisor@b79256080dec + # (save and restore multiple filesystem checkpoint bundles). ateworker-gvisor + # uses it to save container rootfs overlays (rootfs/) and durable-dir volumes + # (fs/) as separate bundles. + - name: v4 state: Enabled # The root sandbox container's image. On GCP, prefer the in-project mirror # gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da. @@ -40,9 +41,9 @@ spec: assets: amd64: gvisor: - url: "gs://gvisor/releases/nightly/2026-10-10/x86_64/gvisor.tar.zstd" - sha256: "4ae7fab988c3647d2a835d6c3c70c6a99021c562db97827592562b6c51c3e88c" + url: "gs://gvisor/releases/nightly/2026-10-17/x86_64/gvisor.tar.zstd" + sha256: "fa72f0d45756c69a30beec21222c78241b0d330c36f4f2185648617cc23415c8" arm64: gvisor: - url: "gs://gvisor/releases/nightly/2026-10-10/aarch64/gvisor.tar.zstd" - sha256: "aaf414f06eea7960238e43165c691f0a36cefc0593fb141e011e96211c7b6f57" + url: "gs://gvisor/releases/nightly/2026-10-17/aarch64/gvisor.tar.zstd" + sha256: "238634978d99f0dd3e631c4f88eae7b19fb36937065428467f0b6d6b10697689" diff --git a/manifests/ate-install/sandboxconfig-validation.yaml b/manifests/ate-install/sandboxconfig-validation.yaml index 1e77e6af02..f0f3866667 100644 --- a/manifests/ate-install/sandboxconfig-validation.yaml +++ b/manifests/ate-install/sandboxconfig-validation.yaml @@ -31,15 +31,13 @@ spec: resources: ["sandboxconfigs"] validations: # gVisor needs a "gvisor" release-tarball asset (gvisor.tar.zstd: runsc plus - # its gvisor-bin/ helpers) for every architecture it advertises. A legacy - # bare-binary "runsc" asset is still accepted for releases that predate the - # tarball. + # its gvisor-bin/ helpers) for every architecture it advertises. - expression: >- object.spec.sandboxClass != 'gvisor' || object.spec.versions.all(v, has(v.assets) && size(v.assets) > 0 && - v.assets.all(arch, 'gvisor' in v.assets[arch] || 'runsc' in v.assets[arch])) - message: "a gvisor SandboxConfig must define a 'gvisor' (release tarball) or legacy 'runsc' asset for every architecture of every version under spec.versions[].assets" + v.assets.all(arch, 'gvisor' in v.assets[arch])) + message: "a gvisor SandboxConfig must define a 'gvisor' (release tarball) asset for every architecture of every version under spec.versions[].assets" # The micro-VM (cloud-hypervisor) runtime needs its asset set for every # architecture it advertises. - expression: >- diff --git a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl index d801d8aaef..ac1c2cabda 100644 --- a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl +++ b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl @@ -16,9 +16,9 @@ # class. Unlike sandboxconfig-gvisor.yaml (applied unconditionally by # hack/install-ate.sh --deploy-ate-system), this is opt-in: apply via # hack/install-microvm-deps.sh --install after staging the asset set -# (assemble.sh + stage-to-gcs.sh / stage-to-rustfs.sh). Every microvm -# ActorTemplate names it explicitly (configName: microvm), so a missing/stale -# config fails loudly at template creation. +# (assemble.sh + stage-to-gcs.sh / stage-to-rustfs.sh). A microvm WorkerPool +# resolves it as its class default (or names it via configRef: microvm), so +# a missing/stale config shows on the pool's status. # # The sandbox binaries (cloud-hypervisor, virtiofsd, guest kernel, guest rootfs) # are FETCHED at runtime from the cluster object store bucket ${BUCKET_NAME} diff --git a/pkg/api/v1alpha1/sandboxconfig_types.go b/pkg/api/v1alpha1/sandboxconfig_types.go index e344536eca..3672affbff 100644 --- a/pkg/api/v1alpha1/sandboxconfig_types.go +++ b/pkg/api/v1alpha1/sandboxconfig_types.go @@ -141,11 +141,10 @@ type SandboxVersionConfig struct { // architecture (GOARCH, e.g. "amd64", "arm64") and then by asset name. The // asset names are interpreted by the sandbox backend: gVisor expects a // "gvisor" asset (the release's gvisor.tar.zstd, which atelet extracts so - // the gvisor-bin/ helpers sit next to runsc; a legacy bare-binary "runsc" - // asset is still accepted); a micro-VM backend expects several (e.g. - // "cloud-hypervisor", "kata-kernel", "kata-image"). The schema is - // intentionally generic; per-class requirements are enforced by a - // ValidatingAdmissionPolicy. + // the gvisor-bin/ helpers sit next to runsc); a micro-VM backend expects + // several (e.g. "cloud-hypervisor", "kata-kernel", "kata-image"). The + // schema is intentionally generic; per-class requirements are enforced by + // a ValidatingAdmissionPolicy. // // +optional Assets map[string]map[string]AssetFile `json:"assets,omitempty"` @@ -157,9 +156,9 @@ type SandboxVersionConfig struct { // +kubebuilder:validation:XValidation:rule="!self.versions.exists(v, v.name == self.defaultVersion && has(v.state) && v.state == 'Disabled')",message="the version named by defaultVersion must not be Disabled" // +kubebuilder:validation:XValidation:rule="self.sandboxClass == 'gvisor' ? self.versions.all(v, has(v.pauseImage) && size(v.pauseImage) > 0) : self.versions.all(v, !has(v.pauseImage) || size(v.pauseImage) == 0)",message="pauseImage is required on every version for gvisor and not allowed for other sandbox classes" type SandboxConfigSpec struct { - // sandboxClass is the sandbox runtime family this config applies to. An - // ActorTemplate only uses SandboxConfigs whose SandboxClass matches its - // sandbox_config.sandbox_class. + // sandboxClass is the sandbox runtime family this config applies to. A + // WorkerPool only uses SandboxConfigs whose sandboxClass matches the entry + // in its spec.sandboxClasses. // // +optional // +kubebuilder:validation:Enum=gvisor;microvm @@ -202,8 +201,9 @@ func (s *SandboxConfigSpec) DefaultVersionConfig() (*SandboxVersionConfig, bool) } // SandboxConfig is cluster-scoped configuration describing the versions of the -// sandbox binaries for a sandbox runtime family. It is referenced by an -// ActorTemplate's sandbox_config.config_name (required) and decouples sandbox +// sandbox binaries for a sandbox runtime family. It is referenced by a +// WorkerPool's spec.sandboxClasses[].configRef (or resolved via the +// sandboxconfig.ate.dev/is-class-default annotation) and decouples sandbox // binary selection from the workload definition. // // +genclient diff --git a/pkg/api/v1alpha1/sandboxconfig_validation_test.go b/pkg/api/v1alpha1/sandboxconfig_validation_test.go index 9678b65ac4..5336a93296 100644 --- a/pkg/api/v1alpha1/sandboxconfig_validation_test.go +++ b/pkg/api/v1alpha1/sandboxconfig_validation_test.go @@ -32,6 +32,10 @@ import ( const validSHA256 = "a397be1abc2420d26bce6c70e6e2ff96c73aaaab929756c56f5e2089ea842b63" +// gvisorAssetMsg is part of the policy's message for a gvisor version without +// the release-tarball asset. +const gvisorAssetMsg = "'gvisor' (release tarball) asset" + // vapManifestPath is the SandboxConfig ValidatingAdmissionPolicy shipped with // the install — loaded here so the test guards the policy we actually ship. const vapManifestPath = "../../../manifests/ate-install/sandboxconfig-validation.yaml" @@ -83,7 +87,8 @@ func gvisorAssets() map[string]map[string]AssetFile { return map[string]map[string]AssetFile{"amd64": {"gvisor": gvisorAsset()}} } -func runscAsset() AssetFile { return AssetFile{URL: "gs://bucket/runsc", SHA256: validSHA256} } +// otherAsset is a valid asset that is not the one a gvisor config needs. +func otherAsset() AssetFile { return AssetFile{URL: "gs://bucket/other", SHA256: validSHA256} } func gvisorAsset() AssetFile { return AssetFile{URL: "gs://bucket/gvisor.tar.bz2", SHA256: validSHA256} @@ -125,19 +130,19 @@ func applyVAP(t *testing.T, ctx context.Context) { } } - // Wait until the policy is enforced: a gvisor config missing runsc (valid - // per the CRD schema) must be denied. + // Wait until the policy is enforced: a gvisor config missing the gvisor + // asset (valid per the CRD schema) must be denied. i := 0 err = wait.PollUntilContextTimeout(ctx, 200*time.Millisecond, 30*time.Second, true, func(ctx context.Context) (bool, error) { i++ sc := sandboxConfig(fmt.Sprintf("vap-warmup-%d", i), SandboxClassGvisor, - map[string]map[string]AssetFile{"amd64": {"notrunsc": runscAsset()}}) + map[string]map[string]AssetFile{"amd64": {"other": otherAsset()}}) createErr := k8sClient.Create(ctx, sc) if createErr == nil { _ = k8sClient.Delete(ctx, sc) // policy not active yet; clean up and retry return false, nil } - return strings.Contains(createErr.Error(), "runsc"), nil + return strings.Contains(createErr.Error(), gvisorAssetMsg), nil }) if err != nil { t.Fatalf("VAP did not become active: %v", err) @@ -157,14 +162,6 @@ func TestSandboxConfigValidation(t *testing.T) { name: "valid gvisor with release tarball", sc: sandboxConfig("ok-gvisor-tarball", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"gvisor": gvisorAsset()}, "arm64": {"gvisor": gvisorAsset()}}), wantErr: false, - }, { - name: "valid gvisor with legacy runsc", - sc: sandboxConfig("ok-gvisor", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"runsc": runscAsset()}, "arm64": {"runsc": runscAsset()}}), - wantErr: false, - }, { - name: "valid gvisor mixing tarball and legacy per arch", - sc: sandboxConfig("ok-gvisor-mixed", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"gvisor": gvisorAsset()}, "arm64": {"runsc": runscAsset()}}), - wantErr: false, }, { name: "valid microvm with full asset set", sc: sandboxConfig("ok-microvm", "microvm", map[string]map[string]AssetFile{"amd64": microVMAssets()}), @@ -192,33 +189,38 @@ func TestSandboxConfigValidation(t *testing.T) { wantErr: true, errMsg: "microvm SandboxConfig must define", }, { - name: "gvisor arch missing gvisor and runsc", - sc: sandboxConfig("bad-no-runsc", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"notrunsc": runscAsset()}}), + name: "gvisor arch missing the gvisor asset", + sc: sandboxConfig("bad-no-gvisor", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"other": otherAsset()}}), + wantErr: true, + errMsg: gvisorAssetMsg, + }, { + name: "gvisor with only a bare runsc asset", + sc: sandboxConfig("bad-bare-runsc", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"runsc": otherAsset()}}), wantErr: true, - errMsg: "runsc", + errMsg: gvisorAssetMsg, }, { - name: "gvisor one arch missing gvisor and runsc", - sc: sandboxConfig("bad-mixed-arch", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"gvisor": gvisorAsset()}, "arm64": {"notrunsc": runscAsset()}}), + name: "gvisor one arch missing the gvisor asset", + sc: sandboxConfig("bad-mixed-arch", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"gvisor": gvisorAsset()}, "arm64": {"other": otherAsset()}}), wantErr: true, - errMsg: "runsc", + errMsg: gvisorAssetMsg, }, { name: "gvisor with no assets", sc: sandboxConfig("bad-empty", SandboxClassGvisor, nil), wantErr: true, - errMsg: "runsc", + errMsg: gvisorAssetMsg, }, { name: "asset missing url", - sc: sandboxConfig("bad-no-url", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"runsc": {SHA256: validSHA256}}}), + sc: sandboxConfig("bad-no-url", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"gvisor": {SHA256: validSHA256}}}), wantErr: true, errMsg: "url", }, { name: "asset missing sha256", - sc: sandboxConfig("bad-no-sha", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"runsc": {URL: "gs://bucket/runsc"}}}), + sc: sandboxConfig("bad-no-sha", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"gvisor": {URL: "gs://bucket/gvisor.tar.bz2"}}}), wantErr: true, errMsg: "sha256", }, { name: "asset sha256 not 64 hex", - sc: sandboxConfig("bad-sha", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"runsc": {URL: "gs://bucket/runsc", SHA256: "deadbeef"}}}), + sc: sandboxConfig("bad-sha", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"gvisor": {URL: "gs://bucket/gvisor.tar.bz2", SHA256: "deadbeef"}}}), wantErr: true, errMsg: "sha256", }, { @@ -312,10 +314,10 @@ func TestSandboxConfigValidation(t *testing.T) { wantErr: true, errMsg: "pauseImage is required on every version for gvisor", }, { - name: "gvisor second version missing gvisor and runsc", - sc: withVersions(sandboxConfig("bad-v2-no-runsc", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v2", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"notrunsc": runscAsset()}})), + name: "gvisor second version missing the gvisor asset", + sc: withVersions(sandboxConfig("bad-v2-no-gvisor", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v2", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"other": otherAsset()}})), wantErr: true, - errMsg: "runsc", + errMsg: gvisorAssetMsg, }, { name: "microvm second version missing an asset", sc: withVersions(sandboxConfig("bad-microvm-v2", SandboxClassMicroVM, map[string]map[string]AssetFile{"amd64": microVMAssets()}), sandboxVersion("v2", SandboxClassMicroVM, map[string]map[string]AssetFile{"amd64": func() map[string]AssetFile { diff --git a/pkg/api/v1alpha1/workerpool_types.go b/pkg/api/v1alpha1/workerpool_types.go index 411b68327e..752e36b7a3 100644 --- a/pkg/api/v1alpha1/workerpool_types.go +++ b/pkg/api/v1alpha1/workerpool_types.go @@ -118,9 +118,7 @@ type WorkerPoolSandboxClass struct { // name selects the sandbox runtime family, which drives the worker pod // shape (KVM/vhost device mounts and node placement). The concrete binary // is still selected by WorkerImage. The sandbox binaries themselves come - // from the SandboxConfig each ActorTemplate names (required). - // - // See Also: TODOs in ActorTemplate SandboxClass + // from the SandboxConfig referenced by ConfigRef (or the class default). // // +required // +kubebuilder:validation:Enum=gvisor;microvm @@ -156,6 +154,20 @@ func (s *WorkerPoolSpec) DefaultSandboxClass() SandboxClass { return s.SandboxClasses[0].Name } +// ResolvedSandboxConfig returns the name of the SandboxConfig the controller +// confirmed for the class the pool's workers run, or "" if it has not +// resolved one yet. It never returns pendingConfigRef: that config is not +// confirmed yet. +func (p *WorkerPool) ResolvedSandboxConfig() string { + class := p.Spec.DefaultSandboxClass() + for _, c := range p.Status.SandboxClasses { + if c.Name == class && c.ConfigRef != nil { + return c.ConfigRef.Name + } + } + return "" +} + type WorkerPoolStatus struct { // replicas is the total number of worker pods. // +kubebuilder:validation:Minimum=0 diff --git a/pkg/proto/ateapipb/ateapi.pb.go b/pkg/proto/ateapipb/ateapi.pb.go index f2263209df..b7280154ae 100644 --- a/pkg/proto/ateapipb/ateapi.pb.go +++ b/pkg/proto/ateapipb/ateapi.pb.go @@ -151,8 +151,7 @@ const ( // from the OCI image with the volumes restored. SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES SnapshotFidelity = 1 // Volumes plus the root filesystem changes made since boot. The actor - // cold-boots on resume with its rootfs writes back in place. Served by the - // micro-VM sandbox class only. + // cold-boots on resume with its rootfs writes back in place. SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS SnapshotFidelity = 2 // Volumes, root filesystem changes, and process memory. The actor resumes // where it left off. @@ -422,6 +421,63 @@ func (ActorMetadataField) EnumDescriptor() ([]byte, []int) { return file_ateapi_proto_rawDescGZIP(), []int{6} } +// DurabilityMode selects when a suspend is acknowledged relative to its +// snapshot being stored durably. +type DurabilityMode int32 + +const ( + DurabilityMode_DURABILITY_MODE_UNSPECIFIED DurabilityMode = 0 + // Return only after the snapshot is complete in durable storage. Applies + // only when the call checkpoints a running actor: an actor that is already + // SUSPENDED is returned with the snapshot it holds, which may still be on + // its node. + DurabilityMode_DURABILITY_MODE_SYNCHRONOUS DurabilityMode = 1 + // Return once the snapshot is captured on the node. The upload to durable + // storage continues in the background. + DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS DurabilityMode = 2 // Keep this in sync with SuspendOptions.mode's maximum. +) + +// Enum value maps for DurabilityMode. +var ( + DurabilityMode_name = map[int32]string{ + 0: "DURABILITY_MODE_UNSPECIFIED", + 1: "DURABILITY_MODE_SYNCHRONOUS", + 2: "DURABILITY_MODE_ASYNCHRONOUS", + } + DurabilityMode_value = map[string]int32{ + "DURABILITY_MODE_UNSPECIFIED": 0, + "DURABILITY_MODE_SYNCHRONOUS": 1, + "DURABILITY_MODE_ASYNCHRONOUS": 2, + } +) + +func (x DurabilityMode) Enum() *DurabilityMode { + p := new(DurabilityMode) + *p = x + return p +} + +func (x DurabilityMode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (DurabilityMode) Descriptor() protoreflect.EnumDescriptor { + return file_ateapi_proto_enumTypes[7].Descriptor() +} + +func (DurabilityMode) Type() protoreflect.EnumType { + return &file_ateapi_proto_enumTypes[7] +} + +func (x DurabilityMode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use DurabilityMode.Descriptor instead. +func (DurabilityMode) EnumDescriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{7} +} + type WorkerState int32 const ( @@ -463,11 +519,11 @@ func (x WorkerState) String() string { } func (WorkerState) Descriptor() protoreflect.EnumDescriptor { - return file_ateapi_proto_enumTypes[7].Descriptor() + return file_ateapi_proto_enumTypes[8].Descriptor() } func (WorkerState) Type() protoreflect.EnumType { - return &file_ateapi_proto_enumTypes[7] + return &file_ateapi_proto_enumTypes[8] } func (x WorkerState) Number() protoreflect.EnumNumber { @@ -476,7 +532,7 @@ func (x WorkerState) Number() protoreflect.EnumNumber { // Deprecated: Use WorkerState.Descriptor instead. func (WorkerState) EnumDescriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{7} + return file_ateapi_proto_rawDescGZIP(), []int{8} } // Scope is whether a top-level resource belongs to an atespace. @@ -516,11 +572,11 @@ func (x Scope) String() string { } func (Scope) Descriptor() protoreflect.EnumDescriptor { - return file_ateapi_proto_enumTypes[8].Descriptor() + return file_ateapi_proto_enumTypes[9].Descriptor() } func (Scope) Type() protoreflect.EnumType { - return &file_ateapi_proto_enumTypes[8] + return &file_ateapi_proto_enumTypes[9] } func (x Scope) Number() protoreflect.EnumNumber { @@ -529,7 +585,7 @@ func (x Scope) Number() protoreflect.EnumNumber { // Deprecated: Use Scope.Descriptor instead. func (Scope) EnumDescriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{8} + return file_ateapi_proto_rawDescGZIP(), []int{9} } // ObjectSnapshot addresses a snapshot held in object storage. @@ -685,7 +741,9 @@ type Snapshot struct { // +k8s:format=k8s-long-name Locality string `protobuf:"bytes,6,opt,name=locality,proto3" json:"locality,omitempty"` // durable_snapshot holds the durable storage representation of this snapshot. - // Unset when the snapshot only exists locally. + // PENDING while the snapshot is uploaded, including a pause's snapshot that + // so far exists only on the node named by locality. COMPLETED once the + // upload is recorded. // // +k8s:optional DurableSnapshot *SnapshotStorage `protobuf:"bytes,7,opt,name=durable_snapshot,json=durableSnapshot,proto3" json:"durable_snapshot,omitempty"` @@ -1087,7 +1145,8 @@ type Actor struct { ActorTemplate *ObjectRef `protobuf:"bytes,4,opt,name=actor_template,json=actorTemplate,proto3" json:"actor_template,omitempty"` // worker_selector is the per-actor placement constraint. The scheduler // evaluates the AND of this selector and the template's workerSelector to - // find eligible pools. + // find eligible pools. It narrows the template's selection; pinning a + // sandbox class belongs on the template's worker_selector. // Changes take effect on the next ResumeActor call. // // +k8s:optional @@ -2572,7 +2631,13 @@ type ActorTemplate struct { // +k8s:subfield(atespace)=+k8s:required Metadata *ResourceMetadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` // worker_selector restricts which worker pools actors from this template - // may use. + // may use, by matching the pools' labels. Unset, every pool is eligible. + // + // The pool decides the sandbox class an actor runs in, so this selector is + // also how a template pins one: if the cluster runs pools of more than one + // sandbox class and the workload needs a particular one, select only pools + // of that class. An actor booting with no snapshot and no selector may land + // on a pool of any class, and keeps that class from its first checkpoint on. // // +k8s:optional WorkerSelector *Selector `protobuf:"bytes,2,opt,name=worker_selector,json=workerSelector,proto3" json:"worker_selector,omitempty"` @@ -2588,10 +2653,6 @@ type ActorTemplate struct { Volumes []*Volume `protobuf:"bytes,4,rep,name=volumes,proto3" json:"volumes,omitempty"` // +k8s:required SnapshotConfig *SnapshotConfig `protobuf:"bytes,5,opt,name=snapshot_config,json=snapshotConfig,proto3" json:"snapshot_config,omitempty"` - // sandbox_config selects the sandbox runtime this version's actors run on. - // - // +k8s:required - SandboxConfig *SandboxConfig `protobuf:"bytes,6,opt,name=sandbox_config,json=sandboxConfig,proto3" json:"sandbox_config,omitempty"` // resources is the actor's compute size. Both cpu and memory limits are // required: the scheduler books them against worker capacity, and a // micro-VM is booted at that size. @@ -2670,13 +2731,6 @@ func (x *ActorTemplate) GetSnapshotConfig() *SnapshotConfig { return nil } -func (x *ActorTemplate) GetSandboxConfig() *SandboxConfig { - if x != nil { - return x.SandboxConfig - } - return nil -} - func (x *ActorTemplate) GetResources() *Resources { if x != nil { return x.Resources @@ -2921,70 +2975,6 @@ func (x *ActorTemplateStatus) GetGoldenSnapshotStatus() *GoldenSnapshotStatus { return nil } -// SandboxConfig selects the sandbox runtime for an ActorTemplate. -type SandboxConfig struct { - state protoimpl.MessageState `protogen:"open.v1"` - // sandbox_class selects the sandbox runtime family: "gvisor" or "microvm". - // Snapshots are not portable across classes. - // - // +k8s:required - // +k8s:maxLength=63 - // +k8s:customValidation # must be "gvisor" or "microvm" - SandboxClass string `protobuf:"bytes,1,opt,name=sandbox_class,json=sandboxClass,proto3" json:"sandbox_class,omitempty"` - // config_name names the cluster-scoped SandboxConfig Kubernetes object - // supplying the sandbox binaries. Required; must match sandbox_class. - // - // +k8s:required - // +k8s:format=k8s-long-name - ConfigName string `protobuf:"bytes,2,opt,name=config_name,json=configName,proto3" json:"config_name,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *SandboxConfig) Reset() { - *x = SandboxConfig{} - mi := &file_ateapi_proto_msgTypes[29] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *SandboxConfig) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SandboxConfig) ProtoMessage() {} - -func (x *SandboxConfig) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[29] - if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SandboxConfig.ProtoReflect.Descriptor instead. -func (*SandboxConfig) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{29} -} - -func (x *SandboxConfig) GetSandboxClass() string { - if x != nil { - return x.SandboxClass - } - return "" -} - -func (x *SandboxConfig) GetConfigName() string { - if x != nil { - return x.ConfigName - } - return "" -} - // SnapshotConfig selects what actor snapshots capture and where they are // stored. type SnapshotConfig struct { @@ -3002,9 +2992,7 @@ type SnapshotConfig struct { // more than this level, and it may package less when a layer cannot be // captured or is too expensive to capture at the time, for example memory // under host pressure. A process resumed in place on its worker keeps its - // memory regardless of this setting. ROOTFS requires - // sandbox_config.sandbox_class "microvm". Defaults to MEMORY - // when unset. + // memory regardless of this setting. Defaults to MEMORY when unset. // // +k8s:required // +k8s:minimum=1 @@ -3021,7 +3009,7 @@ type SnapshotConfig struct { func (x *SnapshotConfig) Reset() { *x = SnapshotConfig{} - mi := &file_ateapi_proto_msgTypes[30] + mi := &file_ateapi_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3033,7 +3021,7 @@ func (x *SnapshotConfig) String() string { func (*SnapshotConfig) ProtoMessage() {} func (x *SnapshotConfig) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[30] + mi := &file_ateapi_proto_msgTypes[29] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3046,7 +3034,7 @@ func (x *SnapshotConfig) ProtoReflect() protoreflect.Message { // Deprecated: Use SnapshotConfig.ProtoReflect.Descriptor instead. func (*SnapshotConfig) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{30} + return file_ateapi_proto_rawDescGZIP(), []int{29} } func (x *SnapshotConfig) GetStorageLocation() string { @@ -3086,7 +3074,7 @@ type GoldenSnapshotConfig struct { func (x *GoldenSnapshotConfig) Reset() { *x = GoldenSnapshotConfig{} - mi := &file_ateapi_proto_msgTypes[31] + mi := &file_ateapi_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3098,7 +3086,7 @@ func (x *GoldenSnapshotConfig) String() string { func (*GoldenSnapshotConfig) ProtoMessage() {} func (x *GoldenSnapshotConfig) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[31] + mi := &file_ateapi_proto_msgTypes[30] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3111,7 +3099,7 @@ func (x *GoldenSnapshotConfig) ProtoReflect() protoreflect.Message { // Deprecated: Use GoldenSnapshotConfig.ProtoReflect.Descriptor instead. func (*GoldenSnapshotConfig) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{31} + return file_ateapi_proto_rawDescGZIP(), []int{30} } func (x *GoldenSnapshotConfig) GetMode() GoldenSnapshotMode { @@ -3191,7 +3179,7 @@ type Container struct { func (x *Container) Reset() { *x = Container{} - mi := &file_ateapi_proto_msgTypes[32] + mi := &file_ateapi_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3203,7 +3191,7 @@ func (x *Container) String() string { func (*Container) ProtoMessage() {} func (x *Container) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[32] + mi := &file_ateapi_proto_msgTypes[31] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3216,7 +3204,7 @@ func (x *Container) ProtoReflect() protoreflect.Message { // Deprecated: Use Container.ProtoReflect.Descriptor instead. func (*Container) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{32} + return file_ateapi_proto_rawDescGZIP(), []int{31} } func (x *Container) GetName() string { @@ -3296,7 +3284,7 @@ type SecurityContext struct { func (x *SecurityContext) Reset() { *x = SecurityContext{} - mi := &file_ateapi_proto_msgTypes[33] + mi := &file_ateapi_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3308,7 +3296,7 @@ func (x *SecurityContext) String() string { func (*SecurityContext) ProtoMessage() {} func (x *SecurityContext) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[33] + mi := &file_ateapi_proto_msgTypes[32] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3321,7 +3309,7 @@ func (x *SecurityContext) ProtoReflect() protoreflect.Message { // Deprecated: Use SecurityContext.ProtoReflect.Descriptor instead. func (*SecurityContext) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{33} + return file_ateapi_proto_rawDescGZIP(), []int{32} } func (x *SecurityContext) GetCapabilities() *Capabilities { @@ -3359,7 +3347,7 @@ type Capabilities struct { func (x *Capabilities) Reset() { *x = Capabilities{} - mi := &file_ateapi_proto_msgTypes[34] + mi := &file_ateapi_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3371,7 +3359,7 @@ func (x *Capabilities) String() string { func (*Capabilities) ProtoMessage() {} func (x *Capabilities) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[34] + mi := &file_ateapi_proto_msgTypes[33] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3384,7 +3372,7 @@ func (x *Capabilities) ProtoReflect() protoreflect.Message { // Deprecated: Use Capabilities.ProtoReflect.Descriptor instead. func (*Capabilities) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{34} + return file_ateapi_proto_rawDescGZIP(), []int{33} } func (x *Capabilities) GetAdd() []string { @@ -3422,7 +3410,7 @@ type EnvVar struct { func (x *EnvVar) Reset() { *x = EnvVar{} - mi := &file_ateapi_proto_msgTypes[35] + mi := &file_ateapi_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3434,7 +3422,7 @@ func (x *EnvVar) String() string { func (*EnvVar) ProtoMessage() {} func (x *EnvVar) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[35] + mi := &file_ateapi_proto_msgTypes[34] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3447,7 +3435,7 @@ func (x *EnvVar) ProtoReflect() protoreflect.Message { // Deprecated: Use EnvVar.ProtoReflect.Descriptor instead. func (*EnvVar) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{35} + return file_ateapi_proto_rawDescGZIP(), []int{34} } func (x *EnvVar) GetName() string { @@ -3487,7 +3475,7 @@ type WakeupProbe struct { func (x *WakeupProbe) Reset() { *x = WakeupProbe{} - mi := &file_ateapi_proto_msgTypes[36] + mi := &file_ateapi_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3499,7 +3487,7 @@ func (x *WakeupProbe) String() string { func (*WakeupProbe) ProtoMessage() {} func (x *WakeupProbe) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[36] + mi := &file_ateapi_proto_msgTypes[35] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3512,7 +3500,7 @@ func (x *WakeupProbe) ProtoReflect() protoreflect.Message { // Deprecated: Use WakeupProbe.ProtoReflect.Descriptor instead. func (*WakeupProbe) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{36} + return file_ateapi_proto_rawDescGZIP(), []int{35} } func (x *WakeupProbe) GetHttpGet() *HttpGetAction { @@ -3552,7 +3540,7 @@ type HttpGetAction struct { func (x *HttpGetAction) Reset() { *x = HttpGetAction{} - mi := &file_ateapi_proto_msgTypes[37] + mi := &file_ateapi_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3564,7 +3552,7 @@ func (x *HttpGetAction) String() string { func (*HttpGetAction) ProtoMessage() {} func (x *HttpGetAction) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[37] + mi := &file_ateapi_proto_msgTypes[36] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3577,7 +3565,7 @@ func (x *HttpGetAction) ProtoReflect() protoreflect.Message { // Deprecated: Use HttpGetAction.ProtoReflect.Descriptor instead. func (*HttpGetAction) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{37} + return file_ateapi_proto_rawDescGZIP(), []int{36} } func (x *HttpGetAction) GetPath() string { @@ -3626,7 +3614,7 @@ type Volume struct { func (x *Volume) Reset() { *x = Volume{} - mi := &file_ateapi_proto_msgTypes[38] + mi := &file_ateapi_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3638,7 +3626,7 @@ func (x *Volume) String() string { func (*Volume) ProtoMessage() {} func (x *Volume) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[38] + mi := &file_ateapi_proto_msgTypes[37] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3651,7 +3639,7 @@ func (x *Volume) ProtoReflect() protoreflect.Message { // Deprecated: Use Volume.ProtoReflect.Descriptor instead. func (*Volume) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{38} + return file_ateapi_proto_rawDescGZIP(), []int{37} } func (x *Volume) GetName() string { @@ -3707,7 +3695,7 @@ type ImageVolumeSource struct { func (x *ImageVolumeSource) Reset() { *x = ImageVolumeSource{} - mi := &file_ateapi_proto_msgTypes[39] + mi := &file_ateapi_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3719,7 +3707,7 @@ func (x *ImageVolumeSource) String() string { func (*ImageVolumeSource) ProtoMessage() {} func (x *ImageVolumeSource) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[39] + mi := &file_ateapi_proto_msgTypes[38] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3732,7 +3720,7 @@ func (x *ImageVolumeSource) ProtoReflect() protoreflect.Message { // Deprecated: Use ImageVolumeSource.ProtoReflect.Descriptor instead. func (*ImageVolumeSource) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{39} + return file_ateapi_proto_rawDescGZIP(), []int{38} } func (x *ImageVolumeSource) GetReference() string { @@ -3752,7 +3740,7 @@ type DurableDirVolumeSource struct { func (x *DurableDirVolumeSource) Reset() { *x = DurableDirVolumeSource{} - mi := &file_ateapi_proto_msgTypes[40] + mi := &file_ateapi_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3764,7 +3752,7 @@ func (x *DurableDirVolumeSource) String() string { func (*DurableDirVolumeSource) ProtoMessage() {} func (x *DurableDirVolumeSource) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[40] + mi := &file_ateapi_proto_msgTypes[39] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3777,7 +3765,7 @@ func (x *DurableDirVolumeSource) ProtoReflect() protoreflect.Message { // Deprecated: Use DurableDirVolumeSource.ProtoReflect.Descriptor instead. func (*DurableDirVolumeSource) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{40} + return file_ateapi_proto_rawDescGZIP(), []int{39} } // ExternalVolumeTemplate provisions an external volume per actor; the volume @@ -3803,7 +3791,7 @@ type ExternalVolumeTemplate struct { func (x *ExternalVolumeTemplate) Reset() { *x = ExternalVolumeTemplate{} - mi := &file_ateapi_proto_msgTypes[41] + mi := &file_ateapi_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3815,7 +3803,7 @@ func (x *ExternalVolumeTemplate) String() string { func (*ExternalVolumeTemplate) ProtoMessage() {} func (x *ExternalVolumeTemplate) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[41] + mi := &file_ateapi_proto_msgTypes[40] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3828,7 +3816,7 @@ func (x *ExternalVolumeTemplate) ProtoReflect() protoreflect.Message { // Deprecated: Use ExternalVolumeTemplate.ProtoReflect.Descriptor instead. func (*ExternalVolumeTemplate) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{41} + return file_ateapi_proto_rawDescGZIP(), []int{40} } func (x *ExternalVolumeTemplate) GetCapacity() string { @@ -3865,7 +3853,7 @@ type SystemInfoVolumeSource struct { func (x *SystemInfoVolumeSource) Reset() { *x = SystemInfoVolumeSource{} - mi := &file_ateapi_proto_msgTypes[42] + mi := &file_ateapi_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3877,7 +3865,7 @@ func (x *SystemInfoVolumeSource) String() string { func (*SystemInfoVolumeSource) ProtoMessage() {} func (x *SystemInfoVolumeSource) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[42] + mi := &file_ateapi_proto_msgTypes[41] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3890,7 +3878,7 @@ func (x *SystemInfoVolumeSource) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemInfoVolumeSource.ProtoReflect.Descriptor instead. func (*SystemInfoVolumeSource) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{42} + return file_ateapi_proto_rawDescGZIP(), []int{41} } func (x *SystemInfoVolumeSource) GetDataSources() []*SystemInfoDataSource { @@ -3916,7 +3904,7 @@ type SystemInfoDataSource struct { func (x *SystemInfoDataSource) Reset() { *x = SystemInfoDataSource{} - mi := &file_ateapi_proto_msgTypes[43] + mi := &file_ateapi_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3928,7 +3916,7 @@ func (x *SystemInfoDataSource) String() string { func (*SystemInfoDataSource) ProtoMessage() {} func (x *SystemInfoDataSource) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[43] + mi := &file_ateapi_proto_msgTypes[42] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3941,7 +3929,7 @@ func (x *SystemInfoDataSource) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemInfoDataSource.ProtoReflect.Descriptor instead. func (*SystemInfoDataSource) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{43} + return file_ateapi_proto_rawDescGZIP(), []int{42} } func (x *SystemInfoDataSource) GetActorMetadata() *ActorMetadataDataSource { @@ -3978,7 +3966,7 @@ type ActorMetadataDataSource struct { func (x *ActorMetadataDataSource) Reset() { *x = ActorMetadataDataSource{} - mi := &file_ateapi_proto_msgTypes[44] + mi := &file_ateapi_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3990,7 +3978,7 @@ func (x *ActorMetadataDataSource) String() string { func (*ActorMetadataDataSource) ProtoMessage() {} func (x *ActorMetadataDataSource) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[44] + mi := &file_ateapi_proto_msgTypes[43] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4003,7 +3991,7 @@ func (x *ActorMetadataDataSource) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorMetadataDataSource.ProtoReflect.Descriptor instead. func (*ActorMetadataDataSource) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{44} + return file_ateapi_proto_rawDescGZIP(), []int{43} } func (x *ActorMetadataDataSource) GetItems() []*ActorMetadataItem { @@ -4035,7 +4023,7 @@ type ActorMetadataItem struct { func (x *ActorMetadataItem) Reset() { *x = ActorMetadataItem{} - mi := &file_ateapi_proto_msgTypes[45] + mi := &file_ateapi_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4047,7 +4035,7 @@ func (x *ActorMetadataItem) String() string { func (*ActorMetadataItem) ProtoMessage() {} func (x *ActorMetadataItem) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[45] + mi := &file_ateapi_proto_msgTypes[44] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4060,7 +4048,7 @@ func (x *ActorMetadataItem) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorMetadataItem.ProtoReflect.Descriptor instead. func (*ActorMetadataItem) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{45} + return file_ateapi_proto_rawDescGZIP(), []int{44} } func (x *ActorMetadataItem) GetField() ActorMetadataField { @@ -4112,7 +4100,7 @@ type TrustBundleDataSource struct { func (x *TrustBundleDataSource) Reset() { *x = TrustBundleDataSource{} - mi := &file_ateapi_proto_msgTypes[46] + mi := &file_ateapi_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4124,7 +4112,7 @@ func (x *TrustBundleDataSource) String() string { func (*TrustBundleDataSource) ProtoMessage() {} func (x *TrustBundleDataSource) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[46] + mi := &file_ateapi_proto_msgTypes[45] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4137,7 +4125,7 @@ func (x *TrustBundleDataSource) ProtoReflect() protoreflect.Message { // Deprecated: Use TrustBundleDataSource.ProtoReflect.Descriptor instead. func (*TrustBundleDataSource) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{46} + return file_ateapi_proto_rawDescGZIP(), []int{45} } func (x *TrustBundleDataSource) GetNames() []string { @@ -4176,7 +4164,7 @@ type VolumeMount struct { func (x *VolumeMount) Reset() { *x = VolumeMount{} - mi := &file_ateapi_proto_msgTypes[47] + mi := &file_ateapi_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4188,7 +4176,7 @@ func (x *VolumeMount) String() string { func (*VolumeMount) ProtoMessage() {} func (x *VolumeMount) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[47] + mi := &file_ateapi_proto_msgTypes[46] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4201,7 +4189,7 @@ func (x *VolumeMount) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeMount.ProtoReflect.Descriptor instead. func (*VolumeMount) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{47} + return file_ateapi_proto_rawDescGZIP(), []int{46} } func (x *VolumeMount) GetName() string { @@ -4230,7 +4218,7 @@ type CreateAtespaceRequest struct { func (x *CreateAtespaceRequest) Reset() { *x = CreateAtespaceRequest{} - mi := &file_ateapi_proto_msgTypes[48] + mi := &file_ateapi_proto_msgTypes[47] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4242,7 +4230,7 @@ func (x *CreateAtespaceRequest) String() string { func (*CreateAtespaceRequest) ProtoMessage() {} func (x *CreateAtespaceRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[48] + mi := &file_ateapi_proto_msgTypes[47] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4255,7 +4243,7 @@ func (x *CreateAtespaceRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateAtespaceRequest.ProtoReflect.Descriptor instead. func (*CreateAtespaceRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{48} + return file_ateapi_proto_rawDescGZIP(), []int{47} } func (x *CreateAtespaceRequest) GetAtespace() *Atespace { @@ -4276,7 +4264,7 @@ type GetAtespaceRequest struct { func (x *GetAtespaceRequest) Reset() { *x = GetAtespaceRequest{} - mi := &file_ateapi_proto_msgTypes[49] + mi := &file_ateapi_proto_msgTypes[48] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4288,7 +4276,7 @@ func (x *GetAtespaceRequest) String() string { func (*GetAtespaceRequest) ProtoMessage() {} func (x *GetAtespaceRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[49] + mi := &file_ateapi_proto_msgTypes[48] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4301,7 +4289,7 @@ func (x *GetAtespaceRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetAtespaceRequest.ProtoReflect.Descriptor instead. func (*GetAtespaceRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{49} + return file_ateapi_proto_rawDescGZIP(), []int{48} } func (x *GetAtespaceRequest) GetAtespace() *ObjectRef { @@ -4323,7 +4311,7 @@ type ListAtespacesRequest struct { func (x *ListAtespacesRequest) Reset() { *x = ListAtespacesRequest{} - mi := &file_ateapi_proto_msgTypes[50] + mi := &file_ateapi_proto_msgTypes[49] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4335,7 +4323,7 @@ func (x *ListAtespacesRequest) String() string { func (*ListAtespacesRequest) ProtoMessage() {} func (x *ListAtespacesRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[50] + mi := &file_ateapi_proto_msgTypes[49] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4348,7 +4336,7 @@ func (x *ListAtespacesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListAtespacesRequest.ProtoReflect.Descriptor instead. func (*ListAtespacesRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{50} + return file_ateapi_proto_rawDescGZIP(), []int{49} } func (x *ListAtespacesRequest) GetOptions() *ListOptions { @@ -4370,7 +4358,7 @@ type ListAtespacesResponse struct { func (x *ListAtespacesResponse) Reset() { *x = ListAtespacesResponse{} - mi := &file_ateapi_proto_msgTypes[51] + mi := &file_ateapi_proto_msgTypes[50] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4382,7 +4370,7 @@ func (x *ListAtespacesResponse) String() string { func (*ListAtespacesResponse) ProtoMessage() {} func (x *ListAtespacesResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[51] + mi := &file_ateapi_proto_msgTypes[50] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4395,7 +4383,7 @@ func (x *ListAtespacesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListAtespacesResponse.ProtoReflect.Descriptor instead. func (*ListAtespacesResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{51} + return file_ateapi_proto_rawDescGZIP(), []int{50} } func (x *ListAtespacesResponse) GetAtespaces() []*Atespace { @@ -4427,7 +4415,7 @@ type DeleteAtespaceRequest struct { func (x *DeleteAtespaceRequest) Reset() { *x = DeleteAtespaceRequest{} - mi := &file_ateapi_proto_msgTypes[52] + mi := &file_ateapi_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4439,7 +4427,7 @@ func (x *DeleteAtespaceRequest) String() string { func (*DeleteAtespaceRequest) ProtoMessage() {} func (x *DeleteAtespaceRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[52] + mi := &file_ateapi_proto_msgTypes[51] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4452,7 +4440,7 @@ func (x *DeleteAtespaceRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteAtespaceRequest.ProtoReflect.Descriptor instead. func (*DeleteAtespaceRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{52} + return file_ateapi_proto_rawDescGZIP(), []int{51} } func (x *DeleteAtespaceRequest) GetAtespace() *ObjectRef { @@ -4475,7 +4463,7 @@ type CreateActorTemplateRequest struct { // version, timestamps) is ignored, as are the status fields. // // +k8s:required - // +k8s:customValidation # volume_mounts must reference declared volumes; ROOTFS fidelity requires the micro-VM sandbox class + // +k8s:customValidation # volume_mounts must reference declared volumes ActorTemplate *ActorTemplate `protobuf:"bytes,1,opt,name=actor_template,json=actorTemplate,proto3" json:"actor_template,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -4483,7 +4471,7 @@ type CreateActorTemplateRequest struct { func (x *CreateActorTemplateRequest) Reset() { *x = CreateActorTemplateRequest{} - mi := &file_ateapi_proto_msgTypes[53] + mi := &file_ateapi_proto_msgTypes[52] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4495,7 +4483,7 @@ func (x *CreateActorTemplateRequest) String() string { func (*CreateActorTemplateRequest) ProtoMessage() {} func (x *CreateActorTemplateRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[53] + mi := &file_ateapi_proto_msgTypes[52] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4508,7 +4496,7 @@ func (x *CreateActorTemplateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateActorTemplateRequest.ProtoReflect.Descriptor instead. func (*CreateActorTemplateRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{53} + return file_ateapi_proto_rawDescGZIP(), []int{52} } func (x *CreateActorTemplateRequest) GetActorTemplate() *ActorTemplate { @@ -4529,7 +4517,7 @@ type GetActorTemplateRequest struct { func (x *GetActorTemplateRequest) Reset() { *x = GetActorTemplateRequest{} - mi := &file_ateapi_proto_msgTypes[54] + mi := &file_ateapi_proto_msgTypes[53] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4541,7 +4529,7 @@ func (x *GetActorTemplateRequest) String() string { func (*GetActorTemplateRequest) ProtoMessage() {} func (x *GetActorTemplateRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[54] + mi := &file_ateapi_proto_msgTypes[53] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4554,7 +4542,7 @@ func (x *GetActorTemplateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetActorTemplateRequest.ProtoReflect.Descriptor instead. func (*GetActorTemplateRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{54} + return file_ateapi_proto_rawDescGZIP(), []int{53} } func (x *GetActorTemplateRequest) GetActorTemplate() *ObjectRef { @@ -4582,7 +4570,7 @@ type ListActorTemplatesRequest struct { func (x *ListActorTemplatesRequest) Reset() { *x = ListActorTemplatesRequest{} - mi := &file_ateapi_proto_msgTypes[55] + mi := &file_ateapi_proto_msgTypes[54] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4594,7 +4582,7 @@ func (x *ListActorTemplatesRequest) String() string { func (*ListActorTemplatesRequest) ProtoMessage() {} func (x *ListActorTemplatesRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[55] + mi := &file_ateapi_proto_msgTypes[54] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4607,7 +4595,7 @@ func (x *ListActorTemplatesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListActorTemplatesRequest.ProtoReflect.Descriptor instead. func (*ListActorTemplatesRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{55} + return file_ateapi_proto_rawDescGZIP(), []int{54} } func (x *ListActorTemplatesRequest) GetAtespace() string { @@ -4637,7 +4625,7 @@ type ListActorTemplatesResponse struct { func (x *ListActorTemplatesResponse) Reset() { *x = ListActorTemplatesResponse{} - mi := &file_ateapi_proto_msgTypes[56] + mi := &file_ateapi_proto_msgTypes[55] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4649,7 +4637,7 @@ func (x *ListActorTemplatesResponse) String() string { func (*ListActorTemplatesResponse) ProtoMessage() {} func (x *ListActorTemplatesResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[56] + mi := &file_ateapi_proto_msgTypes[55] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4662,7 +4650,7 @@ func (x *ListActorTemplatesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListActorTemplatesResponse.ProtoReflect.Descriptor instead. func (*ListActorTemplatesResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{56} + return file_ateapi_proto_rawDescGZIP(), []int{55} } func (x *ListActorTemplatesResponse) GetActorTemplates() []*ActorTemplate { @@ -4694,7 +4682,7 @@ type DeleteActorTemplateRequest struct { func (x *DeleteActorTemplateRequest) Reset() { *x = DeleteActorTemplateRequest{} - mi := &file_ateapi_proto_msgTypes[57] + mi := &file_ateapi_proto_msgTypes[56] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4706,7 +4694,7 @@ func (x *DeleteActorTemplateRequest) String() string { func (*DeleteActorTemplateRequest) ProtoMessage() {} func (x *DeleteActorTemplateRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[57] + mi := &file_ateapi_proto_msgTypes[56] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4719,7 +4707,7 @@ func (x *DeleteActorTemplateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteActorTemplateRequest.ProtoReflect.Descriptor instead. func (*DeleteActorTemplateRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{57} + return file_ateapi_proto_rawDescGZIP(), []int{56} } func (x *DeleteActorTemplateRequest) GetActorTemplate() *ObjectRef { @@ -4747,7 +4735,7 @@ type GetActorRequest struct { func (x *GetActorRequest) Reset() { *x = GetActorRequest{} - mi := &file_ateapi_proto_msgTypes[58] + mi := &file_ateapi_proto_msgTypes[57] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4759,7 +4747,7 @@ func (x *GetActorRequest) String() string { func (*GetActorRequest) ProtoMessage() {} func (x *GetActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[58] + mi := &file_ateapi_proto_msgTypes[57] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4772,7 +4760,7 @@ func (x *GetActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetActorRequest.ProtoReflect.Descriptor instead. func (*GetActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{58} + return file_ateapi_proto_rawDescGZIP(), []int{57} } func (x *GetActorRequest) GetActor() *ObjectRef { @@ -4795,7 +4783,7 @@ type CreateActorRequest struct { func (x *CreateActorRequest) Reset() { *x = CreateActorRequest{} - mi := &file_ateapi_proto_msgTypes[59] + mi := &file_ateapi_proto_msgTypes[58] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4807,7 +4795,7 @@ func (x *CreateActorRequest) String() string { func (*CreateActorRequest) ProtoMessage() {} func (x *CreateActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[59] + mi := &file_ateapi_proto_msgTypes[58] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4820,7 +4808,7 @@ func (x *CreateActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateActorRequest.ProtoReflect.Descriptor instead. func (*CreateActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{59} + return file_ateapi_proto_rawDescGZIP(), []int{58} } func (x *CreateActorRequest) GetActor() *Actor { @@ -4851,7 +4839,7 @@ type UpdateActorRequest struct { func (x *UpdateActorRequest) Reset() { *x = UpdateActorRequest{} - mi := &file_ateapi_proto_msgTypes[60] + mi := &file_ateapi_proto_msgTypes[59] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4863,7 +4851,7 @@ func (x *UpdateActorRequest) String() string { func (*UpdateActorRequest) ProtoMessage() {} func (x *UpdateActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[60] + mi := &file_ateapi_proto_msgTypes[59] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4876,7 +4864,7 @@ func (x *UpdateActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateActorRequest.ProtoReflect.Descriptor instead. func (*UpdateActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{60} + return file_ateapi_proto_rawDescGZIP(), []int{59} } func (x *UpdateActorRequest) GetActor() *Actor { @@ -4890,14 +4878,18 @@ type SuspendActorRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // +k8s:required // +k8s:subfield(atespace)=+k8s:required - Actor *ObjectRef `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + Actor *ObjectRef `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + // options controls how SuspendActor persists the actor's snapshot. + // + // +k8s:optional + Options *SuspendOptions `protobuf:"bytes,2,opt,name=options,proto3" json:"options,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *SuspendActorRequest) Reset() { *x = SuspendActorRequest{} - mi := &file_ateapi_proto_msgTypes[61] + mi := &file_ateapi_proto_msgTypes[60] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4909,7 +4901,7 @@ func (x *SuspendActorRequest) String() string { func (*SuspendActorRequest) ProtoMessage() {} func (x *SuspendActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[61] + mi := &file_ateapi_proto_msgTypes[60] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4922,7 +4914,7 @@ func (x *SuspendActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use SuspendActorRequest.ProtoReflect.Descriptor instead. func (*SuspendActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{61} + return file_ateapi_proto_rawDescGZIP(), []int{60} } func (x *SuspendActorRequest) GetActor() *ObjectRef { @@ -4932,6 +4924,13 @@ func (x *SuspendActorRequest) GetActor() *ObjectRef { return nil } +func (x *SuspendActorRequest) GetOptions() *SuspendOptions { + if x != nil { + return x.Options + } + return nil +} + type SuspendActorResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Actor *Actor `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` @@ -4941,7 +4940,7 @@ type SuspendActorResponse struct { func (x *SuspendActorResponse) Reset() { *x = SuspendActorResponse{} - mi := &file_ateapi_proto_msgTypes[62] + mi := &file_ateapi_proto_msgTypes[61] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4953,7 +4952,7 @@ func (x *SuspendActorResponse) String() string { func (*SuspendActorResponse) ProtoMessage() {} func (x *SuspendActorResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[62] + mi := &file_ateapi_proto_msgTypes[61] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4966,7 +4965,7 @@ func (x *SuspendActorResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SuspendActorResponse.ProtoReflect.Descriptor instead. func (*SuspendActorResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{62} + return file_ateapi_proto_rawDescGZIP(), []int{61} } func (x *SuspendActorResponse) GetActor() *Actor { @@ -4976,74 +4975,35 @@ func (x *SuspendActorResponse) GetActor() *Actor { return nil } -type PauseActorRequest struct { +// SuspendOptions tunes how SuspendActor persists the actor's snapshot. +type SuspendOptions struct { state protoimpl.MessageState `protogen:"open.v1"` - // +k8s:required - // +k8s:subfield(atespace)=+k8s:required - Actor *ObjectRef `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *PauseActorRequest) Reset() { - *x = PauseActorRequest{} - mi := &file_ateapi_proto_msgTypes[63] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *PauseActorRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*PauseActorRequest) ProtoMessage() {} - -func (x *PauseActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[63] - if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use PauseActorRequest.ProtoReflect.Descriptor instead. -func (*PauseActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{63} -} - -func (x *PauseActorRequest) GetActor() *ObjectRef { - if x != nil { - return x.Actor - } - return nil -} - -type PauseActorResponse struct { - state protoimpl.MessageState `protogen:"open.v1"` - Actor *Actor `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + // mode selects when SuspendActor returns relative to the snapshot reaching + // durable storage. If unspecified, defaults to DURABILITY_MODE_ASYNCHRONOUS. + // + // +k8s:optional + // +k8s:minimum=1 + // +k8s:maximum=2 # keep this in sync with the DurabilityMode enum + Mode DurabilityMode `protobuf:"varint,1,opt,name=mode,proto3,enum=ateapi.DurabilityMode" json:"mode,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } -func (x *PauseActorResponse) Reset() { - *x = PauseActorResponse{} - mi := &file_ateapi_proto_msgTypes[64] +func (x *SuspendOptions) Reset() { + *x = SuspendOptions{} + mi := &file_ateapi_proto_msgTypes[62] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *PauseActorResponse) String() string { +func (x *SuspendOptions) String() string { return protoimpl.X.MessageStringOf(x) } -func (*PauseActorResponse) ProtoMessage() {} +func (*SuspendOptions) ProtoMessage() {} -func (x *PauseActorResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[64] +func (x *SuspendOptions) ProtoReflect() protoreflect.Message { + mi := &file_ateapi_proto_msgTypes[62] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5054,16 +5014,16 @@ func (x *PauseActorResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use PauseActorResponse.ProtoReflect.Descriptor instead. -func (*PauseActorResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{64} +// Deprecated: Use SuspendOptions.ProtoReflect.Descriptor instead. +func (*SuspendOptions) Descriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{62} } -func (x *PauseActorResponse) GetActor() *Actor { +func (x *SuspendOptions) GetMode() DurabilityMode { if x != nil { - return x.Actor + return x.Mode } - return nil + return DurabilityMode_DURABILITY_MODE_UNSPECIFIED } type ResumeActorRequest struct { @@ -5077,7 +5037,7 @@ type ResumeActorRequest struct { func (x *ResumeActorRequest) Reset() { *x = ResumeActorRequest{} - mi := &file_ateapi_proto_msgTypes[65] + mi := &file_ateapi_proto_msgTypes[63] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5089,7 +5049,7 @@ func (x *ResumeActorRequest) String() string { func (*ResumeActorRequest) ProtoMessage() {} func (x *ResumeActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[65] + mi := &file_ateapi_proto_msgTypes[63] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5102,7 +5062,7 @@ func (x *ResumeActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ResumeActorRequest.ProtoReflect.Descriptor instead. func (*ResumeActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{65} + return file_ateapi_proto_rawDescGZIP(), []int{63} } func (x *ResumeActorRequest) GetActor() *ObjectRef { @@ -5124,7 +5084,7 @@ type ResumeActorResponse struct { func (x *ResumeActorResponse) Reset() { *x = ResumeActorResponse{} - mi := &file_ateapi_proto_msgTypes[66] + mi := &file_ateapi_proto_msgTypes[64] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5136,7 +5096,7 @@ func (x *ResumeActorResponse) String() string { func (*ResumeActorResponse) ProtoMessage() {} func (x *ResumeActorResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[66] + mi := &file_ateapi_proto_msgTypes[64] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5149,7 +5109,7 @@ func (x *ResumeActorResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ResumeActorResponse.ProtoReflect.Descriptor instead. func (*ResumeActorResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{66} + return file_ateapi_proto_rawDescGZIP(), []int{64} } func (x *ResumeActorResponse) GetActor() *Actor { @@ -5179,7 +5139,7 @@ type RevertActorRequest struct { func (x *RevertActorRequest) Reset() { *x = RevertActorRequest{} - mi := &file_ateapi_proto_msgTypes[67] + mi := &file_ateapi_proto_msgTypes[65] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5191,7 +5151,7 @@ func (x *RevertActorRequest) String() string { func (*RevertActorRequest) ProtoMessage() {} func (x *RevertActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[67] + mi := &file_ateapi_proto_msgTypes[65] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5204,7 +5164,7 @@ func (x *RevertActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RevertActorRequest.ProtoReflect.Descriptor instead. func (*RevertActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{67} + return file_ateapi_proto_rawDescGZIP(), []int{65} } func (x *RevertActorRequest) GetActor() *ObjectRef { @@ -5226,7 +5186,7 @@ type RevertActorResponse struct { func (x *RevertActorResponse) Reset() { *x = RevertActorResponse{} - mi := &file_ateapi_proto_msgTypes[68] + mi := &file_ateapi_proto_msgTypes[66] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5238,7 +5198,7 @@ func (x *RevertActorResponse) String() string { func (*RevertActorResponse) ProtoMessage() {} func (x *RevertActorResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[68] + mi := &file_ateapi_proto_msgTypes[66] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5251,7 +5211,7 @@ func (x *RevertActorResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RevertActorResponse.ProtoReflect.Descriptor instead. func (*RevertActorResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{68} + return file_ateapi_proto_rawDescGZIP(), []int{66} } func (x *RevertActorResponse) GetActor() *Actor { @@ -5280,7 +5240,7 @@ type DeleteActorRequest struct { func (x *DeleteActorRequest) Reset() { *x = DeleteActorRequest{} - mi := &file_ateapi_proto_msgTypes[69] + mi := &file_ateapi_proto_msgTypes[67] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5292,7 +5252,7 @@ func (x *DeleteActorRequest) String() string { func (*DeleteActorRequest) ProtoMessage() {} func (x *DeleteActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[69] + mi := &file_ateapi_proto_msgTypes[67] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5305,7 +5265,7 @@ func (x *DeleteActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteActorRequest.ProtoReflect.Descriptor instead. func (*DeleteActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{69} + return file_ateapi_proto_rawDescGZIP(), []int{67} } func (x *DeleteActorRequest) GetActor() *ObjectRef { @@ -5349,7 +5309,7 @@ type GetActorEgressPolicyRequest struct { func (x *GetActorEgressPolicyRequest) Reset() { *x = GetActorEgressPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[70] + mi := &file_ateapi_proto_msgTypes[68] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5361,7 +5321,7 @@ func (x *GetActorEgressPolicyRequest) String() string { func (*GetActorEgressPolicyRequest) ProtoMessage() {} func (x *GetActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[70] + mi := &file_ateapi_proto_msgTypes[68] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5374,7 +5334,7 @@ func (x *GetActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetActorEgressPolicyRequest.ProtoReflect.Descriptor instead. func (*GetActorEgressPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{70} + return file_ateapi_proto_rawDescGZIP(), []int{68} } func (x *GetActorEgressPolicyRequest) GetActor() *ObjectRef { @@ -5412,7 +5372,7 @@ type CreateActorEgressPolicyRequest struct { func (x *CreateActorEgressPolicyRequest) Reset() { *x = CreateActorEgressPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[71] + mi := &file_ateapi_proto_msgTypes[69] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5424,7 +5384,7 @@ func (x *CreateActorEgressPolicyRequest) String() string { func (*CreateActorEgressPolicyRequest) ProtoMessage() {} func (x *CreateActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[71] + mi := &file_ateapi_proto_msgTypes[69] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5437,7 +5397,7 @@ func (x *CreateActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateActorEgressPolicyRequest.ProtoReflect.Descriptor instead. func (*CreateActorEgressPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{71} + return file_ateapi_proto_rawDescGZIP(), []int{69} } func (x *CreateActorEgressPolicyRequest) GetActor() *ObjectRef { @@ -5475,7 +5435,7 @@ type UpdateActorEgressPolicyRequest struct { func (x *UpdateActorEgressPolicyRequest) Reset() { *x = UpdateActorEgressPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[72] + mi := &file_ateapi_proto_msgTypes[70] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5487,7 +5447,7 @@ func (x *UpdateActorEgressPolicyRequest) String() string { func (*UpdateActorEgressPolicyRequest) ProtoMessage() {} func (x *UpdateActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[72] + mi := &file_ateapi_proto_msgTypes[70] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5500,7 +5460,7 @@ func (x *UpdateActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateActorEgressPolicyRequest.ProtoReflect.Descriptor instead. func (*UpdateActorEgressPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{72} + return file_ateapi_proto_rawDescGZIP(), []int{70} } func (x *UpdateActorEgressPolicyRequest) GetActor() *ObjectRef { @@ -5541,7 +5501,7 @@ type DeleteActorEgressPolicyRequest struct { func (x *DeleteActorEgressPolicyRequest) Reset() { *x = DeleteActorEgressPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[73] + mi := &file_ateapi_proto_msgTypes[71] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5553,7 +5513,7 @@ func (x *DeleteActorEgressPolicyRequest) String() string { func (*DeleteActorEgressPolicyRequest) ProtoMessage() {} func (x *DeleteActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[73] + mi := &file_ateapi_proto_msgTypes[71] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5566,7 +5526,7 @@ func (x *DeleteActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteActorEgressPolicyRequest.ProtoReflect.Descriptor instead. func (*DeleteActorEgressPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{73} + return file_ateapi_proto_rawDescGZIP(), []int{71} } func (x *DeleteActorEgressPolicyRequest) GetActor() *ObjectRef { @@ -5601,7 +5561,7 @@ type GetTagRequest struct { func (x *GetTagRequest) Reset() { *x = GetTagRequest{} - mi := &file_ateapi_proto_msgTypes[74] + mi := &file_ateapi_proto_msgTypes[72] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5613,7 +5573,7 @@ func (x *GetTagRequest) String() string { func (*GetTagRequest) ProtoMessage() {} func (x *GetTagRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[74] + mi := &file_ateapi_proto_msgTypes[72] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5626,7 +5586,7 @@ func (x *GetTagRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetTagRequest.ProtoReflect.Descriptor instead. func (*GetTagRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{74} + return file_ateapi_proto_rawDescGZIP(), []int{72} } func (x *GetTagRequest) GetTag() *ObjectRef { @@ -5665,7 +5625,7 @@ type MintActorJwtRequest struct { func (x *MintActorJwtRequest) Reset() { *x = MintActorJwtRequest{} - mi := &file_ateapi_proto_msgTypes[75] + mi := &file_ateapi_proto_msgTypes[73] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5677,7 +5637,7 @@ func (x *MintActorJwtRequest) String() string { func (*MintActorJwtRequest) ProtoMessage() {} func (x *MintActorJwtRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[75] + mi := &file_ateapi_proto_msgTypes[73] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5690,7 +5650,7 @@ func (x *MintActorJwtRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MintActorJwtRequest.ProtoReflect.Descriptor instead. func (*MintActorJwtRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{75} + return file_ateapi_proto_rawDescGZIP(), []int{73} } func (x *MintActorJwtRequest) GetActor() *ObjectRef { @@ -5745,7 +5705,7 @@ type MintActorJwtResponse struct { func (x *MintActorJwtResponse) Reset() { *x = MintActorJwtResponse{} - mi := &file_ateapi_proto_msgTypes[76] + mi := &file_ateapi_proto_msgTypes[74] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5757,7 +5717,7 @@ func (x *MintActorJwtResponse) String() string { func (*MintActorJwtResponse) ProtoMessage() {} func (x *MintActorJwtResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[76] + mi := &file_ateapi_proto_msgTypes[74] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5770,7 +5730,7 @@ func (x *MintActorJwtResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MintActorJwtResponse.ProtoReflect.Descriptor instead. func (*MintActorJwtResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{76} + return file_ateapi_proto_rawDescGZIP(), []int{74} } func (x *MintActorJwtResponse) GetActorJwt() string { @@ -5815,7 +5775,7 @@ type MintActorCertificateRequest struct { func (x *MintActorCertificateRequest) Reset() { *x = MintActorCertificateRequest{} - mi := &file_ateapi_proto_msgTypes[77] + mi := &file_ateapi_proto_msgTypes[75] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5827,7 +5787,7 @@ func (x *MintActorCertificateRequest) String() string { func (*MintActorCertificateRequest) ProtoMessage() {} func (x *MintActorCertificateRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[77] + mi := &file_ateapi_proto_msgTypes[75] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5840,7 +5800,7 @@ func (x *MintActorCertificateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MintActorCertificateRequest.ProtoReflect.Descriptor instead. func (*MintActorCertificateRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{77} + return file_ateapi_proto_rawDescGZIP(), []int{75} } func (x *MintActorCertificateRequest) GetActor() *ObjectRef { @@ -5877,7 +5837,7 @@ type MintActorCertificateResponse struct { func (x *MintActorCertificateResponse) Reset() { *x = MintActorCertificateResponse{} - mi := &file_ateapi_proto_msgTypes[78] + mi := &file_ateapi_proto_msgTypes[76] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5889,7 +5849,7 @@ func (x *MintActorCertificateResponse) String() string { func (*MintActorCertificateResponse) ProtoMessage() {} func (x *MintActorCertificateResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[78] + mi := &file_ateapi_proto_msgTypes[76] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5902,7 +5862,7 @@ func (x *MintActorCertificateResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MintActorCertificateResponse.ProtoReflect.Descriptor instead. func (*MintActorCertificateResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{78} + return file_ateapi_proto_rawDescGZIP(), []int{76} } func (x *MintActorCertificateResponse) GetActorCertificates() [][]byte { @@ -5929,7 +5889,7 @@ type ListTagsRequest struct { func (x *ListTagsRequest) Reset() { *x = ListTagsRequest{} - mi := &file_ateapi_proto_msgTypes[79] + mi := &file_ateapi_proto_msgTypes[77] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5941,7 +5901,7 @@ func (x *ListTagsRequest) String() string { func (*ListTagsRequest) ProtoMessage() {} func (x *ListTagsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[79] + mi := &file_ateapi_proto_msgTypes[77] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5954,7 +5914,7 @@ func (x *ListTagsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListTagsRequest.ProtoReflect.Descriptor instead. func (*ListTagsRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{79} + return file_ateapi_proto_rawDescGZIP(), []int{77} } func (x *ListTagsRequest) GetAtespace() string { @@ -5981,7 +5941,7 @@ type ListTagsResponse struct { func (x *ListTagsResponse) Reset() { *x = ListTagsResponse{} - mi := &file_ateapi_proto_msgTypes[80] + mi := &file_ateapi_proto_msgTypes[78] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5993,7 +5953,7 @@ func (x *ListTagsResponse) String() string { func (*ListTagsResponse) ProtoMessage() {} func (x *ListTagsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[80] + mi := &file_ateapi_proto_msgTypes[78] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6006,7 +5966,7 @@ func (x *ListTagsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListTagsResponse.ProtoReflect.Descriptor instead. func (*ListTagsResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{80} + return file_ateapi_proto_rawDescGZIP(), []int{78} } func (x *ListTagsResponse) GetTags() []*Tag { @@ -6051,7 +6011,7 @@ type CreateTagRequest struct { func (x *CreateTagRequest) Reset() { *x = CreateTagRequest{} - mi := &file_ateapi_proto_msgTypes[81] + mi := &file_ateapi_proto_msgTypes[79] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6063,7 +6023,7 @@ func (x *CreateTagRequest) String() string { func (*CreateTagRequest) ProtoMessage() {} func (x *CreateTagRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[81] + mi := &file_ateapi_proto_msgTypes[79] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6076,7 +6036,7 @@ func (x *CreateTagRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateTagRequest.ProtoReflect.Descriptor instead. func (*CreateTagRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{81} + return file_ateapi_proto_rawDescGZIP(), []int{79} } func (x *CreateTagRequest) GetTag() *Tag { @@ -6106,7 +6066,7 @@ type UpdateTagRequest struct { func (x *UpdateTagRequest) Reset() { *x = UpdateTagRequest{} - mi := &file_ateapi_proto_msgTypes[82] + mi := &file_ateapi_proto_msgTypes[80] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6118,7 +6078,7 @@ func (x *UpdateTagRequest) String() string { func (*UpdateTagRequest) ProtoMessage() {} func (x *UpdateTagRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[82] + mi := &file_ateapi_proto_msgTypes[80] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6131,7 +6091,7 @@ func (x *UpdateTagRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateTagRequest.ProtoReflect.Descriptor instead. func (*UpdateTagRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{82} + return file_ateapi_proto_rawDescGZIP(), []int{80} } func (x *UpdateTagRequest) GetTag() *Tag { @@ -6156,7 +6116,7 @@ type DeleteTagRequest struct { func (x *DeleteTagRequest) Reset() { *x = DeleteTagRequest{} - mi := &file_ateapi_proto_msgTypes[83] + mi := &file_ateapi_proto_msgTypes[81] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6168,7 +6128,7 @@ func (x *DeleteTagRequest) String() string { func (*DeleteTagRequest) ProtoMessage() {} func (x *DeleteTagRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[83] + mi := &file_ateapi_proto_msgTypes[81] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6181,7 +6141,7 @@ func (x *DeleteTagRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteTagRequest.ProtoReflect.Descriptor instead. func (*DeleteTagRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{83} + return file_ateapi_proto_rawDescGZIP(), []int{81} } func (x *DeleteTagRequest) GetTag() *ObjectRef { @@ -6220,7 +6180,7 @@ type DeleteOptions struct { func (x *DeleteOptions) Reset() { *x = DeleteOptions{} - mi := &file_ateapi_proto_msgTypes[84] + mi := &file_ateapi_proto_msgTypes[82] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6232,7 +6192,7 @@ func (x *DeleteOptions) String() string { func (*DeleteOptions) ProtoMessage() {} func (x *DeleteOptions) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[84] + mi := &file_ateapi_proto_msgTypes[82] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6245,7 +6205,7 @@ func (x *DeleteOptions) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteOptions.ProtoReflect.Descriptor instead. func (*DeleteOptions) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{84} + return file_ateapi_proto_rawDescGZIP(), []int{82} } func (x *DeleteOptions) GetVersion() int64 { @@ -6284,7 +6244,7 @@ type ListOptions struct { func (x *ListOptions) Reset() { *x = ListOptions{} - mi := &file_ateapi_proto_msgTypes[85] + mi := &file_ateapi_proto_msgTypes[83] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6296,7 +6256,7 @@ func (x *ListOptions) String() string { func (*ListOptions) ProtoMessage() {} func (x *ListOptions) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[85] + mi := &file_ateapi_proto_msgTypes[83] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6309,7 +6269,7 @@ func (x *ListOptions) ProtoReflect() protoreflect.Message { // Deprecated: Use ListOptions.ProtoReflect.Descriptor instead. func (*ListOptions) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{85} + return file_ateapi_proto_rawDescGZIP(), []int{83} } func (x *ListOptions) GetPageSize() int32 { @@ -6345,7 +6305,7 @@ type ListWorkerActorAssignmentsRequest struct { func (x *ListWorkerActorAssignmentsRequest) Reset() { *x = ListWorkerActorAssignmentsRequest{} - mi := &file_ateapi_proto_msgTypes[86] + mi := &file_ateapi_proto_msgTypes[84] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6357,7 +6317,7 @@ func (x *ListWorkerActorAssignmentsRequest) String() string { func (*ListWorkerActorAssignmentsRequest) ProtoMessage() {} func (x *ListWorkerActorAssignmentsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[86] + mi := &file_ateapi_proto_msgTypes[84] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6370,7 +6330,7 @@ func (x *ListWorkerActorAssignmentsRequest) ProtoReflect() protoreflect.Message // Deprecated: Use ListWorkerActorAssignmentsRequest.ProtoReflect.Descriptor instead. func (*ListWorkerActorAssignmentsRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{86} + return file_ateapi_proto_rawDescGZIP(), []int{84} } func (x *ListWorkerActorAssignmentsRequest) GetWorker() *ObjectRef { @@ -6400,7 +6360,7 @@ type ListWorkerActorAssignmentsResponse struct { func (x *ListWorkerActorAssignmentsResponse) Reset() { *x = ListWorkerActorAssignmentsResponse{} - mi := &file_ateapi_proto_msgTypes[87] + mi := &file_ateapi_proto_msgTypes[85] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6412,7 +6372,7 @@ func (x *ListWorkerActorAssignmentsResponse) String() string { func (*ListWorkerActorAssignmentsResponse) ProtoMessage() {} func (x *ListWorkerActorAssignmentsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[87] + mi := &file_ateapi_proto_msgTypes[85] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6425,7 +6385,7 @@ func (x *ListWorkerActorAssignmentsResponse) ProtoReflect() protoreflect.Message // Deprecated: Use ListWorkerActorAssignmentsResponse.ProtoReflect.Descriptor instead. func (*ListWorkerActorAssignmentsResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{87} + return file_ateapi_proto_rawDescGZIP(), []int{85} } func (x *ListWorkerActorAssignmentsResponse) GetActorAssignments() []*ActorAssignment { @@ -6454,7 +6414,7 @@ type ListWorkersRequest struct { func (x *ListWorkersRequest) Reset() { *x = ListWorkersRequest{} - mi := &file_ateapi_proto_msgTypes[88] + mi := &file_ateapi_proto_msgTypes[86] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6466,7 +6426,7 @@ func (x *ListWorkersRequest) String() string { func (*ListWorkersRequest) ProtoMessage() {} func (x *ListWorkersRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[88] + mi := &file_ateapi_proto_msgTypes[86] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6479,7 +6439,7 @@ func (x *ListWorkersRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListWorkersRequest.ProtoReflect.Descriptor instead. func (*ListWorkersRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{88} + return file_ateapi_proto_rawDescGZIP(), []int{86} } func (x *ListWorkersRequest) GetOptions() *ListOptions { @@ -6501,7 +6461,7 @@ type ListWorkersResponse struct { func (x *ListWorkersResponse) Reset() { *x = ListWorkersResponse{} - mi := &file_ateapi_proto_msgTypes[89] + mi := &file_ateapi_proto_msgTypes[87] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6513,7 +6473,7 @@ func (x *ListWorkersResponse) String() string { func (*ListWorkersResponse) ProtoMessage() {} func (x *ListWorkersResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[89] + mi := &file_ateapi_proto_msgTypes[87] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6526,7 +6486,7 @@ func (x *ListWorkersResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListWorkersResponse.ProtoReflect.Descriptor instead. func (*ListWorkersResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{89} + return file_ateapi_proto_rawDescGZIP(), []int{87} } func (x *ListWorkersResponse) GetWorkers() []*Worker { @@ -6556,7 +6516,7 @@ type GetWorkerRequest struct { func (x *GetWorkerRequest) Reset() { *x = GetWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[90] + mi := &file_ateapi_proto_msgTypes[88] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6568,7 +6528,7 @@ func (x *GetWorkerRequest) String() string { func (*GetWorkerRequest) ProtoMessage() {} func (x *GetWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[90] + mi := &file_ateapi_proto_msgTypes[88] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6581,7 +6541,7 @@ func (x *GetWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetWorkerRequest.ProtoReflect.Descriptor instead. func (*GetWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{90} + return file_ateapi_proto_rawDescGZIP(), []int{88} } func (x *GetWorkerRequest) GetWorker() *ObjectRef { @@ -6603,7 +6563,7 @@ type CreateWorkerRequest struct { func (x *CreateWorkerRequest) Reset() { *x = CreateWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[91] + mi := &file_ateapi_proto_msgTypes[89] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6615,7 +6575,7 @@ func (x *CreateWorkerRequest) String() string { func (*CreateWorkerRequest) ProtoMessage() {} func (x *CreateWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[91] + mi := &file_ateapi_proto_msgTypes[89] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6628,7 +6588,7 @@ func (x *CreateWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateWorkerRequest.ProtoReflect.Descriptor instead. func (*CreateWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{91} + return file_ateapi_proto_rawDescGZIP(), []int{89} } func (x *CreateWorkerRequest) GetWorker() *Worker { @@ -6663,7 +6623,7 @@ type UpdateWorkerRequest struct { func (x *UpdateWorkerRequest) Reset() { *x = UpdateWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[92] + mi := &file_ateapi_proto_msgTypes[90] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6675,7 +6635,7 @@ func (x *UpdateWorkerRequest) String() string { func (*UpdateWorkerRequest) ProtoMessage() {} func (x *UpdateWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[92] + mi := &file_ateapi_proto_msgTypes[90] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6688,7 +6648,7 @@ func (x *UpdateWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateWorkerRequest.ProtoReflect.Descriptor instead. func (*UpdateWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{92} + return file_ateapi_proto_rawDescGZIP(), []int{90} } func (x *UpdateWorkerRequest) GetWorker() *Worker { @@ -6715,7 +6675,7 @@ type DeleteWorkerRequest struct { func (x *DeleteWorkerRequest) Reset() { *x = DeleteWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[93] + mi := &file_ateapi_proto_msgTypes[91] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6727,7 +6687,7 @@ func (x *DeleteWorkerRequest) String() string { func (*DeleteWorkerRequest) ProtoMessage() {} func (x *DeleteWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[93] + mi := &file_ateapi_proto_msgTypes[91] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6740,7 +6700,7 @@ func (x *DeleteWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteWorkerRequest.ProtoReflect.Descriptor instead. func (*DeleteWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{93} + return file_ateapi_proto_rawDescGZIP(), []int{91} } func (x *DeleteWorkerRequest) GetWorker() *ObjectRef { @@ -6770,7 +6730,7 @@ type DrainWorkerRequest struct { func (x *DrainWorkerRequest) Reset() { *x = DrainWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[94] + mi := &file_ateapi_proto_msgTypes[92] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6782,7 +6742,7 @@ func (x *DrainWorkerRequest) String() string { func (*DrainWorkerRequest) ProtoMessage() {} func (x *DrainWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[94] + mi := &file_ateapi_proto_msgTypes[92] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6795,7 +6755,7 @@ func (x *DrainWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DrainWorkerRequest.ProtoReflect.Descriptor instead. func (*DrainWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{94} + return file_ateapi_proto_rawDescGZIP(), []int{92} } func (x *DrainWorkerRequest) GetWorker() *ObjectRef { @@ -6816,7 +6776,7 @@ type DrainWorkerResponse struct { func (x *DrainWorkerResponse) Reset() { *x = DrainWorkerResponse{} - mi := &file_ateapi_proto_msgTypes[95] + mi := &file_ateapi_proto_msgTypes[93] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6828,7 +6788,7 @@ func (x *DrainWorkerResponse) String() string { func (*DrainWorkerResponse) ProtoMessage() {} func (x *DrainWorkerResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[95] + mi := &file_ateapi_proto_msgTypes[93] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6841,7 +6801,7 @@ func (x *DrainWorkerResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use DrainWorkerResponse.ProtoReflect.Descriptor instead. func (*DrainWorkerResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{95} + return file_ateapi_proto_rawDescGZIP(), []int{93} } func (x *DrainWorkerResponse) GetWorker() *Worker { @@ -6865,7 +6825,7 @@ type MarkWorkerAvailableRequest struct { func (x *MarkWorkerAvailableRequest) Reset() { *x = MarkWorkerAvailableRequest{} - mi := &file_ateapi_proto_msgTypes[96] + mi := &file_ateapi_proto_msgTypes[94] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6877,7 +6837,7 @@ func (x *MarkWorkerAvailableRequest) String() string { func (*MarkWorkerAvailableRequest) ProtoMessage() {} func (x *MarkWorkerAvailableRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[96] + mi := &file_ateapi_proto_msgTypes[94] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6890,7 +6850,7 @@ func (x *MarkWorkerAvailableRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MarkWorkerAvailableRequest.ProtoReflect.Descriptor instead. func (*MarkWorkerAvailableRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{96} + return file_ateapi_proto_rawDescGZIP(), []int{94} } func (x *MarkWorkerAvailableRequest) GetWorker() *ObjectRef { @@ -6911,7 +6871,7 @@ type MarkWorkerAvailableResponse struct { func (x *MarkWorkerAvailableResponse) Reset() { *x = MarkWorkerAvailableResponse{} - mi := &file_ateapi_proto_msgTypes[97] + mi := &file_ateapi_proto_msgTypes[95] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6923,7 +6883,7 @@ func (x *MarkWorkerAvailableResponse) String() string { func (*MarkWorkerAvailableResponse) ProtoMessage() {} func (x *MarkWorkerAvailableResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[97] + mi := &file_ateapi_proto_msgTypes[95] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6936,7 +6896,7 @@ func (x *MarkWorkerAvailableResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MarkWorkerAvailableResponse.ProtoReflect.Descriptor instead. func (*MarkWorkerAvailableResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{97} + return file_ateapi_proto_rawDescGZIP(), []int{95} } func (x *MarkWorkerAvailableResponse) GetWorker() *Worker { @@ -6960,7 +6920,7 @@ type MarkWorkerUnavailableRequest struct { func (x *MarkWorkerUnavailableRequest) Reset() { *x = MarkWorkerUnavailableRequest{} - mi := &file_ateapi_proto_msgTypes[98] + mi := &file_ateapi_proto_msgTypes[96] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6972,7 +6932,7 @@ func (x *MarkWorkerUnavailableRequest) String() string { func (*MarkWorkerUnavailableRequest) ProtoMessage() {} func (x *MarkWorkerUnavailableRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[98] + mi := &file_ateapi_proto_msgTypes[96] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6985,7 +6945,7 @@ func (x *MarkWorkerUnavailableRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MarkWorkerUnavailableRequest.ProtoReflect.Descriptor instead. func (*MarkWorkerUnavailableRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{98} + return file_ateapi_proto_rawDescGZIP(), []int{96} } func (x *MarkWorkerUnavailableRequest) GetWorker() *ObjectRef { @@ -7006,7 +6966,7 @@ type MarkWorkerUnavailableResponse struct { func (x *MarkWorkerUnavailableResponse) Reset() { *x = MarkWorkerUnavailableResponse{} - mi := &file_ateapi_proto_msgTypes[99] + mi := &file_ateapi_proto_msgTypes[97] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7018,7 +6978,7 @@ func (x *MarkWorkerUnavailableResponse) String() string { func (*MarkWorkerUnavailableResponse) ProtoMessage() {} func (x *MarkWorkerUnavailableResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[99] + mi := &file_ateapi_proto_msgTypes[97] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7031,7 +6991,7 @@ func (x *MarkWorkerUnavailableResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MarkWorkerUnavailableResponse.ProtoReflect.Descriptor instead. func (*MarkWorkerUnavailableResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{99} + return file_ateapi_proto_rawDescGZIP(), []int{97} } func (x *MarkWorkerUnavailableResponse) GetWorker() *Worker { @@ -7060,7 +7020,7 @@ type ListActorsRequest struct { func (x *ListActorsRequest) Reset() { *x = ListActorsRequest{} - mi := &file_ateapi_proto_msgTypes[100] + mi := &file_ateapi_proto_msgTypes[98] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7072,7 +7032,7 @@ func (x *ListActorsRequest) String() string { func (*ListActorsRequest) ProtoMessage() {} func (x *ListActorsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[100] + mi := &file_ateapi_proto_msgTypes[98] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7085,7 +7045,7 @@ func (x *ListActorsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListActorsRequest.ProtoReflect.Descriptor instead. func (*ListActorsRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{100} + return file_ateapi_proto_rawDescGZIP(), []int{98} } func (x *ListActorsRequest) GetAtespace() string { @@ -7114,7 +7074,7 @@ type ListActorsResponse struct { func (x *ListActorsResponse) Reset() { *x = ListActorsResponse{} - mi := &file_ateapi_proto_msgTypes[101] + mi := &file_ateapi_proto_msgTypes[99] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7126,7 +7086,7 @@ func (x *ListActorsResponse) String() string { func (*ListActorsResponse) ProtoMessage() {} func (x *ListActorsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[101] + mi := &file_ateapi_proto_msgTypes[99] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7139,7 +7099,7 @@ func (x *ListActorsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListActorsResponse.ProtoReflect.Descriptor instead. func (*ListActorsResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{101} + return file_ateapi_proto_rawDescGZIP(), []int{99} } func (x *ListActorsResponse) GetActors() []*Actor { @@ -7256,7 +7216,7 @@ type Worker struct { func (x *Worker) Reset() { *x = Worker{} - mi := &file_ateapi_proto_msgTypes[102] + mi := &file_ateapi_proto_msgTypes[100] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7268,7 +7228,7 @@ func (x *Worker) String() string { func (*Worker) ProtoMessage() {} func (x *Worker) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[102] + mi := &file_ateapi_proto_msgTypes[100] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7281,7 +7241,7 @@ func (x *Worker) ProtoReflect() protoreflect.Message { // Deprecated: Use Worker.ProtoReflect.Descriptor instead. func (*Worker) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{102} + return file_ateapi_proto_rawDescGZIP(), []int{100} } func (x *Worker) GetMetadata() *ResourceMetadata { @@ -7417,7 +7377,7 @@ type WorkerStatus struct { func (x *WorkerStatus) Reset() { *x = WorkerStatus{} - mi := &file_ateapi_proto_msgTypes[103] + mi := &file_ateapi_proto_msgTypes[101] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7429,7 +7389,7 @@ func (x *WorkerStatus) String() string { func (*WorkerStatus) ProtoMessage() {} func (x *WorkerStatus) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[103] + mi := &file_ateapi_proto_msgTypes[101] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7442,7 +7402,7 @@ func (x *WorkerStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkerStatus.ProtoReflect.Descriptor instead. func (*WorkerStatus) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{103} + return file_ateapi_proto_rawDescGZIP(), []int{101} } func (x *WorkerStatus) GetState() WorkerState { @@ -7510,7 +7470,7 @@ type SandboxRuntime struct { func (x *SandboxRuntime) Reset() { *x = SandboxRuntime{} - mi := &file_ateapi_proto_msgTypes[104] + mi := &file_ateapi_proto_msgTypes[102] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7522,7 +7482,7 @@ func (x *SandboxRuntime) String() string { func (*SandboxRuntime) ProtoMessage() {} func (x *SandboxRuntime) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[104] + mi := &file_ateapi_proto_msgTypes[102] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7535,7 +7495,7 @@ func (x *SandboxRuntime) ProtoReflect() protoreflect.Message { // Deprecated: Use SandboxRuntime.ProtoReflect.Descriptor instead. func (*SandboxRuntime) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{104} + return file_ateapi_proto_rawDescGZIP(), []int{102} } func (x *SandboxRuntime) GetSandboxClass() string { @@ -7563,7 +7523,7 @@ type VersionedSandboxCompat struct { // +k8s:maxLength=64 SchemaVersion string `protobuf:"bytes,1,opt,name=schema_version,json=schemaVersion,proto3" json:"schema_version,omitempty"` // Key-value compatibility attributes defined by this schema version, covering - // both runtime assets (e.g. "sandbox_runtime_hash") and effective guest + // both runtime assets (e.g. "sandbox_bundle_hash") and effective guest // hardware features after masking (e.g. "architecture", "cpu_features"). // Matched against a snapshot under the same schema_version: every attribute // on the snapshot must be present on the worker with an equal value (except @@ -7586,7 +7546,7 @@ type VersionedSandboxCompat struct { func (x *VersionedSandboxCompat) Reset() { *x = VersionedSandboxCompat{} - mi := &file_ateapi_proto_msgTypes[105] + mi := &file_ateapi_proto_msgTypes[103] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7598,7 +7558,7 @@ func (x *VersionedSandboxCompat) String() string { func (*VersionedSandboxCompat) ProtoMessage() {} func (x *VersionedSandboxCompat) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[105] + mi := &file_ateapi_proto_msgTypes[103] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7611,7 +7571,7 @@ func (x *VersionedSandboxCompat) ProtoReflect() protoreflect.Message { // Deprecated: Use VersionedSandboxCompat.ProtoReflect.Descriptor instead. func (*VersionedSandboxCompat) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{105} + return file_ateapi_proto_rawDescGZIP(), []int{103} } func (x *VersionedSandboxCompat) GetSchemaVersion() string { @@ -7631,7 +7591,7 @@ func (x *VersionedSandboxCompat) GetAttributes() []*AttributeEntry { // AttributeEntry is a single key-value compatibility attribute. type AttributeEntry struct { state protoimpl.MessageState `protogen:"open.v1"` - // Attribute name (e.g. "architecture", "cpu_features", "sandbox_runtime_hash"). + // Attribute name (e.g. "architecture", "cpu_features", "sandbox_bundle_hash"). // // +k8s:required // +k8s:maxLength=128 @@ -7647,7 +7607,7 @@ type AttributeEntry struct { func (x *AttributeEntry) Reset() { *x = AttributeEntry{} - mi := &file_ateapi_proto_msgTypes[106] + mi := &file_ateapi_proto_msgTypes[104] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7659,7 +7619,7 @@ func (x *AttributeEntry) String() string { func (*AttributeEntry) ProtoMessage() {} func (x *AttributeEntry) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[106] + mi := &file_ateapi_proto_msgTypes[104] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7672,7 +7632,7 @@ func (x *AttributeEntry) ProtoReflect() protoreflect.Message { // Deprecated: Use AttributeEntry.ProtoReflect.Descriptor instead. func (*AttributeEntry) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{106} + return file_ateapi_proto_rawDescGZIP(), []int{104} } func (x *AttributeEntry) GetKey() string { @@ -7711,7 +7671,7 @@ type WorkerResources struct { func (x *WorkerResources) Reset() { *x = WorkerResources{} - mi := &file_ateapi_proto_msgTypes[107] + mi := &file_ateapi_proto_msgTypes[105] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7723,7 +7683,7 @@ func (x *WorkerResources) String() string { func (*WorkerResources) ProtoMessage() {} func (x *WorkerResources) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[107] + mi := &file_ateapi_proto_msgTypes[105] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7736,7 +7696,7 @@ func (x *WorkerResources) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkerResources.ProtoReflect.Descriptor instead. func (*WorkerResources) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{107} + return file_ateapi_proto_rawDescGZIP(), []int{105} } func (x *WorkerResources) GetResources() *Resources { @@ -7789,7 +7749,7 @@ type ActorAssignment struct { func (x *ActorAssignment) Reset() { *x = ActorAssignment{} - mi := &file_ateapi_proto_msgTypes[108] + mi := &file_ateapi_proto_msgTypes[106] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7801,7 +7761,7 @@ func (x *ActorAssignment) String() string { func (*ActorAssignment) ProtoMessage() {} func (x *ActorAssignment) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[108] + mi := &file_ateapi_proto_msgTypes[106] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7814,7 +7774,7 @@ func (x *ActorAssignment) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorAssignment.ProtoReflect.Descriptor instead. func (*ActorAssignment) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{108} + return file_ateapi_proto_rawDescGZIP(), []int{106} } func (x *ActorAssignment) GetMetadata() *ResourceMetadata { @@ -7885,7 +7845,7 @@ type RegisterWorkerRequest struct { func (x *RegisterWorkerRequest) Reset() { *x = RegisterWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[109] + mi := &file_ateapi_proto_msgTypes[107] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7897,7 +7857,7 @@ func (x *RegisterWorkerRequest) String() string { func (*RegisterWorkerRequest) ProtoMessage() {} func (x *RegisterWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[109] + mi := &file_ateapi_proto_msgTypes[107] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7910,7 +7870,7 @@ func (x *RegisterWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RegisterWorkerRequest.ProtoReflect.Descriptor instead. func (*RegisterWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{109} + return file_ateapi_proto_rawDescGZIP(), []int{107} } func (x *RegisterWorkerRequest) GetWorker() *ObjectRef { @@ -7951,7 +7911,7 @@ type RegisterWorkerResponse struct { func (x *RegisterWorkerResponse) Reset() { *x = RegisterWorkerResponse{} - mi := &file_ateapi_proto_msgTypes[110] + mi := &file_ateapi_proto_msgTypes[108] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7963,7 +7923,7 @@ func (x *RegisterWorkerResponse) String() string { func (*RegisterWorkerResponse) ProtoMessage() {} func (x *RegisterWorkerResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[110] + mi := &file_ateapi_proto_msgTypes[108] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7976,7 +7936,7 @@ func (x *RegisterWorkerResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RegisterWorkerResponse.ProtoReflect.Descriptor instead. func (*RegisterWorkerResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{110} + return file_ateapi_proto_rawDescGZIP(), []int{108} } func (x *RegisterWorkerResponse) GetWorker() *Worker { @@ -8023,7 +7983,7 @@ type MintWorkerActorCertificateRequest struct { func (x *MintWorkerActorCertificateRequest) Reset() { *x = MintWorkerActorCertificateRequest{} - mi := &file_ateapi_proto_msgTypes[111] + mi := &file_ateapi_proto_msgTypes[109] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8035,7 +7995,7 @@ func (x *MintWorkerActorCertificateRequest) String() string { func (*MintWorkerActorCertificateRequest) ProtoMessage() {} func (x *MintWorkerActorCertificateRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[111] + mi := &file_ateapi_proto_msgTypes[109] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8048,7 +8008,7 @@ func (x *MintWorkerActorCertificateRequest) ProtoReflect() protoreflect.Message // Deprecated: Use MintWorkerActorCertificateRequest.ProtoReflect.Descriptor instead. func (*MintWorkerActorCertificateRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{111} + return file_ateapi_proto_rawDescGZIP(), []int{109} } func (x *MintWorkerActorCertificateRequest) GetActor() *ObjectRef { @@ -8092,7 +8052,7 @@ type MintWorkerActorCertificateResponse struct { func (x *MintWorkerActorCertificateResponse) Reset() { *x = MintWorkerActorCertificateResponse{} - mi := &file_ateapi_proto_msgTypes[112] + mi := &file_ateapi_proto_msgTypes[110] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8104,7 +8064,7 @@ func (x *MintWorkerActorCertificateResponse) String() string { func (*MintWorkerActorCertificateResponse) ProtoMessage() {} func (x *MintWorkerActorCertificateResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[112] + mi := &file_ateapi_proto_msgTypes[110] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8117,7 +8077,7 @@ func (x *MintWorkerActorCertificateResponse) ProtoReflect() protoreflect.Message // Deprecated: Use MintWorkerActorCertificateResponse.ProtoReflect.Descriptor instead. func (*MintWorkerActorCertificateResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{112} + return file_ateapi_proto_rawDescGZIP(), []int{110} } func (x *MintWorkerActorCertificateResponse) GetActorCertificates() [][]byte { @@ -8158,7 +8118,7 @@ type MintWorkerActorJwtRequest struct { func (x *MintWorkerActorJwtRequest) Reset() { *x = MintWorkerActorJwtRequest{} - mi := &file_ateapi_proto_msgTypes[113] + mi := &file_ateapi_proto_msgTypes[111] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8170,7 +8130,7 @@ func (x *MintWorkerActorJwtRequest) String() string { func (*MintWorkerActorJwtRequest) ProtoMessage() {} func (x *MintWorkerActorJwtRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[113] + mi := &file_ateapi_proto_msgTypes[111] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8183,7 +8143,7 @@ func (x *MintWorkerActorJwtRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MintWorkerActorJwtRequest.ProtoReflect.Descriptor instead. func (*MintWorkerActorJwtRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{113} + return file_ateapi_proto_rawDescGZIP(), []int{111} } func (x *MintWorkerActorJwtRequest) GetActor() *ObjectRef { @@ -8229,7 +8189,7 @@ type MintWorkerActorJwtResponse struct { func (x *MintWorkerActorJwtResponse) Reset() { *x = MintWorkerActorJwtResponse{} - mi := &file_ateapi_proto_msgTypes[114] + mi := &file_ateapi_proto_msgTypes[112] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8241,7 +8201,7 @@ func (x *MintWorkerActorJwtResponse) String() string { func (*MintWorkerActorJwtResponse) ProtoMessage() {} func (x *MintWorkerActorJwtResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[114] + mi := &file_ateapi_proto_msgTypes[112] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8254,7 +8214,7 @@ func (x *MintWorkerActorJwtResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MintWorkerActorJwtResponse.ProtoReflect.Descriptor instead. func (*MintWorkerActorJwtResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{114} + return file_ateapi_proto_rawDescGZIP(), []int{112} } func (x *MintWorkerActorJwtResponse) GetActorJwt() string { @@ -8271,6 +8231,9 @@ func (x *MintWorkerActorJwtResponse) GetExpireTime() *timestamppb.Timestamp { return nil } +// TODO: add a durability mode, carried through atelet's +// RequestActorSuspendRequest, so the Worker can choose how the snapshot is +// made durable. Until then the suspend uses SuspendActor's default mode. type RequestActorSuspendRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // The Worker making the request. atespace is always empty; Workers are @@ -8299,7 +8262,7 @@ type RequestActorSuspendRequest struct { func (x *RequestActorSuspendRequest) Reset() { *x = RequestActorSuspendRequest{} - mi := &file_ateapi_proto_msgTypes[115] + mi := &file_ateapi_proto_msgTypes[113] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8311,7 +8274,7 @@ func (x *RequestActorSuspendRequest) String() string { func (*RequestActorSuspendRequest) ProtoMessage() {} func (x *RequestActorSuspendRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[115] + mi := &file_ateapi_proto_msgTypes[113] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8324,7 +8287,7 @@ func (x *RequestActorSuspendRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RequestActorSuspendRequest.ProtoReflect.Descriptor instead. func (*RequestActorSuspendRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{115} + return file_ateapi_proto_rawDescGZIP(), []int{113} } func (x *RequestActorSuspendRequest) GetWorker() *ObjectRef { @@ -8358,7 +8321,7 @@ type RequestActorSuspendResponse struct { func (x *RequestActorSuspendResponse) Reset() { *x = RequestActorSuspendResponse{} - mi := &file_ateapi_proto_msgTypes[116] + mi := &file_ateapi_proto_msgTypes[114] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8370,7 +8333,7 @@ func (x *RequestActorSuspendResponse) String() string { func (*RequestActorSuspendResponse) ProtoMessage() {} func (x *RequestActorSuspendResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[116] + mi := &file_ateapi_proto_msgTypes[114] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8383,7 +8346,7 @@ func (x *RequestActorSuspendResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RequestActorSuspendResponse.ProtoReflect.Descriptor instead. func (*RequestActorSuspendResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{116} + return file_ateapi_proto_rawDescGZIP(), []int{114} } func (x *RequestActorSuspendResponse) GetActor() *Actor { @@ -8414,7 +8377,7 @@ type ReportActorSnapshotStateRequest struct { func (x *ReportActorSnapshotStateRequest) Reset() { *x = ReportActorSnapshotStateRequest{} - mi := &file_ateapi_proto_msgTypes[117] + mi := &file_ateapi_proto_msgTypes[115] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8426,7 +8389,7 @@ func (x *ReportActorSnapshotStateRequest) String() string { func (*ReportActorSnapshotStateRequest) ProtoMessage() {} func (x *ReportActorSnapshotStateRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[117] + mi := &file_ateapi_proto_msgTypes[115] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8439,7 +8402,7 @@ func (x *ReportActorSnapshotStateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ReportActorSnapshotStateRequest.ProtoReflect.Descriptor instead. func (*ReportActorSnapshotStateRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{117} + return file_ateapi_proto_rawDescGZIP(), []int{115} } func (x *ReportActorSnapshotStateRequest) GetActor() *ObjectRef { @@ -8471,7 +8434,7 @@ type ReportActorSnapshotStateResponse struct { func (x *ReportActorSnapshotStateResponse) Reset() { *x = ReportActorSnapshotStateResponse{} - mi := &file_ateapi_proto_msgTypes[118] + mi := &file_ateapi_proto_msgTypes[116] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8483,7 +8446,7 @@ func (x *ReportActorSnapshotStateResponse) String() string { func (*ReportActorSnapshotStateResponse) ProtoMessage() {} func (x *ReportActorSnapshotStateResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[118] + mi := &file_ateapi_proto_msgTypes[116] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8496,7 +8459,7 @@ func (x *ReportActorSnapshotStateResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReportActorSnapshotStateResponse.ProtoReflect.Descriptor instead. func (*ReportActorSnapshotStateResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{118} + return file_ateapi_proto_rawDescGZIP(), []int{116} } // ActorSnapshotState is what atelet holds for one Actor on its node. It @@ -8530,7 +8493,7 @@ type ActorSnapshotState struct { func (x *ActorSnapshotState) Reset() { *x = ActorSnapshotState{} - mi := &file_ateapi_proto_msgTypes[119] + mi := &file_ateapi_proto_msgTypes[117] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8542,7 +8505,7 @@ func (x *ActorSnapshotState) String() string { func (*ActorSnapshotState) ProtoMessage() {} func (x *ActorSnapshotState) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[119] + mi := &file_ateapi_proto_msgTypes[117] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8555,7 +8518,7 @@ func (x *ActorSnapshotState) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorSnapshotState.ProtoReflect.Descriptor instead. func (*ActorSnapshotState) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{119} + return file_ateapi_proto_rawDescGZIP(), []int{117} } func (x *ActorSnapshotState) GetLatestLocal() *SnapshotReport { @@ -8627,7 +8590,7 @@ type SnapshotReport struct { func (x *SnapshotReport) Reset() { *x = SnapshotReport{} - mi := &file_ateapi_proto_msgTypes[120] + mi := &file_ateapi_proto_msgTypes[118] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8639,7 +8602,7 @@ func (x *SnapshotReport) String() string { func (*SnapshotReport) ProtoMessage() {} func (x *SnapshotReport) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[120] + mi := &file_ateapi_proto_msgTypes[118] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8652,7 +8615,7 @@ func (x *SnapshotReport) ProtoReflect() protoreflect.Message { // Deprecated: Use SnapshotReport.ProtoReflect.Descriptor instead. func (*SnapshotReport) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{120} + return file_ateapi_proto_rawDescGZIP(), []int{118} } func (x *SnapshotReport) GetUuid() string { @@ -8722,7 +8685,7 @@ type AccessPolicy struct { func (x *AccessPolicy) Reset() { *x = AccessPolicy{} - mi := &file_ateapi_proto_msgTypes[121] + mi := &file_ateapi_proto_msgTypes[119] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8734,7 +8697,7 @@ func (x *AccessPolicy) String() string { func (*AccessPolicy) ProtoMessage() {} func (x *AccessPolicy) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[121] + mi := &file_ateapi_proto_msgTypes[119] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8747,7 +8710,7 @@ func (x *AccessPolicy) ProtoReflect() protoreflect.Message { // Deprecated: Use AccessPolicy.ProtoReflect.Descriptor instead. func (*AccessPolicy) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{121} + return file_ateapi_proto_rawDescGZIP(), []int{119} } func (x *AccessPolicy) GetMetadata() *ResourceMetadata { @@ -8789,7 +8752,7 @@ type Binding struct { func (x *Binding) Reset() { *x = Binding{} - mi := &file_ateapi_proto_msgTypes[122] + mi := &file_ateapi_proto_msgTypes[120] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8801,7 +8764,7 @@ func (x *Binding) String() string { func (*Binding) ProtoMessage() {} func (x *Binding) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[122] + mi := &file_ateapi_proto_msgTypes[120] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8814,7 +8777,7 @@ func (x *Binding) ProtoReflect() protoreflect.Message { // Deprecated: Use Binding.ProtoReflect.Descriptor instead. func (*Binding) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{122} + return file_ateapi_proto_rawDescGZIP(), []int{120} } func (x *Binding) GetRole() string { @@ -8846,7 +8809,7 @@ type GetGlobalAccessPolicyRequest struct { func (x *GetGlobalAccessPolicyRequest) Reset() { *x = GetGlobalAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[123] + mi := &file_ateapi_proto_msgTypes[121] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8858,7 +8821,7 @@ func (x *GetGlobalAccessPolicyRequest) String() string { func (*GetGlobalAccessPolicyRequest) ProtoMessage() {} func (x *GetGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[123] + mi := &file_ateapi_proto_msgTypes[121] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8871,7 +8834,7 @@ func (x *GetGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetGlobalAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*GetGlobalAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{123} + return file_ateapi_proto_rawDescGZIP(), []int{121} } func (x *GetGlobalAccessPolicyRequest) GetName() string { @@ -8895,7 +8858,7 @@ type CreateGlobalAccessPolicyRequest struct { func (x *CreateGlobalAccessPolicyRequest) Reset() { *x = CreateGlobalAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[124] + mi := &file_ateapi_proto_msgTypes[122] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8907,7 +8870,7 @@ func (x *CreateGlobalAccessPolicyRequest) String() string { func (*CreateGlobalAccessPolicyRequest) ProtoMessage() {} func (x *CreateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[124] + mi := &file_ateapi_proto_msgTypes[122] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8920,7 +8883,7 @@ func (x *CreateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateGlobalAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*CreateGlobalAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{124} + return file_ateapi_proto_rawDescGZIP(), []int{122} } func (x *CreateGlobalAccessPolicyRequest) GetAccessPolicy() *AccessPolicy { @@ -8944,7 +8907,7 @@ type UpdateGlobalAccessPolicyRequest struct { func (x *UpdateGlobalAccessPolicyRequest) Reset() { *x = UpdateGlobalAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[125] + mi := &file_ateapi_proto_msgTypes[123] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8956,7 +8919,7 @@ func (x *UpdateGlobalAccessPolicyRequest) String() string { func (*UpdateGlobalAccessPolicyRequest) ProtoMessage() {} func (x *UpdateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[125] + mi := &file_ateapi_proto_msgTypes[123] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8969,7 +8932,7 @@ func (x *UpdateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateGlobalAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*UpdateGlobalAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{125} + return file_ateapi_proto_rawDescGZIP(), []int{123} } func (x *UpdateGlobalAccessPolicyRequest) GetAccessPolicy() *AccessPolicy { @@ -8999,7 +8962,7 @@ type GetAtespaceAccessPolicyRequest struct { func (x *GetAtespaceAccessPolicyRequest) Reset() { *x = GetAtespaceAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[126] + mi := &file_ateapi_proto_msgTypes[124] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9011,7 +8974,7 @@ func (x *GetAtespaceAccessPolicyRequest) String() string { func (*GetAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *GetAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[126] + mi := &file_ateapi_proto_msgTypes[124] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9024,7 +8987,7 @@ func (x *GetAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*GetAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{126} + return file_ateapi_proto_rawDescGZIP(), []int{124} } func (x *GetAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -9060,7 +9023,7 @@ type CreateAtespaceAccessPolicyRequest struct { func (x *CreateAtespaceAccessPolicyRequest) Reset() { *x = CreateAtespaceAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[127] + mi := &file_ateapi_proto_msgTypes[125] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9072,7 +9035,7 @@ func (x *CreateAtespaceAccessPolicyRequest) String() string { func (*CreateAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *CreateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[127] + mi := &file_ateapi_proto_msgTypes[125] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9085,7 +9048,7 @@ func (x *CreateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message // Deprecated: Use CreateAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*CreateAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{127} + return file_ateapi_proto_rawDescGZIP(), []int{125} } func (x *CreateAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -9121,7 +9084,7 @@ type UpdateAtespaceAccessPolicyRequest struct { func (x *UpdateAtespaceAccessPolicyRequest) Reset() { *x = UpdateAtespaceAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[128] + mi := &file_ateapi_proto_msgTypes[126] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9133,7 +9096,7 @@ func (x *UpdateAtespaceAccessPolicyRequest) String() string { func (*UpdateAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *UpdateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[128] + mi := &file_ateapi_proto_msgTypes[126] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9146,7 +9109,7 @@ func (x *UpdateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message // Deprecated: Use UpdateAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*UpdateAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{128} + return file_ateapi_proto_rawDescGZIP(), []int{126} } func (x *UpdateAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -9187,7 +9150,7 @@ type DeleteAtespaceAccessPolicyRequest struct { func (x *DeleteAtespaceAccessPolicyRequest) Reset() { *x = DeleteAtespaceAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[129] + mi := &file_ateapi_proto_msgTypes[127] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9199,7 +9162,7 @@ func (x *DeleteAtespaceAccessPolicyRequest) String() string { func (*DeleteAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *DeleteAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[129] + mi := &file_ateapi_proto_msgTypes[127] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9212,7 +9175,7 @@ func (x *DeleteAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message // Deprecated: Use DeleteAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*DeleteAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{129} + return file_ateapi_proto_rawDescGZIP(), []int{127} } func (x *DeleteAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -9258,7 +9221,7 @@ type ResourceAnnotation struct { func (x *ResourceAnnotation) Reset() { *x = ResourceAnnotation{} - mi := &file_ateapi_proto_msgTypes[130] + mi := &file_ateapi_proto_msgTypes[128] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9270,7 +9233,7 @@ func (x *ResourceAnnotation) String() string { func (*ResourceAnnotation) ProtoMessage() {} func (x *ResourceAnnotation) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[130] + mi := &file_ateapi_proto_msgTypes[128] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9283,7 +9246,7 @@ func (x *ResourceAnnotation) ProtoReflect() protoreflect.Message { // Deprecated: Use ResourceAnnotation.ProtoReflect.Descriptor instead. func (*ResourceAnnotation) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{130} + return file_ateapi_proto_rawDescGZIP(), []int{128} } func (x *ResourceAnnotation) GetParents() []string { @@ -9326,7 +9289,7 @@ type MethodAnnotation struct { func (x *MethodAnnotation) Reset() { *x = MethodAnnotation{} - mi := &file_ateapi_proto_msgTypes[131] + mi := &file_ateapi_proto_msgTypes[129] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9338,7 +9301,7 @@ func (x *MethodAnnotation) String() string { func (*MethodAnnotation) ProtoMessage() {} func (x *MethodAnnotation) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[131] + mi := &file_ateapi_proto_msgTypes[129] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9351,7 +9314,7 @@ func (x *MethodAnnotation) ProtoReflect() protoreflect.Message { // Deprecated: Use MethodAnnotation.ProtoReflect.Descriptor instead. func (*MethodAnnotation) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{131} + return file_ateapi_proto_rawDescGZIP(), []int{129} } func (x *MethodAnnotation) GetResource() string { @@ -9519,7 +9482,7 @@ const file_ateapi_proto_rawDesc = "" + "\bmetadata\x18\x01 \x01(\v2\x18.ateapi.ResourceMetadataR\bmetadata:\x11\x82\xb5\x18\r\x18\x02\"\tAtespaces\";\n" + "\tObjectRef\x12\x1a\n" + "\batespace\x18\x01 \x01(\tR\batespace\x12\x12\n" + - "\x04name\x18\x02 \x01(\tR\x04name\"\xda\x03\n" + + "\x04name\x18\x02 \x01(\tR\x04name\"\x9c\x03\n" + "\rActorTemplate\x124\n" + "\bmetadata\x18\x01 \x01(\v2\x18.ateapi.ResourceMetadataR\bmetadata\x129\n" + "\x0fworker_selector\x18\x02 \x01(\v2\x10.ateapi.SelectorR\x0eworkerSelector\x121\n" + @@ -9527,8 +9490,7 @@ const file_ateapi_proto_rawDesc = "" + "containers\x18\x03 \x03(\v2\x11.ateapi.ContainerR\n" + "containers\x12(\n" + "\avolumes\x18\x04 \x03(\v2\x0e.ateapi.VolumeR\avolumes\x12?\n" + - "\x0fsnapshot_config\x18\x05 \x01(\v2\x16.ateapi.SnapshotConfigR\x0esnapshotConfig\x12<\n" + - "\x0esandbox_config\x18\x06 \x01(\v2\x15.ateapi.SandboxConfigR\rsandboxConfig\x12/\n" + + "\x0fsnapshot_config\x18\x05 \x01(\v2\x16.ateapi.SnapshotConfigR\x0esnapshotConfig\x12/\n" + "\tresources\x18\a \x01(\v2\x11.ateapi.ResourcesR\tresources\x123\n" + "\x06status\x18\b \x01(\v2\x1b.ateapi.ActorTemplateStatusR\x06status:\x16\x82\xb5\x18\x12\x18\x01\"\x0eActorTemplates\"3\n" + "\tResources\x12&\n" + @@ -9542,11 +9504,7 @@ const file_ateapi_proto_rawDesc = "" + "\rsnapshot_time\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\fsnapshotTime\x12#\n" + "\rerror_message\x18\x03 \x01(\tR\ferrorMessage\"i\n" + "\x13ActorTemplateStatus\x12R\n" + - "\x16golden_snapshot_status\x18\x01 \x01(\v2\x1c.ateapi.GoldenSnapshotStatusR\x14goldenSnapshotStatus\"U\n" + - "\rSandboxConfig\x12#\n" + - "\rsandbox_class\x18\x01 \x01(\tR\fsandboxClass\x12\x1f\n" + - "\vconfig_name\x18\x02 \x01(\tR\n" + - "configName\"\xd8\x01\n" + + "\x16golden_snapshot_status\x18\x01 \x01(\v2\x1c.ateapi.GoldenSnapshotStatusR\x14goldenSnapshotStatus\"\xd8\x01\n" + "\x0eSnapshotConfig\x12)\n" + "\x10storage_location\x18\x01 \x01(\tR\x0fstorageLocation\x12G\n" + "\x12preferred_fidelity\x18\x02 \x01(\x0e2\x18.ateapi.SnapshotFidelityR\x11preferredFidelity\x12R\n" + @@ -9638,15 +9596,14 @@ const file_ateapi_proto_rawDesc = "" + "\x12CreateActorRequest\x12#\n" + "\x05actor\x18\x01 \x01(\v2\r.ateapi.ActorR\x05actor\"9\n" + "\x12UpdateActorRequest\x12#\n" + - "\x05actor\x18\x01 \x01(\v2\r.ateapi.ActorR\x05actor\">\n" + + "\x05actor\x18\x01 \x01(\v2\r.ateapi.ActorR\x05actor\"p\n" + "\x13SuspendActorRequest\x12'\n" + - "\x05actor\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\";\n" + + "\x05actor\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\x120\n" + + "\aoptions\x18\x02 \x01(\v2\x16.ateapi.SuspendOptionsR\aoptions\";\n" + "\x14SuspendActorResponse\x12#\n" + "\x05actor\x18\x01 \x01(\v2\r.ateapi.ActorR\x05actor\"<\n" + - "\x11PauseActorRequest\x12'\n" + - "\x05actor\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\"9\n" + - "\x12PauseActorResponse\x12#\n" + - "\x05actor\x18\x01 \x01(\v2\r.ateapi.ActorR\x05actor\"=\n" + + "\x0eSuspendOptions\x12*\n" + + "\x04mode\x18\x01 \x01(\x0e2\x16.ateapi.DurabilityModeR\x04mode\"=\n" + "\x12ResumeActorRequest\x12'\n" + "\x05actor\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\"T\n" + "\x13ResumeActorResponse\x12#\n" + @@ -9909,7 +9866,11 @@ const file_ateapi_proto_rawDesc = "" + " ACTOR_METADATA_FIELD_UNSPECIFIED\x10\x00\x12\x1d\n" + "\x19ACTOR_METADATA_FIELD_NAME\x10\x01\x12!\n" + "\x1dACTOR_METADATA_FIELD_ATESPACE\x10\x02\x12\x1c\n" + - "\x18ACTOR_METADATA_FIELD_UID\x10\x03*}\n" + + "\x18ACTOR_METADATA_FIELD_UID\x10\x03*t\n" + + "\x0eDurabilityMode\x12\x1f\n" + + "\x1bDURABILITY_MODE_UNSPECIFIED\x10\x00\x12\x1f\n" + + "\x1bDURABILITY_MODE_SYNCHRONOUS\x10\x01\x12 \n" + + "\x1cDURABILITY_MODE_ASYNCHRONOUS\x10\x02*}\n" + "\vWorkerState\x12\x1c\n" + "\x18WORKER_STATE_UNSPECIFIED\x10\x00\x12\x17\n" + "\x13WORKER_STATE_ACTIVE\x10\x01\x12\x19\n" + @@ -9918,7 +9879,7 @@ const file_ateapi_proto_rawDesc = "" + "\x05Scope\x12\x15\n" + "\x11SCOPE_UNSPECIFIED\x10\x00\x12\x12\n" + "\x0eSCOPE_ATESPACE\x10\x01\x12\x10\n" + - "\fSCOPE_GLOBAL\x10\x022\xbb\x1f\n" + + "\fSCOPE_GLOBAL\x10\x022\xe9\x1e\n" + "\aControl\x12?\n" + "\bGetActor\x12\x17.ateapi.GetActorRequest\x1a\r.ateapi.Actor\"\v\x82\xb5\x18\a\n" + "\x05Actor\x12E\n" + @@ -9927,9 +9888,6 @@ const file_ateapi_proto_rawDesc = "" + "\vUpdateActor\x12\x1a.ateapi.UpdateActorRequest\x1a\r.ateapi.Actor\"\v\x82\xb5\x18\a\n" + "\x05Actor\x12V\n" + "\fSuspendActor\x12\x1b.ateapi.SuspendActorRequest\x1a\x1c.ateapi.SuspendActorResponse\"\v\x82\xb5\x18\a\n" + - "\x05Actor\x12P\n" + - "\n" + - "PauseActor\x12\x19.ateapi.PauseActorRequest\x1a\x1a.ateapi.PauseActorResponse\"\v\x82\xb5\x18\a\n" + "\x05Actor\x12S\n" + "\vResumeActor\x12\x1a.ateapi.ResumeActorRequest\x1a\x1b.ateapi.ResumeActorResponse\"\v\x82\xb5\x18\a\n" + "\x05Actor\x12S\n" + @@ -10035,8 +9993,8 @@ func file_ateapi_proto_rawDescGZIP() []byte { return file_ateapi_proto_rawDescData } -var file_ateapi_proto_enumTypes = make([]protoimpl.EnumInfo, 9) -var file_ateapi_proto_msgTypes = make([]protoimpl.MessageInfo, 135) +var file_ateapi_proto_enumTypes = make([]protoimpl.EnumInfo, 10) +var file_ateapi_proto_msgTypes = make([]protoimpl.MessageInfo, 133) var file_ateapi_proto_goTypes = []any{ (SnapshotOwner)(0), // 0: ateapi.SnapshotOwner (SnapshotStorageStatus)(0), // 1: ateapi.SnapshotStorageStatus @@ -10045,38 +10003,38 @@ var file_ateapi_proto_goTypes = []any{ (ActorState)(0), // 4: ateapi.ActorState (GoldenSnapshotMode)(0), // 5: ateapi.GoldenSnapshotMode (ActorMetadataField)(0), // 6: ateapi.ActorMetadataField - (WorkerState)(0), // 7: ateapi.WorkerState - (Scope)(0), // 8: ateapi.Scope - (*ObjectSnapshot)(nil), // 9: ateapi.ObjectSnapshot - (*SnapshotStorage)(nil), // 10: ateapi.SnapshotStorage - (*Snapshot)(nil), // 11: ateapi.Snapshot - (*Selector)(nil), // 12: ateapi.Selector - (*ResourceMetadata)(nil), // 13: ateapi.ResourceMetadata - (*ExternalVolume)(nil), // 14: ateapi.ExternalVolume - (*Actor)(nil), // 15: ateapi.Actor - (*EgressPolicy)(nil), // 16: ateapi.EgressPolicy - (*EgressRule)(nil), // 17: ateapi.EgressRule - (*HttpRule)(nil), // 18: ateapi.HttpRule - (*HttpsRule)(nil), // 19: ateapi.HttpsRule - (*TlsPassthroughRule)(nil), // 20: ateapi.TlsPassthroughRule - (*Ports)(nil), // 21: ateapi.Ports - (*AllPorts)(nil), // 22: ateapi.AllPorts - (*HttpRuleEffects)(nil), // 23: ateapi.HttpRuleEffects - (*CredentialHeader)(nil), // 24: ateapi.CredentialHeader - (*ActorJwtSource)(nil), // 25: ateapi.ActorJwtSource - (*ActorStatus)(nil), // 26: ateapi.ActorStatus - (*ActorCrash)(nil), // 27: ateapi.ActorCrash - (*WorkerAssignment)(nil), // 28: ateapi.WorkerAssignment - (*TagStatus)(nil), // 29: ateapi.TagStatus - (*Tag)(nil), // 30: ateapi.Tag - (*Atespace)(nil), // 31: ateapi.Atespace - (*ObjectRef)(nil), // 32: ateapi.ObjectRef - (*ActorTemplate)(nil), // 33: ateapi.ActorTemplate - (*Resources)(nil), // 34: ateapi.Resources - (*Limits)(nil), // 35: ateapi.Limits - (*GoldenSnapshotStatus)(nil), // 36: ateapi.GoldenSnapshotStatus - (*ActorTemplateStatus)(nil), // 37: ateapi.ActorTemplateStatus - (*SandboxConfig)(nil), // 38: ateapi.SandboxConfig + (DurabilityMode)(0), // 7: ateapi.DurabilityMode + (WorkerState)(0), // 8: ateapi.WorkerState + (Scope)(0), // 9: ateapi.Scope + (*ObjectSnapshot)(nil), // 10: ateapi.ObjectSnapshot + (*SnapshotStorage)(nil), // 11: ateapi.SnapshotStorage + (*Snapshot)(nil), // 12: ateapi.Snapshot + (*Selector)(nil), // 13: ateapi.Selector + (*ResourceMetadata)(nil), // 14: ateapi.ResourceMetadata + (*ExternalVolume)(nil), // 15: ateapi.ExternalVolume + (*Actor)(nil), // 16: ateapi.Actor + (*EgressPolicy)(nil), // 17: ateapi.EgressPolicy + (*EgressRule)(nil), // 18: ateapi.EgressRule + (*HttpRule)(nil), // 19: ateapi.HttpRule + (*HttpsRule)(nil), // 20: ateapi.HttpsRule + (*TlsPassthroughRule)(nil), // 21: ateapi.TlsPassthroughRule + (*Ports)(nil), // 22: ateapi.Ports + (*AllPorts)(nil), // 23: ateapi.AllPorts + (*HttpRuleEffects)(nil), // 24: ateapi.HttpRuleEffects + (*CredentialHeader)(nil), // 25: ateapi.CredentialHeader + (*ActorJwtSource)(nil), // 26: ateapi.ActorJwtSource + (*ActorStatus)(nil), // 27: ateapi.ActorStatus + (*ActorCrash)(nil), // 28: ateapi.ActorCrash + (*WorkerAssignment)(nil), // 29: ateapi.WorkerAssignment + (*TagStatus)(nil), // 30: ateapi.TagStatus + (*Tag)(nil), // 31: ateapi.Tag + (*Atespace)(nil), // 32: ateapi.Atespace + (*ObjectRef)(nil), // 33: ateapi.ObjectRef + (*ActorTemplate)(nil), // 34: ateapi.ActorTemplate + (*Resources)(nil), // 35: ateapi.Resources + (*Limits)(nil), // 36: ateapi.Limits + (*GoldenSnapshotStatus)(nil), // 37: ateapi.GoldenSnapshotStatus + (*ActorTemplateStatus)(nil), // 38: ateapi.ActorTemplateStatus (*SnapshotConfig)(nil), // 39: ateapi.SnapshotConfig (*GoldenSnapshotConfig)(nil), // 40: ateapi.GoldenSnapshotConfig (*Container)(nil), // 41: ateapi.Container @@ -10110,371 +10068,367 @@ var file_ateapi_proto_goTypes = []any{ (*UpdateActorRequest)(nil), // 69: ateapi.UpdateActorRequest (*SuspendActorRequest)(nil), // 70: ateapi.SuspendActorRequest (*SuspendActorResponse)(nil), // 71: ateapi.SuspendActorResponse - (*PauseActorRequest)(nil), // 72: ateapi.PauseActorRequest - (*PauseActorResponse)(nil), // 73: ateapi.PauseActorResponse - (*ResumeActorRequest)(nil), // 74: ateapi.ResumeActorRequest - (*ResumeActorResponse)(nil), // 75: ateapi.ResumeActorResponse - (*RevertActorRequest)(nil), // 76: ateapi.RevertActorRequest - (*RevertActorResponse)(nil), // 77: ateapi.RevertActorResponse - (*DeleteActorRequest)(nil), // 78: ateapi.DeleteActorRequest - (*GetActorEgressPolicyRequest)(nil), // 79: ateapi.GetActorEgressPolicyRequest - (*CreateActorEgressPolicyRequest)(nil), // 80: ateapi.CreateActorEgressPolicyRequest - (*UpdateActorEgressPolicyRequest)(nil), // 81: ateapi.UpdateActorEgressPolicyRequest - (*DeleteActorEgressPolicyRequest)(nil), // 82: ateapi.DeleteActorEgressPolicyRequest - (*GetTagRequest)(nil), // 83: ateapi.GetTagRequest - (*MintActorJwtRequest)(nil), // 84: ateapi.MintActorJwtRequest - (*MintActorJwtResponse)(nil), // 85: ateapi.MintActorJwtResponse - (*MintActorCertificateRequest)(nil), // 86: ateapi.MintActorCertificateRequest - (*MintActorCertificateResponse)(nil), // 87: ateapi.MintActorCertificateResponse - (*ListTagsRequest)(nil), // 88: ateapi.ListTagsRequest - (*ListTagsResponse)(nil), // 89: ateapi.ListTagsResponse - (*CreateTagRequest)(nil), // 90: ateapi.CreateTagRequest - (*UpdateTagRequest)(nil), // 91: ateapi.UpdateTagRequest - (*DeleteTagRequest)(nil), // 92: ateapi.DeleteTagRequest - (*DeleteOptions)(nil), // 93: ateapi.DeleteOptions - (*ListOptions)(nil), // 94: ateapi.ListOptions - (*ListWorkerActorAssignmentsRequest)(nil), // 95: ateapi.ListWorkerActorAssignmentsRequest - (*ListWorkerActorAssignmentsResponse)(nil), // 96: ateapi.ListWorkerActorAssignmentsResponse - (*ListWorkersRequest)(nil), // 97: ateapi.ListWorkersRequest - (*ListWorkersResponse)(nil), // 98: ateapi.ListWorkersResponse - (*GetWorkerRequest)(nil), // 99: ateapi.GetWorkerRequest - (*CreateWorkerRequest)(nil), // 100: ateapi.CreateWorkerRequest - (*UpdateWorkerRequest)(nil), // 101: ateapi.UpdateWorkerRequest - (*DeleteWorkerRequest)(nil), // 102: ateapi.DeleteWorkerRequest - (*DrainWorkerRequest)(nil), // 103: ateapi.DrainWorkerRequest - (*DrainWorkerResponse)(nil), // 104: ateapi.DrainWorkerResponse - (*MarkWorkerAvailableRequest)(nil), // 105: ateapi.MarkWorkerAvailableRequest - (*MarkWorkerAvailableResponse)(nil), // 106: ateapi.MarkWorkerAvailableResponse - (*MarkWorkerUnavailableRequest)(nil), // 107: ateapi.MarkWorkerUnavailableRequest - (*MarkWorkerUnavailableResponse)(nil), // 108: ateapi.MarkWorkerUnavailableResponse - (*ListActorsRequest)(nil), // 109: ateapi.ListActorsRequest - (*ListActorsResponse)(nil), // 110: ateapi.ListActorsResponse - (*Worker)(nil), // 111: ateapi.Worker - (*WorkerStatus)(nil), // 112: ateapi.WorkerStatus - (*SandboxRuntime)(nil), // 113: ateapi.SandboxRuntime - (*VersionedSandboxCompat)(nil), // 114: ateapi.VersionedSandboxCompat - (*AttributeEntry)(nil), // 115: ateapi.AttributeEntry - (*WorkerResources)(nil), // 116: ateapi.WorkerResources - (*ActorAssignment)(nil), // 117: ateapi.ActorAssignment - (*RegisterWorkerRequest)(nil), // 118: ateapi.RegisterWorkerRequest - (*RegisterWorkerResponse)(nil), // 119: ateapi.RegisterWorkerResponse - (*MintWorkerActorCertificateRequest)(nil), // 120: ateapi.MintWorkerActorCertificateRequest - (*MintWorkerActorCertificateResponse)(nil), // 121: ateapi.MintWorkerActorCertificateResponse - (*MintWorkerActorJwtRequest)(nil), // 122: ateapi.MintWorkerActorJwtRequest - (*MintWorkerActorJwtResponse)(nil), // 123: ateapi.MintWorkerActorJwtResponse - (*RequestActorSuspendRequest)(nil), // 124: ateapi.RequestActorSuspendRequest - (*RequestActorSuspendResponse)(nil), // 125: ateapi.RequestActorSuspendResponse - (*ReportActorSnapshotStateRequest)(nil), // 126: ateapi.ReportActorSnapshotStateRequest - (*ReportActorSnapshotStateResponse)(nil), // 127: ateapi.ReportActorSnapshotStateResponse - (*ActorSnapshotState)(nil), // 128: ateapi.ActorSnapshotState - (*SnapshotReport)(nil), // 129: ateapi.SnapshotReport - (*AccessPolicy)(nil), // 130: ateapi.AccessPolicy - (*Binding)(nil), // 131: ateapi.Binding - (*GetGlobalAccessPolicyRequest)(nil), // 132: ateapi.GetGlobalAccessPolicyRequest - (*CreateGlobalAccessPolicyRequest)(nil), // 133: ateapi.CreateGlobalAccessPolicyRequest - (*UpdateGlobalAccessPolicyRequest)(nil), // 134: ateapi.UpdateGlobalAccessPolicyRequest - (*GetAtespaceAccessPolicyRequest)(nil), // 135: ateapi.GetAtespaceAccessPolicyRequest - (*CreateAtespaceAccessPolicyRequest)(nil), // 136: ateapi.CreateAtespaceAccessPolicyRequest - (*UpdateAtespaceAccessPolicyRequest)(nil), // 137: ateapi.UpdateAtespaceAccessPolicyRequest - (*DeleteAtespaceAccessPolicyRequest)(nil), // 138: ateapi.DeleteAtespaceAccessPolicyRequest - (*ResourceAnnotation)(nil), // 139: ateapi.ResourceAnnotation - (*MethodAnnotation)(nil), // 140: ateapi.MethodAnnotation - nil, // 141: ateapi.Selector.MatchLabelsEntry - nil, // 142: ateapi.ExternalVolume.VolumeContextEntry - nil, // 143: ateapi.Worker.LabelsEntry - (*timestamppb.Timestamp)(nil), // 144: google.protobuf.Timestamp - (*descriptorpb.MessageOptions)(nil), // 145: google.protobuf.MessageOptions - (*descriptorpb.MethodOptions)(nil), // 146: google.protobuf.MethodOptions + (*SuspendOptions)(nil), // 72: ateapi.SuspendOptions + (*ResumeActorRequest)(nil), // 73: ateapi.ResumeActorRequest + (*ResumeActorResponse)(nil), // 74: ateapi.ResumeActorResponse + (*RevertActorRequest)(nil), // 75: ateapi.RevertActorRequest + (*RevertActorResponse)(nil), // 76: ateapi.RevertActorResponse + (*DeleteActorRequest)(nil), // 77: ateapi.DeleteActorRequest + (*GetActorEgressPolicyRequest)(nil), // 78: ateapi.GetActorEgressPolicyRequest + (*CreateActorEgressPolicyRequest)(nil), // 79: ateapi.CreateActorEgressPolicyRequest + (*UpdateActorEgressPolicyRequest)(nil), // 80: ateapi.UpdateActorEgressPolicyRequest + (*DeleteActorEgressPolicyRequest)(nil), // 81: ateapi.DeleteActorEgressPolicyRequest + (*GetTagRequest)(nil), // 82: ateapi.GetTagRequest + (*MintActorJwtRequest)(nil), // 83: ateapi.MintActorJwtRequest + (*MintActorJwtResponse)(nil), // 84: ateapi.MintActorJwtResponse + (*MintActorCertificateRequest)(nil), // 85: ateapi.MintActorCertificateRequest + (*MintActorCertificateResponse)(nil), // 86: ateapi.MintActorCertificateResponse + (*ListTagsRequest)(nil), // 87: ateapi.ListTagsRequest + (*ListTagsResponse)(nil), // 88: ateapi.ListTagsResponse + (*CreateTagRequest)(nil), // 89: ateapi.CreateTagRequest + (*UpdateTagRequest)(nil), // 90: ateapi.UpdateTagRequest + (*DeleteTagRequest)(nil), // 91: ateapi.DeleteTagRequest + (*DeleteOptions)(nil), // 92: ateapi.DeleteOptions + (*ListOptions)(nil), // 93: ateapi.ListOptions + (*ListWorkerActorAssignmentsRequest)(nil), // 94: ateapi.ListWorkerActorAssignmentsRequest + (*ListWorkerActorAssignmentsResponse)(nil), // 95: ateapi.ListWorkerActorAssignmentsResponse + (*ListWorkersRequest)(nil), // 96: ateapi.ListWorkersRequest + (*ListWorkersResponse)(nil), // 97: ateapi.ListWorkersResponse + (*GetWorkerRequest)(nil), // 98: ateapi.GetWorkerRequest + (*CreateWorkerRequest)(nil), // 99: ateapi.CreateWorkerRequest + (*UpdateWorkerRequest)(nil), // 100: ateapi.UpdateWorkerRequest + (*DeleteWorkerRequest)(nil), // 101: ateapi.DeleteWorkerRequest + (*DrainWorkerRequest)(nil), // 102: ateapi.DrainWorkerRequest + (*DrainWorkerResponse)(nil), // 103: ateapi.DrainWorkerResponse + (*MarkWorkerAvailableRequest)(nil), // 104: ateapi.MarkWorkerAvailableRequest + (*MarkWorkerAvailableResponse)(nil), // 105: ateapi.MarkWorkerAvailableResponse + (*MarkWorkerUnavailableRequest)(nil), // 106: ateapi.MarkWorkerUnavailableRequest + (*MarkWorkerUnavailableResponse)(nil), // 107: ateapi.MarkWorkerUnavailableResponse + (*ListActorsRequest)(nil), // 108: ateapi.ListActorsRequest + (*ListActorsResponse)(nil), // 109: ateapi.ListActorsResponse + (*Worker)(nil), // 110: ateapi.Worker + (*WorkerStatus)(nil), // 111: ateapi.WorkerStatus + (*SandboxRuntime)(nil), // 112: ateapi.SandboxRuntime + (*VersionedSandboxCompat)(nil), // 113: ateapi.VersionedSandboxCompat + (*AttributeEntry)(nil), // 114: ateapi.AttributeEntry + (*WorkerResources)(nil), // 115: ateapi.WorkerResources + (*ActorAssignment)(nil), // 116: ateapi.ActorAssignment + (*RegisterWorkerRequest)(nil), // 117: ateapi.RegisterWorkerRequest + (*RegisterWorkerResponse)(nil), // 118: ateapi.RegisterWorkerResponse + (*MintWorkerActorCertificateRequest)(nil), // 119: ateapi.MintWorkerActorCertificateRequest + (*MintWorkerActorCertificateResponse)(nil), // 120: ateapi.MintWorkerActorCertificateResponse + (*MintWorkerActorJwtRequest)(nil), // 121: ateapi.MintWorkerActorJwtRequest + (*MintWorkerActorJwtResponse)(nil), // 122: ateapi.MintWorkerActorJwtResponse + (*RequestActorSuspendRequest)(nil), // 123: ateapi.RequestActorSuspendRequest + (*RequestActorSuspendResponse)(nil), // 124: ateapi.RequestActorSuspendResponse + (*ReportActorSnapshotStateRequest)(nil), // 125: ateapi.ReportActorSnapshotStateRequest + (*ReportActorSnapshotStateResponse)(nil), // 126: ateapi.ReportActorSnapshotStateResponse + (*ActorSnapshotState)(nil), // 127: ateapi.ActorSnapshotState + (*SnapshotReport)(nil), // 128: ateapi.SnapshotReport + (*AccessPolicy)(nil), // 129: ateapi.AccessPolicy + (*Binding)(nil), // 130: ateapi.Binding + (*GetGlobalAccessPolicyRequest)(nil), // 131: ateapi.GetGlobalAccessPolicyRequest + (*CreateGlobalAccessPolicyRequest)(nil), // 132: ateapi.CreateGlobalAccessPolicyRequest + (*UpdateGlobalAccessPolicyRequest)(nil), // 133: ateapi.UpdateGlobalAccessPolicyRequest + (*GetAtespaceAccessPolicyRequest)(nil), // 134: ateapi.GetAtespaceAccessPolicyRequest + (*CreateAtespaceAccessPolicyRequest)(nil), // 135: ateapi.CreateAtespaceAccessPolicyRequest + (*UpdateAtespaceAccessPolicyRequest)(nil), // 136: ateapi.UpdateAtespaceAccessPolicyRequest + (*DeleteAtespaceAccessPolicyRequest)(nil), // 137: ateapi.DeleteAtespaceAccessPolicyRequest + (*ResourceAnnotation)(nil), // 138: ateapi.ResourceAnnotation + (*MethodAnnotation)(nil), // 139: ateapi.MethodAnnotation + nil, // 140: ateapi.Selector.MatchLabelsEntry + nil, // 141: ateapi.ExternalVolume.VolumeContextEntry + nil, // 142: ateapi.Worker.LabelsEntry + (*timestamppb.Timestamp)(nil), // 143: google.protobuf.Timestamp + (*descriptorpb.MessageOptions)(nil), // 144: google.protobuf.MessageOptions + (*descriptorpb.MethodOptions)(nil), // 145: google.protobuf.MethodOptions } var file_ateapi_proto_depIdxs = []int32{ 1, // 0: ateapi.SnapshotStorage.status:type_name -> ateapi.SnapshotStorageStatus - 9, // 1: ateapi.SnapshotStorage.object:type_name -> ateapi.ObjectSnapshot + 10, // 1: ateapi.SnapshotStorage.object:type_name -> ateapi.ObjectSnapshot 0, // 2: ateapi.Snapshot.owner:type_name -> ateapi.SnapshotOwner 2, // 3: ateapi.Snapshot.fidelity:type_name -> ateapi.SnapshotFidelity - 10, // 4: ateapi.Snapshot.durable_snapshot:type_name -> ateapi.SnapshotStorage - 113, // 5: ateapi.Snapshot.sandbox_runtime:type_name -> ateapi.SandboxRuntime - 141, // 6: ateapi.Selector.match_labels:type_name -> ateapi.Selector.MatchLabelsEntry - 144, // 7: ateapi.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp - 144, // 8: ateapi.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp - 142, // 9: ateapi.ExternalVolume.volume_context:type_name -> ateapi.ExternalVolume.VolumeContextEntry - 13, // 10: ateapi.Actor.metadata:type_name -> ateapi.ResourceMetadata - 32, // 11: ateapi.Actor.actor_template:type_name -> ateapi.ObjectRef - 12, // 12: ateapi.Actor.worker_selector:type_name -> ateapi.Selector - 32, // 13: ateapi.Actor.source_tag:type_name -> ateapi.ObjectRef - 26, // 14: ateapi.Actor.status:type_name -> ateapi.ActorStatus - 13, // 15: ateapi.EgressPolicy.metadata:type_name -> ateapi.ResourceMetadata - 17, // 16: ateapi.EgressPolicy.rules:type_name -> ateapi.EgressRule - 18, // 17: ateapi.EgressRule.http:type_name -> ateapi.HttpRule - 19, // 18: ateapi.EgressRule.https:type_name -> ateapi.HttpsRule - 20, // 19: ateapi.EgressRule.tls_passthrough:type_name -> ateapi.TlsPassthroughRule - 21, // 20: ateapi.HttpRule.ports:type_name -> ateapi.Ports - 23, // 21: ateapi.HttpRule.effects:type_name -> ateapi.HttpRuleEffects - 21, // 22: ateapi.HttpsRule.ports:type_name -> ateapi.Ports - 23, // 23: ateapi.HttpsRule.effects:type_name -> ateapi.HttpRuleEffects - 21, // 24: ateapi.TlsPassthroughRule.ports:type_name -> ateapi.Ports - 22, // 25: ateapi.Ports.all:type_name -> ateapi.AllPorts - 24, // 26: ateapi.HttpRuleEffects.replace_headers:type_name -> ateapi.CredentialHeader - 25, // 27: ateapi.CredentialHeader.actor_jwt:type_name -> ateapi.ActorJwtSource + 11, // 4: ateapi.Snapshot.durable_snapshot:type_name -> ateapi.SnapshotStorage + 112, // 5: ateapi.Snapshot.sandbox_runtime:type_name -> ateapi.SandboxRuntime + 140, // 6: ateapi.Selector.match_labels:type_name -> ateapi.Selector.MatchLabelsEntry + 143, // 7: ateapi.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp + 143, // 8: ateapi.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp + 141, // 9: ateapi.ExternalVolume.volume_context:type_name -> ateapi.ExternalVolume.VolumeContextEntry + 14, // 10: ateapi.Actor.metadata:type_name -> ateapi.ResourceMetadata + 33, // 11: ateapi.Actor.actor_template:type_name -> ateapi.ObjectRef + 13, // 12: ateapi.Actor.worker_selector:type_name -> ateapi.Selector + 33, // 13: ateapi.Actor.source_tag:type_name -> ateapi.ObjectRef + 27, // 14: ateapi.Actor.status:type_name -> ateapi.ActorStatus + 14, // 15: ateapi.EgressPolicy.metadata:type_name -> ateapi.ResourceMetadata + 18, // 16: ateapi.EgressPolicy.rules:type_name -> ateapi.EgressRule + 19, // 17: ateapi.EgressRule.http:type_name -> ateapi.HttpRule + 20, // 18: ateapi.EgressRule.https:type_name -> ateapi.HttpsRule + 21, // 19: ateapi.EgressRule.tls_passthrough:type_name -> ateapi.TlsPassthroughRule + 22, // 20: ateapi.HttpRule.ports:type_name -> ateapi.Ports + 24, // 21: ateapi.HttpRule.effects:type_name -> ateapi.HttpRuleEffects + 22, // 22: ateapi.HttpsRule.ports:type_name -> ateapi.Ports + 24, // 23: ateapi.HttpsRule.effects:type_name -> ateapi.HttpRuleEffects + 22, // 24: ateapi.TlsPassthroughRule.ports:type_name -> ateapi.Ports + 23, // 25: ateapi.Ports.all:type_name -> ateapi.AllPorts + 25, // 26: ateapi.HttpRuleEffects.replace_headers:type_name -> ateapi.CredentialHeader + 26, // 27: ateapi.CredentialHeader.actor_jwt:type_name -> ateapi.ActorJwtSource 4, // 28: ateapi.ActorStatus.state:type_name -> ateapi.ActorState - 28, // 29: ateapi.ActorStatus.worker_assignment:type_name -> ateapi.WorkerAssignment - 14, // 30: ateapi.ActorStatus.external_volumes:type_name -> ateapi.ExternalVolume - 27, // 31: ateapi.ActorStatus.crash:type_name -> ateapi.ActorCrash - 11, // 32: ateapi.ActorStatus.snapshots:type_name -> ateapi.Snapshot - 144, // 33: ateapi.ActorCrash.crash_time:type_name -> google.protobuf.Timestamp - 32, // 34: ateapi.WorkerAssignment.worker:type_name -> ateapi.ObjectRef - 113, // 35: ateapi.WorkerAssignment.sandbox_runtime:type_name -> ateapi.SandboxRuntime - 11, // 36: ateapi.TagStatus.snapshot:type_name -> ateapi.Snapshot - 13, // 37: ateapi.Tag.metadata:type_name -> ateapi.ResourceMetadata - 29, // 38: ateapi.Tag.status:type_name -> ateapi.TagStatus + 29, // 29: ateapi.ActorStatus.worker_assignment:type_name -> ateapi.WorkerAssignment + 15, // 30: ateapi.ActorStatus.external_volumes:type_name -> ateapi.ExternalVolume + 28, // 31: ateapi.ActorStatus.crash:type_name -> ateapi.ActorCrash + 12, // 32: ateapi.ActorStatus.snapshots:type_name -> ateapi.Snapshot + 143, // 33: ateapi.ActorCrash.crash_time:type_name -> google.protobuf.Timestamp + 33, // 34: ateapi.WorkerAssignment.worker:type_name -> ateapi.ObjectRef + 112, // 35: ateapi.WorkerAssignment.sandbox_runtime:type_name -> ateapi.SandboxRuntime + 12, // 36: ateapi.TagStatus.snapshot:type_name -> ateapi.Snapshot + 14, // 37: ateapi.Tag.metadata:type_name -> ateapi.ResourceMetadata + 30, // 38: ateapi.Tag.status:type_name -> ateapi.TagStatus 3, // 39: ateapi.Tag.scope:type_name -> ateapi.TagScope - 32, // 40: ateapi.Tag.source_actor:type_name -> ateapi.ObjectRef - 13, // 41: ateapi.Atespace.metadata:type_name -> ateapi.ResourceMetadata - 13, // 42: ateapi.ActorTemplate.metadata:type_name -> ateapi.ResourceMetadata - 12, // 43: ateapi.ActorTemplate.worker_selector:type_name -> ateapi.Selector + 33, // 40: ateapi.Tag.source_actor:type_name -> ateapi.ObjectRef + 14, // 41: ateapi.Atespace.metadata:type_name -> ateapi.ResourceMetadata + 14, // 42: ateapi.ActorTemplate.metadata:type_name -> ateapi.ResourceMetadata + 13, // 43: ateapi.ActorTemplate.worker_selector:type_name -> ateapi.Selector 41, // 44: ateapi.ActorTemplate.containers:type_name -> ateapi.Container 47, // 45: ateapi.ActorTemplate.volumes:type_name -> ateapi.Volume 39, // 46: ateapi.ActorTemplate.snapshot_config:type_name -> ateapi.SnapshotConfig - 38, // 47: ateapi.ActorTemplate.sandbox_config:type_name -> ateapi.SandboxConfig - 34, // 48: ateapi.ActorTemplate.resources:type_name -> ateapi.Resources - 37, // 49: ateapi.ActorTemplate.status:type_name -> ateapi.ActorTemplateStatus - 35, // 50: ateapi.Resources.limits:type_name -> ateapi.Limits - 32, // 51: ateapi.GoldenSnapshotStatus.golden_tag:type_name -> ateapi.ObjectRef - 144, // 52: ateapi.GoldenSnapshotStatus.snapshot_time:type_name -> google.protobuf.Timestamp - 36, // 53: ateapi.ActorTemplateStatus.golden_snapshot_status:type_name -> ateapi.GoldenSnapshotStatus - 2, // 54: ateapi.SnapshotConfig.preferred_fidelity:type_name -> ateapi.SnapshotFidelity - 40, // 55: ateapi.SnapshotConfig.golden_snapshot_config:type_name -> ateapi.GoldenSnapshotConfig - 5, // 56: ateapi.GoldenSnapshotConfig.mode:type_name -> ateapi.GoldenSnapshotMode - 44, // 57: ateapi.Container.env:type_name -> ateapi.EnvVar - 45, // 58: ateapi.Container.wakeup_probe:type_name -> ateapi.WakeupProbe - 56, // 59: ateapi.Container.volume_mounts:type_name -> ateapi.VolumeMount - 42, // 60: ateapi.Container.security_context:type_name -> ateapi.SecurityContext - 34, // 61: ateapi.Container.resources:type_name -> ateapi.Resources - 43, // 62: ateapi.SecurityContext.capabilities:type_name -> ateapi.Capabilities - 46, // 63: ateapi.WakeupProbe.http_get:type_name -> ateapi.HttpGetAction - 49, // 64: ateapi.Volume.durable_dir:type_name -> ateapi.DurableDirVolumeSource - 50, // 65: ateapi.Volume.external_volume_template:type_name -> ateapi.ExternalVolumeTemplate - 51, // 66: ateapi.Volume.system_info:type_name -> ateapi.SystemInfoVolumeSource - 48, // 67: ateapi.Volume.image:type_name -> ateapi.ImageVolumeSource - 52, // 68: ateapi.SystemInfoVolumeSource.data_sources:type_name -> ateapi.SystemInfoDataSource - 53, // 69: ateapi.SystemInfoDataSource.actor_metadata:type_name -> ateapi.ActorMetadataDataSource - 55, // 70: ateapi.SystemInfoDataSource.trust_bundle:type_name -> ateapi.TrustBundleDataSource - 54, // 71: ateapi.ActorMetadataDataSource.items:type_name -> ateapi.ActorMetadataItem - 6, // 72: ateapi.ActorMetadataItem.field:type_name -> ateapi.ActorMetadataField - 31, // 73: ateapi.CreateAtespaceRequest.atespace:type_name -> ateapi.Atespace - 32, // 74: ateapi.GetAtespaceRequest.atespace:type_name -> ateapi.ObjectRef - 94, // 75: ateapi.ListAtespacesRequest.options:type_name -> ateapi.ListOptions - 31, // 76: ateapi.ListAtespacesResponse.atespaces:type_name -> ateapi.Atespace - 32, // 77: ateapi.DeleteAtespaceRequest.atespace:type_name -> ateapi.ObjectRef - 93, // 78: ateapi.DeleteAtespaceRequest.options:type_name -> ateapi.DeleteOptions - 33, // 79: ateapi.CreateActorTemplateRequest.actor_template:type_name -> ateapi.ActorTemplate - 32, // 80: ateapi.GetActorTemplateRequest.actor_template:type_name -> ateapi.ObjectRef - 94, // 81: ateapi.ListActorTemplatesRequest.options:type_name -> ateapi.ListOptions - 33, // 82: ateapi.ListActorTemplatesResponse.actor_templates:type_name -> ateapi.ActorTemplate - 32, // 83: ateapi.DeleteActorTemplateRequest.actor_template:type_name -> ateapi.ObjectRef - 93, // 84: ateapi.DeleteActorTemplateRequest.options:type_name -> ateapi.DeleteOptions - 32, // 85: ateapi.GetActorRequest.actor:type_name -> ateapi.ObjectRef - 15, // 86: ateapi.CreateActorRequest.actor:type_name -> ateapi.Actor - 15, // 87: ateapi.UpdateActorRequest.actor:type_name -> ateapi.Actor - 32, // 88: ateapi.SuspendActorRequest.actor:type_name -> ateapi.ObjectRef - 15, // 89: ateapi.SuspendActorResponse.actor:type_name -> ateapi.Actor - 32, // 90: ateapi.PauseActorRequest.actor:type_name -> ateapi.ObjectRef - 15, // 91: ateapi.PauseActorResponse.actor:type_name -> ateapi.Actor - 32, // 92: ateapi.ResumeActorRequest.actor:type_name -> ateapi.ObjectRef - 15, // 93: ateapi.ResumeActorResponse.actor:type_name -> ateapi.Actor - 32, // 94: ateapi.RevertActorRequest.actor:type_name -> ateapi.ObjectRef - 15, // 95: ateapi.RevertActorResponse.actor:type_name -> ateapi.Actor - 32, // 96: ateapi.DeleteActorRequest.actor:type_name -> ateapi.ObjectRef - 93, // 97: ateapi.DeleteActorRequest.options:type_name -> ateapi.DeleteOptions - 32, // 98: ateapi.GetActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef - 32, // 99: ateapi.CreateActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef - 16, // 100: ateapi.CreateActorEgressPolicyRequest.egress_policy:type_name -> ateapi.EgressPolicy - 32, // 101: ateapi.UpdateActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef - 16, // 102: ateapi.UpdateActorEgressPolicyRequest.egress_policy:type_name -> ateapi.EgressPolicy - 32, // 103: ateapi.DeleteActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef - 93, // 104: ateapi.DeleteActorEgressPolicyRequest.options:type_name -> ateapi.DeleteOptions - 32, // 105: ateapi.GetTagRequest.tag:type_name -> ateapi.ObjectRef - 32, // 106: ateapi.MintActorJwtRequest.actor:type_name -> ateapi.ObjectRef - 144, // 107: ateapi.MintActorJwtResponse.expire_time:type_name -> google.protobuf.Timestamp - 32, // 108: ateapi.MintActorCertificateRequest.actor:type_name -> ateapi.ObjectRef - 94, // 109: ateapi.ListTagsRequest.options:type_name -> ateapi.ListOptions - 30, // 110: ateapi.ListTagsResponse.tags:type_name -> ateapi.Tag - 30, // 111: ateapi.CreateTagRequest.tag:type_name -> ateapi.Tag - 30, // 112: ateapi.UpdateTagRequest.tag:type_name -> ateapi.Tag - 32, // 113: ateapi.DeleteTagRequest.tag:type_name -> ateapi.ObjectRef - 93, // 114: ateapi.DeleteTagRequest.options:type_name -> ateapi.DeleteOptions - 32, // 115: ateapi.ListWorkerActorAssignmentsRequest.worker:type_name -> ateapi.ObjectRef - 94, // 116: ateapi.ListWorkerActorAssignmentsRequest.options:type_name -> ateapi.ListOptions - 117, // 117: ateapi.ListWorkerActorAssignmentsResponse.actor_assignments:type_name -> ateapi.ActorAssignment - 94, // 118: ateapi.ListWorkersRequest.options:type_name -> ateapi.ListOptions - 111, // 119: ateapi.ListWorkersResponse.workers:type_name -> ateapi.Worker - 32, // 120: ateapi.GetWorkerRequest.worker:type_name -> ateapi.ObjectRef - 111, // 121: ateapi.CreateWorkerRequest.worker:type_name -> ateapi.Worker - 111, // 122: ateapi.UpdateWorkerRequest.worker:type_name -> ateapi.Worker - 32, // 123: ateapi.DeleteWorkerRequest.worker:type_name -> ateapi.ObjectRef - 93, // 124: ateapi.DeleteWorkerRequest.options:type_name -> ateapi.DeleteOptions - 32, // 125: ateapi.DrainWorkerRequest.worker:type_name -> ateapi.ObjectRef - 111, // 126: ateapi.DrainWorkerResponse.worker:type_name -> ateapi.Worker - 32, // 127: ateapi.MarkWorkerAvailableRequest.worker:type_name -> ateapi.ObjectRef - 111, // 128: ateapi.MarkWorkerAvailableResponse.worker:type_name -> ateapi.Worker - 32, // 129: ateapi.MarkWorkerUnavailableRequest.worker:type_name -> ateapi.ObjectRef - 111, // 130: ateapi.MarkWorkerUnavailableResponse.worker:type_name -> ateapi.Worker - 94, // 131: ateapi.ListActorsRequest.options:type_name -> ateapi.ListOptions - 15, // 132: ateapi.ListActorsResponse.actors:type_name -> ateapi.Actor - 13, // 133: ateapi.Worker.metadata:type_name -> ateapi.ResourceMetadata - 143, // 134: ateapi.Worker.labels:type_name -> ateapi.Worker.LabelsEntry - 112, // 135: ateapi.Worker.status:type_name -> ateapi.WorkerStatus - 7, // 136: ateapi.WorkerStatus.state:type_name -> ateapi.WorkerState - 116, // 137: ateapi.WorkerStatus.capacity:type_name -> ateapi.WorkerResources - 116, // 138: ateapi.WorkerStatus.allocated:type_name -> ateapi.WorkerResources - 113, // 139: ateapi.WorkerStatus.default_runtime:type_name -> ateapi.SandboxRuntime - 113, // 140: ateapi.WorkerStatus.restorable_runtimes:type_name -> ateapi.SandboxRuntime - 114, // 141: ateapi.SandboxRuntime.version:type_name -> ateapi.VersionedSandboxCompat - 115, // 142: ateapi.VersionedSandboxCompat.attributes:type_name -> ateapi.AttributeEntry - 34, // 143: ateapi.WorkerResources.resources:type_name -> ateapi.Resources - 13, // 144: ateapi.ActorAssignment.metadata:type_name -> ateapi.ResourceMetadata - 32, // 145: ateapi.ActorAssignment.actor:type_name -> ateapi.ObjectRef - 34, // 146: ateapi.ActorAssignment.resources:type_name -> ateapi.Resources - 32, // 147: ateapi.RegisterWorkerRequest.worker:type_name -> ateapi.ObjectRef - 116, // 148: ateapi.RegisterWorkerRequest.capacity:type_name -> ateapi.WorkerResources - 113, // 149: ateapi.RegisterWorkerRequest.default_runtime:type_name -> ateapi.SandboxRuntime - 113, // 150: ateapi.RegisterWorkerRequest.restorable_runtimes:type_name -> ateapi.SandboxRuntime - 111, // 151: ateapi.RegisterWorkerResponse.worker:type_name -> ateapi.Worker - 32, // 152: ateapi.MintWorkerActorCertificateRequest.actor:type_name -> ateapi.ObjectRef - 32, // 153: ateapi.MintWorkerActorCertificateRequest.worker:type_name -> ateapi.ObjectRef - 32, // 154: ateapi.MintWorkerActorJwtRequest.actor:type_name -> ateapi.ObjectRef - 144, // 155: ateapi.MintWorkerActorJwtResponse.expire_time:type_name -> google.protobuf.Timestamp - 32, // 156: ateapi.RequestActorSuspendRequest.worker:type_name -> ateapi.ObjectRef - 32, // 157: ateapi.RequestActorSuspendRequest.actor:type_name -> ateapi.ObjectRef - 15, // 158: ateapi.RequestActorSuspendResponse.actor:type_name -> ateapi.Actor - 32, // 159: ateapi.ReportActorSnapshotStateRequest.actor:type_name -> ateapi.ObjectRef - 128, // 160: ateapi.ReportActorSnapshotStateRequest.state:type_name -> ateapi.ActorSnapshotState - 129, // 161: ateapi.ActorSnapshotState.latest_local:type_name -> ateapi.SnapshotReport - 129, // 162: ateapi.ActorSnapshotState.uploading:type_name -> ateapi.SnapshotReport - 129, // 163: ateapi.ActorSnapshotState.latest_durable:type_name -> ateapi.SnapshotReport - 144, // 164: ateapi.ActorSnapshotState.upload_due_time:type_name -> google.protobuf.Timestamp - 2, // 165: ateapi.SnapshotReport.fidelity:type_name -> ateapi.SnapshotFidelity - 9, // 166: ateapi.SnapshotReport.object:type_name -> ateapi.ObjectSnapshot - 144, // 167: ateapi.SnapshotReport.local_commit_time:type_name -> google.protobuf.Timestamp - 144, // 168: ateapi.SnapshotReport.durable_complete_time:type_name -> google.protobuf.Timestamp - 13, // 169: ateapi.AccessPolicy.metadata:type_name -> ateapi.ResourceMetadata - 131, // 170: ateapi.AccessPolicy.bindings:type_name -> ateapi.Binding - 130, // 171: ateapi.CreateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 130, // 172: ateapi.UpdateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 32, // 173: ateapi.GetAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 32, // 174: ateapi.CreateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 130, // 175: ateapi.CreateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 32, // 176: ateapi.UpdateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 130, // 177: ateapi.UpdateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 32, // 178: ateapi.DeleteAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 93, // 179: ateapi.DeleteAtespaceAccessPolicyRequest.options:type_name -> ateapi.DeleteOptions - 8, // 180: ateapi.ResourceAnnotation.scope:type_name -> ateapi.Scope - 145, // 181: ateapi.resource:extendee -> google.protobuf.MessageOptions - 146, // 182: ateapi.method:extendee -> google.protobuf.MethodOptions - 139, // 183: ateapi.resource:type_name -> ateapi.ResourceAnnotation - 140, // 184: ateapi.method:type_name -> ateapi.MethodAnnotation - 67, // 185: ateapi.Control.GetActor:input_type -> ateapi.GetActorRequest - 68, // 186: ateapi.Control.CreateActor:input_type -> ateapi.CreateActorRequest - 69, // 187: ateapi.Control.UpdateActor:input_type -> ateapi.UpdateActorRequest - 70, // 188: ateapi.Control.SuspendActor:input_type -> ateapi.SuspendActorRequest - 72, // 189: ateapi.Control.PauseActor:input_type -> ateapi.PauseActorRequest - 74, // 190: ateapi.Control.ResumeActor:input_type -> ateapi.ResumeActorRequest - 76, // 191: ateapi.Control.RevertActor:input_type -> ateapi.RevertActorRequest - 78, // 192: ateapi.Control.DeleteActor:input_type -> ateapi.DeleteActorRequest - 79, // 193: ateapi.Control.GetActorEgressPolicy:input_type -> ateapi.GetActorEgressPolicyRequest - 80, // 194: ateapi.Control.CreateActorEgressPolicy:input_type -> ateapi.CreateActorEgressPolicyRequest - 81, // 195: ateapi.Control.UpdateActorEgressPolicy:input_type -> ateapi.UpdateActorEgressPolicyRequest - 82, // 196: ateapi.Control.DeleteActorEgressPolicy:input_type -> ateapi.DeleteActorEgressPolicyRequest - 84, // 197: ateapi.Control.MintActorJwt:input_type -> ateapi.MintActorJwtRequest - 86, // 198: ateapi.Control.MintActorCertificate:input_type -> ateapi.MintActorCertificateRequest - 90, // 199: ateapi.Control.CreateTag:input_type -> ateapi.CreateTagRequest - 83, // 200: ateapi.Control.GetTag:input_type -> ateapi.GetTagRequest - 88, // 201: ateapi.Control.ListTags:input_type -> ateapi.ListTagsRequest - 91, // 202: ateapi.Control.UpdateTag:input_type -> ateapi.UpdateTagRequest - 92, // 203: ateapi.Control.DeleteTag:input_type -> ateapi.DeleteTagRequest - 97, // 204: ateapi.Control.ListWorkers:input_type -> ateapi.ListWorkersRequest - 99, // 205: ateapi.Control.GetWorker:input_type -> ateapi.GetWorkerRequest - 100, // 206: ateapi.Control.CreateWorker:input_type -> ateapi.CreateWorkerRequest - 101, // 207: ateapi.Control.UpdateWorker:input_type -> ateapi.UpdateWorkerRequest - 102, // 208: ateapi.Control.DeleteWorker:input_type -> ateapi.DeleteWorkerRequest - 103, // 209: ateapi.Control.DrainWorker:input_type -> ateapi.DrainWorkerRequest - 105, // 210: ateapi.Control.MarkWorkerAvailable:input_type -> ateapi.MarkWorkerAvailableRequest - 107, // 211: ateapi.Control.MarkWorkerUnavailable:input_type -> ateapi.MarkWorkerUnavailableRequest - 95, // 212: ateapi.Control.ListWorkerActorAssignments:input_type -> ateapi.ListWorkerActorAssignmentsRequest - 109, // 213: ateapi.Control.ListActors:input_type -> ateapi.ListActorsRequest - 57, // 214: ateapi.Control.CreateAtespace:input_type -> ateapi.CreateAtespaceRequest - 58, // 215: ateapi.Control.GetAtespace:input_type -> ateapi.GetAtespaceRequest - 59, // 216: ateapi.Control.ListAtespaces:input_type -> ateapi.ListAtespacesRequest - 61, // 217: ateapi.Control.DeleteAtespace:input_type -> ateapi.DeleteAtespaceRequest - 62, // 218: ateapi.Control.CreateActorTemplate:input_type -> ateapi.CreateActorTemplateRequest - 63, // 219: ateapi.Control.GetActorTemplate:input_type -> ateapi.GetActorTemplateRequest - 64, // 220: ateapi.Control.ListActorTemplates:input_type -> ateapi.ListActorTemplatesRequest - 66, // 221: ateapi.Control.DeleteActorTemplate:input_type -> ateapi.DeleteActorTemplateRequest - 132, // 222: ateapi.Control.GetGlobalAccessPolicy:input_type -> ateapi.GetGlobalAccessPolicyRequest - 133, // 223: ateapi.Control.CreateGlobalAccessPolicy:input_type -> ateapi.CreateGlobalAccessPolicyRequest - 134, // 224: ateapi.Control.UpdateGlobalAccessPolicy:input_type -> ateapi.UpdateGlobalAccessPolicyRequest - 135, // 225: ateapi.Control.GetAtespaceAccessPolicy:input_type -> ateapi.GetAtespaceAccessPolicyRequest - 136, // 226: ateapi.Control.CreateAtespaceAccessPolicy:input_type -> ateapi.CreateAtespaceAccessPolicyRequest - 137, // 227: ateapi.Control.UpdateAtespaceAccessPolicy:input_type -> ateapi.UpdateAtespaceAccessPolicyRequest - 138, // 228: ateapi.Control.DeleteAtespaceAccessPolicy:input_type -> ateapi.DeleteAtespaceAccessPolicyRequest - 118, // 229: ateapi.WorkerService.RegisterWorker:input_type -> ateapi.RegisterWorkerRequest - 120, // 230: ateapi.WorkerService.MintWorkerActorCertificate:input_type -> ateapi.MintWorkerActorCertificateRequest - 122, // 231: ateapi.WorkerService.MintWorkerActorJwt:input_type -> ateapi.MintWorkerActorJwtRequest - 124, // 232: ateapi.WorkerService.RequestActorSuspend:input_type -> ateapi.RequestActorSuspendRequest - 126, // 233: ateapi.WorkerService.ReportActorSnapshotState:input_type -> ateapi.ReportActorSnapshotStateRequest - 15, // 234: ateapi.Control.GetActor:output_type -> ateapi.Actor - 15, // 235: ateapi.Control.CreateActor:output_type -> ateapi.Actor - 15, // 236: ateapi.Control.UpdateActor:output_type -> ateapi.Actor - 71, // 237: ateapi.Control.SuspendActor:output_type -> ateapi.SuspendActorResponse - 73, // 238: ateapi.Control.PauseActor:output_type -> ateapi.PauseActorResponse - 75, // 239: ateapi.Control.ResumeActor:output_type -> ateapi.ResumeActorResponse - 77, // 240: ateapi.Control.RevertActor:output_type -> ateapi.RevertActorResponse - 15, // 241: ateapi.Control.DeleteActor:output_type -> ateapi.Actor - 16, // 242: ateapi.Control.GetActorEgressPolicy:output_type -> ateapi.EgressPolicy - 16, // 243: ateapi.Control.CreateActorEgressPolicy:output_type -> ateapi.EgressPolicy - 16, // 244: ateapi.Control.UpdateActorEgressPolicy:output_type -> ateapi.EgressPolicy - 16, // 245: ateapi.Control.DeleteActorEgressPolicy:output_type -> ateapi.EgressPolicy - 85, // 246: ateapi.Control.MintActorJwt:output_type -> ateapi.MintActorJwtResponse - 87, // 247: ateapi.Control.MintActorCertificate:output_type -> ateapi.MintActorCertificateResponse - 30, // 248: ateapi.Control.CreateTag:output_type -> ateapi.Tag - 30, // 249: ateapi.Control.GetTag:output_type -> ateapi.Tag - 89, // 250: ateapi.Control.ListTags:output_type -> ateapi.ListTagsResponse - 30, // 251: ateapi.Control.UpdateTag:output_type -> ateapi.Tag - 30, // 252: ateapi.Control.DeleteTag:output_type -> ateapi.Tag - 98, // 253: ateapi.Control.ListWorkers:output_type -> ateapi.ListWorkersResponse - 111, // 254: ateapi.Control.GetWorker:output_type -> ateapi.Worker - 111, // 255: ateapi.Control.CreateWorker:output_type -> ateapi.Worker - 111, // 256: ateapi.Control.UpdateWorker:output_type -> ateapi.Worker - 111, // 257: ateapi.Control.DeleteWorker:output_type -> ateapi.Worker - 104, // 258: ateapi.Control.DrainWorker:output_type -> ateapi.DrainWorkerResponse - 106, // 259: ateapi.Control.MarkWorkerAvailable:output_type -> ateapi.MarkWorkerAvailableResponse - 108, // 260: ateapi.Control.MarkWorkerUnavailable:output_type -> ateapi.MarkWorkerUnavailableResponse - 96, // 261: ateapi.Control.ListWorkerActorAssignments:output_type -> ateapi.ListWorkerActorAssignmentsResponse - 110, // 262: ateapi.Control.ListActors:output_type -> ateapi.ListActorsResponse - 31, // 263: ateapi.Control.CreateAtespace:output_type -> ateapi.Atespace - 31, // 264: ateapi.Control.GetAtespace:output_type -> ateapi.Atespace - 60, // 265: ateapi.Control.ListAtespaces:output_type -> ateapi.ListAtespacesResponse - 31, // 266: ateapi.Control.DeleteAtespace:output_type -> ateapi.Atespace - 33, // 267: ateapi.Control.CreateActorTemplate:output_type -> ateapi.ActorTemplate - 33, // 268: ateapi.Control.GetActorTemplate:output_type -> ateapi.ActorTemplate - 65, // 269: ateapi.Control.ListActorTemplates:output_type -> ateapi.ListActorTemplatesResponse - 33, // 270: ateapi.Control.DeleteActorTemplate:output_type -> ateapi.ActorTemplate - 130, // 271: ateapi.Control.GetGlobalAccessPolicy:output_type -> ateapi.AccessPolicy - 130, // 272: ateapi.Control.CreateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy - 130, // 273: ateapi.Control.UpdateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy - 130, // 274: ateapi.Control.GetAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 130, // 275: ateapi.Control.CreateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 130, // 276: ateapi.Control.UpdateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 130, // 277: ateapi.Control.DeleteAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 119, // 278: ateapi.WorkerService.RegisterWorker:output_type -> ateapi.RegisterWorkerResponse - 121, // 279: ateapi.WorkerService.MintWorkerActorCertificate:output_type -> ateapi.MintWorkerActorCertificateResponse - 123, // 280: ateapi.WorkerService.MintWorkerActorJwt:output_type -> ateapi.MintWorkerActorJwtResponse - 125, // 281: ateapi.WorkerService.RequestActorSuspend:output_type -> ateapi.RequestActorSuspendResponse - 127, // 282: ateapi.WorkerService.ReportActorSnapshotState:output_type -> ateapi.ReportActorSnapshotStateResponse - 234, // [234:283] is the sub-list for method output_type - 185, // [185:234] is the sub-list for method input_type - 183, // [183:185] is the sub-list for extension type_name - 181, // [181:183] is the sub-list for extension extendee - 0, // [0:181] is the sub-list for field type_name + 35, // 47: ateapi.ActorTemplate.resources:type_name -> ateapi.Resources + 38, // 48: ateapi.ActorTemplate.status:type_name -> ateapi.ActorTemplateStatus + 36, // 49: ateapi.Resources.limits:type_name -> ateapi.Limits + 33, // 50: ateapi.GoldenSnapshotStatus.golden_tag:type_name -> ateapi.ObjectRef + 143, // 51: ateapi.GoldenSnapshotStatus.snapshot_time:type_name -> google.protobuf.Timestamp + 37, // 52: ateapi.ActorTemplateStatus.golden_snapshot_status:type_name -> ateapi.GoldenSnapshotStatus + 2, // 53: ateapi.SnapshotConfig.preferred_fidelity:type_name -> ateapi.SnapshotFidelity + 40, // 54: ateapi.SnapshotConfig.golden_snapshot_config:type_name -> ateapi.GoldenSnapshotConfig + 5, // 55: ateapi.GoldenSnapshotConfig.mode:type_name -> ateapi.GoldenSnapshotMode + 44, // 56: ateapi.Container.env:type_name -> ateapi.EnvVar + 45, // 57: ateapi.Container.wakeup_probe:type_name -> ateapi.WakeupProbe + 56, // 58: ateapi.Container.volume_mounts:type_name -> ateapi.VolumeMount + 42, // 59: ateapi.Container.security_context:type_name -> ateapi.SecurityContext + 35, // 60: ateapi.Container.resources:type_name -> ateapi.Resources + 43, // 61: ateapi.SecurityContext.capabilities:type_name -> ateapi.Capabilities + 46, // 62: ateapi.WakeupProbe.http_get:type_name -> ateapi.HttpGetAction + 49, // 63: ateapi.Volume.durable_dir:type_name -> ateapi.DurableDirVolumeSource + 50, // 64: ateapi.Volume.external_volume_template:type_name -> ateapi.ExternalVolumeTemplate + 51, // 65: ateapi.Volume.system_info:type_name -> ateapi.SystemInfoVolumeSource + 48, // 66: ateapi.Volume.image:type_name -> ateapi.ImageVolumeSource + 52, // 67: ateapi.SystemInfoVolumeSource.data_sources:type_name -> ateapi.SystemInfoDataSource + 53, // 68: ateapi.SystemInfoDataSource.actor_metadata:type_name -> ateapi.ActorMetadataDataSource + 55, // 69: ateapi.SystemInfoDataSource.trust_bundle:type_name -> ateapi.TrustBundleDataSource + 54, // 70: ateapi.ActorMetadataDataSource.items:type_name -> ateapi.ActorMetadataItem + 6, // 71: ateapi.ActorMetadataItem.field:type_name -> ateapi.ActorMetadataField + 32, // 72: ateapi.CreateAtespaceRequest.atespace:type_name -> ateapi.Atespace + 33, // 73: ateapi.GetAtespaceRequest.atespace:type_name -> ateapi.ObjectRef + 93, // 74: ateapi.ListAtespacesRequest.options:type_name -> ateapi.ListOptions + 32, // 75: ateapi.ListAtespacesResponse.atespaces:type_name -> ateapi.Atespace + 33, // 76: ateapi.DeleteAtespaceRequest.atespace:type_name -> ateapi.ObjectRef + 92, // 77: ateapi.DeleteAtespaceRequest.options:type_name -> ateapi.DeleteOptions + 34, // 78: ateapi.CreateActorTemplateRequest.actor_template:type_name -> ateapi.ActorTemplate + 33, // 79: ateapi.GetActorTemplateRequest.actor_template:type_name -> ateapi.ObjectRef + 93, // 80: ateapi.ListActorTemplatesRequest.options:type_name -> ateapi.ListOptions + 34, // 81: ateapi.ListActorTemplatesResponse.actor_templates:type_name -> ateapi.ActorTemplate + 33, // 82: ateapi.DeleteActorTemplateRequest.actor_template:type_name -> ateapi.ObjectRef + 92, // 83: ateapi.DeleteActorTemplateRequest.options:type_name -> ateapi.DeleteOptions + 33, // 84: ateapi.GetActorRequest.actor:type_name -> ateapi.ObjectRef + 16, // 85: ateapi.CreateActorRequest.actor:type_name -> ateapi.Actor + 16, // 86: ateapi.UpdateActorRequest.actor:type_name -> ateapi.Actor + 33, // 87: ateapi.SuspendActorRequest.actor:type_name -> ateapi.ObjectRef + 72, // 88: ateapi.SuspendActorRequest.options:type_name -> ateapi.SuspendOptions + 16, // 89: ateapi.SuspendActorResponse.actor:type_name -> ateapi.Actor + 7, // 90: ateapi.SuspendOptions.mode:type_name -> ateapi.DurabilityMode + 33, // 91: ateapi.ResumeActorRequest.actor:type_name -> ateapi.ObjectRef + 16, // 92: ateapi.ResumeActorResponse.actor:type_name -> ateapi.Actor + 33, // 93: ateapi.RevertActorRequest.actor:type_name -> ateapi.ObjectRef + 16, // 94: ateapi.RevertActorResponse.actor:type_name -> ateapi.Actor + 33, // 95: ateapi.DeleteActorRequest.actor:type_name -> ateapi.ObjectRef + 92, // 96: ateapi.DeleteActorRequest.options:type_name -> ateapi.DeleteOptions + 33, // 97: ateapi.GetActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef + 33, // 98: ateapi.CreateActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef + 17, // 99: ateapi.CreateActorEgressPolicyRequest.egress_policy:type_name -> ateapi.EgressPolicy + 33, // 100: ateapi.UpdateActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef + 17, // 101: ateapi.UpdateActorEgressPolicyRequest.egress_policy:type_name -> ateapi.EgressPolicy + 33, // 102: ateapi.DeleteActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef + 92, // 103: ateapi.DeleteActorEgressPolicyRequest.options:type_name -> ateapi.DeleteOptions + 33, // 104: ateapi.GetTagRequest.tag:type_name -> ateapi.ObjectRef + 33, // 105: ateapi.MintActorJwtRequest.actor:type_name -> ateapi.ObjectRef + 143, // 106: ateapi.MintActorJwtResponse.expire_time:type_name -> google.protobuf.Timestamp + 33, // 107: ateapi.MintActorCertificateRequest.actor:type_name -> ateapi.ObjectRef + 93, // 108: ateapi.ListTagsRequest.options:type_name -> ateapi.ListOptions + 31, // 109: ateapi.ListTagsResponse.tags:type_name -> ateapi.Tag + 31, // 110: ateapi.CreateTagRequest.tag:type_name -> ateapi.Tag + 31, // 111: ateapi.UpdateTagRequest.tag:type_name -> ateapi.Tag + 33, // 112: ateapi.DeleteTagRequest.tag:type_name -> ateapi.ObjectRef + 92, // 113: ateapi.DeleteTagRequest.options:type_name -> ateapi.DeleteOptions + 33, // 114: ateapi.ListWorkerActorAssignmentsRequest.worker:type_name -> ateapi.ObjectRef + 93, // 115: ateapi.ListWorkerActorAssignmentsRequest.options:type_name -> ateapi.ListOptions + 116, // 116: ateapi.ListWorkerActorAssignmentsResponse.actor_assignments:type_name -> ateapi.ActorAssignment + 93, // 117: ateapi.ListWorkersRequest.options:type_name -> ateapi.ListOptions + 110, // 118: ateapi.ListWorkersResponse.workers:type_name -> ateapi.Worker + 33, // 119: ateapi.GetWorkerRequest.worker:type_name -> ateapi.ObjectRef + 110, // 120: ateapi.CreateWorkerRequest.worker:type_name -> ateapi.Worker + 110, // 121: ateapi.UpdateWorkerRequest.worker:type_name -> ateapi.Worker + 33, // 122: ateapi.DeleteWorkerRequest.worker:type_name -> ateapi.ObjectRef + 92, // 123: ateapi.DeleteWorkerRequest.options:type_name -> ateapi.DeleteOptions + 33, // 124: ateapi.DrainWorkerRequest.worker:type_name -> ateapi.ObjectRef + 110, // 125: ateapi.DrainWorkerResponse.worker:type_name -> ateapi.Worker + 33, // 126: ateapi.MarkWorkerAvailableRequest.worker:type_name -> ateapi.ObjectRef + 110, // 127: ateapi.MarkWorkerAvailableResponse.worker:type_name -> ateapi.Worker + 33, // 128: ateapi.MarkWorkerUnavailableRequest.worker:type_name -> ateapi.ObjectRef + 110, // 129: ateapi.MarkWorkerUnavailableResponse.worker:type_name -> ateapi.Worker + 93, // 130: ateapi.ListActorsRequest.options:type_name -> ateapi.ListOptions + 16, // 131: ateapi.ListActorsResponse.actors:type_name -> ateapi.Actor + 14, // 132: ateapi.Worker.metadata:type_name -> ateapi.ResourceMetadata + 142, // 133: ateapi.Worker.labels:type_name -> ateapi.Worker.LabelsEntry + 111, // 134: ateapi.Worker.status:type_name -> ateapi.WorkerStatus + 8, // 135: ateapi.WorkerStatus.state:type_name -> ateapi.WorkerState + 115, // 136: ateapi.WorkerStatus.capacity:type_name -> ateapi.WorkerResources + 115, // 137: ateapi.WorkerStatus.allocated:type_name -> ateapi.WorkerResources + 112, // 138: ateapi.WorkerStatus.default_runtime:type_name -> ateapi.SandboxRuntime + 112, // 139: ateapi.WorkerStatus.restorable_runtimes:type_name -> ateapi.SandboxRuntime + 113, // 140: ateapi.SandboxRuntime.version:type_name -> ateapi.VersionedSandboxCompat + 114, // 141: ateapi.VersionedSandboxCompat.attributes:type_name -> ateapi.AttributeEntry + 35, // 142: ateapi.WorkerResources.resources:type_name -> ateapi.Resources + 14, // 143: ateapi.ActorAssignment.metadata:type_name -> ateapi.ResourceMetadata + 33, // 144: ateapi.ActorAssignment.actor:type_name -> ateapi.ObjectRef + 35, // 145: ateapi.ActorAssignment.resources:type_name -> ateapi.Resources + 33, // 146: ateapi.RegisterWorkerRequest.worker:type_name -> ateapi.ObjectRef + 115, // 147: ateapi.RegisterWorkerRequest.capacity:type_name -> ateapi.WorkerResources + 112, // 148: ateapi.RegisterWorkerRequest.default_runtime:type_name -> ateapi.SandboxRuntime + 112, // 149: ateapi.RegisterWorkerRequest.restorable_runtimes:type_name -> ateapi.SandboxRuntime + 110, // 150: ateapi.RegisterWorkerResponse.worker:type_name -> ateapi.Worker + 33, // 151: ateapi.MintWorkerActorCertificateRequest.actor:type_name -> ateapi.ObjectRef + 33, // 152: ateapi.MintWorkerActorCertificateRequest.worker:type_name -> ateapi.ObjectRef + 33, // 153: ateapi.MintWorkerActorJwtRequest.actor:type_name -> ateapi.ObjectRef + 143, // 154: ateapi.MintWorkerActorJwtResponse.expire_time:type_name -> google.protobuf.Timestamp + 33, // 155: ateapi.RequestActorSuspendRequest.worker:type_name -> ateapi.ObjectRef + 33, // 156: ateapi.RequestActorSuspendRequest.actor:type_name -> ateapi.ObjectRef + 16, // 157: ateapi.RequestActorSuspendResponse.actor:type_name -> ateapi.Actor + 33, // 158: ateapi.ReportActorSnapshotStateRequest.actor:type_name -> ateapi.ObjectRef + 127, // 159: ateapi.ReportActorSnapshotStateRequest.state:type_name -> ateapi.ActorSnapshotState + 128, // 160: ateapi.ActorSnapshotState.latest_local:type_name -> ateapi.SnapshotReport + 128, // 161: ateapi.ActorSnapshotState.uploading:type_name -> ateapi.SnapshotReport + 128, // 162: ateapi.ActorSnapshotState.latest_durable:type_name -> ateapi.SnapshotReport + 143, // 163: ateapi.ActorSnapshotState.upload_due_time:type_name -> google.protobuf.Timestamp + 2, // 164: ateapi.SnapshotReport.fidelity:type_name -> ateapi.SnapshotFidelity + 10, // 165: ateapi.SnapshotReport.object:type_name -> ateapi.ObjectSnapshot + 143, // 166: ateapi.SnapshotReport.local_commit_time:type_name -> google.protobuf.Timestamp + 143, // 167: ateapi.SnapshotReport.durable_complete_time:type_name -> google.protobuf.Timestamp + 14, // 168: ateapi.AccessPolicy.metadata:type_name -> ateapi.ResourceMetadata + 130, // 169: ateapi.AccessPolicy.bindings:type_name -> ateapi.Binding + 129, // 170: ateapi.CreateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 129, // 171: ateapi.UpdateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 33, // 172: ateapi.GetAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 33, // 173: ateapi.CreateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 129, // 174: ateapi.CreateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 33, // 175: ateapi.UpdateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 129, // 176: ateapi.UpdateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 33, // 177: ateapi.DeleteAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 92, // 178: ateapi.DeleteAtespaceAccessPolicyRequest.options:type_name -> ateapi.DeleteOptions + 9, // 179: ateapi.ResourceAnnotation.scope:type_name -> ateapi.Scope + 144, // 180: ateapi.resource:extendee -> google.protobuf.MessageOptions + 145, // 181: ateapi.method:extendee -> google.protobuf.MethodOptions + 138, // 182: ateapi.resource:type_name -> ateapi.ResourceAnnotation + 139, // 183: ateapi.method:type_name -> ateapi.MethodAnnotation + 67, // 184: ateapi.Control.GetActor:input_type -> ateapi.GetActorRequest + 68, // 185: ateapi.Control.CreateActor:input_type -> ateapi.CreateActorRequest + 69, // 186: ateapi.Control.UpdateActor:input_type -> ateapi.UpdateActorRequest + 70, // 187: ateapi.Control.SuspendActor:input_type -> ateapi.SuspendActorRequest + 73, // 188: ateapi.Control.ResumeActor:input_type -> ateapi.ResumeActorRequest + 75, // 189: ateapi.Control.RevertActor:input_type -> ateapi.RevertActorRequest + 77, // 190: ateapi.Control.DeleteActor:input_type -> ateapi.DeleteActorRequest + 78, // 191: ateapi.Control.GetActorEgressPolicy:input_type -> ateapi.GetActorEgressPolicyRequest + 79, // 192: ateapi.Control.CreateActorEgressPolicy:input_type -> ateapi.CreateActorEgressPolicyRequest + 80, // 193: ateapi.Control.UpdateActorEgressPolicy:input_type -> ateapi.UpdateActorEgressPolicyRequest + 81, // 194: ateapi.Control.DeleteActorEgressPolicy:input_type -> ateapi.DeleteActorEgressPolicyRequest + 83, // 195: ateapi.Control.MintActorJwt:input_type -> ateapi.MintActorJwtRequest + 85, // 196: ateapi.Control.MintActorCertificate:input_type -> ateapi.MintActorCertificateRequest + 89, // 197: ateapi.Control.CreateTag:input_type -> ateapi.CreateTagRequest + 82, // 198: ateapi.Control.GetTag:input_type -> ateapi.GetTagRequest + 87, // 199: ateapi.Control.ListTags:input_type -> ateapi.ListTagsRequest + 90, // 200: ateapi.Control.UpdateTag:input_type -> ateapi.UpdateTagRequest + 91, // 201: ateapi.Control.DeleteTag:input_type -> ateapi.DeleteTagRequest + 96, // 202: ateapi.Control.ListWorkers:input_type -> ateapi.ListWorkersRequest + 98, // 203: ateapi.Control.GetWorker:input_type -> ateapi.GetWorkerRequest + 99, // 204: ateapi.Control.CreateWorker:input_type -> ateapi.CreateWorkerRequest + 100, // 205: ateapi.Control.UpdateWorker:input_type -> ateapi.UpdateWorkerRequest + 101, // 206: ateapi.Control.DeleteWorker:input_type -> ateapi.DeleteWorkerRequest + 102, // 207: ateapi.Control.DrainWorker:input_type -> ateapi.DrainWorkerRequest + 104, // 208: ateapi.Control.MarkWorkerAvailable:input_type -> ateapi.MarkWorkerAvailableRequest + 106, // 209: ateapi.Control.MarkWorkerUnavailable:input_type -> ateapi.MarkWorkerUnavailableRequest + 94, // 210: ateapi.Control.ListWorkerActorAssignments:input_type -> ateapi.ListWorkerActorAssignmentsRequest + 108, // 211: ateapi.Control.ListActors:input_type -> ateapi.ListActorsRequest + 57, // 212: ateapi.Control.CreateAtespace:input_type -> ateapi.CreateAtespaceRequest + 58, // 213: ateapi.Control.GetAtespace:input_type -> ateapi.GetAtespaceRequest + 59, // 214: ateapi.Control.ListAtespaces:input_type -> ateapi.ListAtespacesRequest + 61, // 215: ateapi.Control.DeleteAtespace:input_type -> ateapi.DeleteAtespaceRequest + 62, // 216: ateapi.Control.CreateActorTemplate:input_type -> ateapi.CreateActorTemplateRequest + 63, // 217: ateapi.Control.GetActorTemplate:input_type -> ateapi.GetActorTemplateRequest + 64, // 218: ateapi.Control.ListActorTemplates:input_type -> ateapi.ListActorTemplatesRequest + 66, // 219: ateapi.Control.DeleteActorTemplate:input_type -> ateapi.DeleteActorTemplateRequest + 131, // 220: ateapi.Control.GetGlobalAccessPolicy:input_type -> ateapi.GetGlobalAccessPolicyRequest + 132, // 221: ateapi.Control.CreateGlobalAccessPolicy:input_type -> ateapi.CreateGlobalAccessPolicyRequest + 133, // 222: ateapi.Control.UpdateGlobalAccessPolicy:input_type -> ateapi.UpdateGlobalAccessPolicyRequest + 134, // 223: ateapi.Control.GetAtespaceAccessPolicy:input_type -> ateapi.GetAtespaceAccessPolicyRequest + 135, // 224: ateapi.Control.CreateAtespaceAccessPolicy:input_type -> ateapi.CreateAtespaceAccessPolicyRequest + 136, // 225: ateapi.Control.UpdateAtespaceAccessPolicy:input_type -> ateapi.UpdateAtespaceAccessPolicyRequest + 137, // 226: ateapi.Control.DeleteAtespaceAccessPolicy:input_type -> ateapi.DeleteAtespaceAccessPolicyRequest + 117, // 227: ateapi.WorkerService.RegisterWorker:input_type -> ateapi.RegisterWorkerRequest + 119, // 228: ateapi.WorkerService.MintWorkerActorCertificate:input_type -> ateapi.MintWorkerActorCertificateRequest + 121, // 229: ateapi.WorkerService.MintWorkerActorJwt:input_type -> ateapi.MintWorkerActorJwtRequest + 123, // 230: ateapi.WorkerService.RequestActorSuspend:input_type -> ateapi.RequestActorSuspendRequest + 125, // 231: ateapi.WorkerService.ReportActorSnapshotState:input_type -> ateapi.ReportActorSnapshotStateRequest + 16, // 232: ateapi.Control.GetActor:output_type -> ateapi.Actor + 16, // 233: ateapi.Control.CreateActor:output_type -> ateapi.Actor + 16, // 234: ateapi.Control.UpdateActor:output_type -> ateapi.Actor + 71, // 235: ateapi.Control.SuspendActor:output_type -> ateapi.SuspendActorResponse + 74, // 236: ateapi.Control.ResumeActor:output_type -> ateapi.ResumeActorResponse + 76, // 237: ateapi.Control.RevertActor:output_type -> ateapi.RevertActorResponse + 16, // 238: ateapi.Control.DeleteActor:output_type -> ateapi.Actor + 17, // 239: ateapi.Control.GetActorEgressPolicy:output_type -> ateapi.EgressPolicy + 17, // 240: ateapi.Control.CreateActorEgressPolicy:output_type -> ateapi.EgressPolicy + 17, // 241: ateapi.Control.UpdateActorEgressPolicy:output_type -> ateapi.EgressPolicy + 17, // 242: ateapi.Control.DeleteActorEgressPolicy:output_type -> ateapi.EgressPolicy + 84, // 243: ateapi.Control.MintActorJwt:output_type -> ateapi.MintActorJwtResponse + 86, // 244: ateapi.Control.MintActorCertificate:output_type -> ateapi.MintActorCertificateResponse + 31, // 245: ateapi.Control.CreateTag:output_type -> ateapi.Tag + 31, // 246: ateapi.Control.GetTag:output_type -> ateapi.Tag + 88, // 247: ateapi.Control.ListTags:output_type -> ateapi.ListTagsResponse + 31, // 248: ateapi.Control.UpdateTag:output_type -> ateapi.Tag + 31, // 249: ateapi.Control.DeleteTag:output_type -> ateapi.Tag + 97, // 250: ateapi.Control.ListWorkers:output_type -> ateapi.ListWorkersResponse + 110, // 251: ateapi.Control.GetWorker:output_type -> ateapi.Worker + 110, // 252: ateapi.Control.CreateWorker:output_type -> ateapi.Worker + 110, // 253: ateapi.Control.UpdateWorker:output_type -> ateapi.Worker + 110, // 254: ateapi.Control.DeleteWorker:output_type -> ateapi.Worker + 103, // 255: ateapi.Control.DrainWorker:output_type -> ateapi.DrainWorkerResponse + 105, // 256: ateapi.Control.MarkWorkerAvailable:output_type -> ateapi.MarkWorkerAvailableResponse + 107, // 257: ateapi.Control.MarkWorkerUnavailable:output_type -> ateapi.MarkWorkerUnavailableResponse + 95, // 258: ateapi.Control.ListWorkerActorAssignments:output_type -> ateapi.ListWorkerActorAssignmentsResponse + 109, // 259: ateapi.Control.ListActors:output_type -> ateapi.ListActorsResponse + 32, // 260: ateapi.Control.CreateAtespace:output_type -> ateapi.Atespace + 32, // 261: ateapi.Control.GetAtespace:output_type -> ateapi.Atespace + 60, // 262: ateapi.Control.ListAtespaces:output_type -> ateapi.ListAtespacesResponse + 32, // 263: ateapi.Control.DeleteAtespace:output_type -> ateapi.Atespace + 34, // 264: ateapi.Control.CreateActorTemplate:output_type -> ateapi.ActorTemplate + 34, // 265: ateapi.Control.GetActorTemplate:output_type -> ateapi.ActorTemplate + 65, // 266: ateapi.Control.ListActorTemplates:output_type -> ateapi.ListActorTemplatesResponse + 34, // 267: ateapi.Control.DeleteActorTemplate:output_type -> ateapi.ActorTemplate + 129, // 268: ateapi.Control.GetGlobalAccessPolicy:output_type -> ateapi.AccessPolicy + 129, // 269: ateapi.Control.CreateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy + 129, // 270: ateapi.Control.UpdateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy + 129, // 271: ateapi.Control.GetAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 129, // 272: ateapi.Control.CreateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 129, // 273: ateapi.Control.UpdateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 129, // 274: ateapi.Control.DeleteAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 118, // 275: ateapi.WorkerService.RegisterWorker:output_type -> ateapi.RegisterWorkerResponse + 120, // 276: ateapi.WorkerService.MintWorkerActorCertificate:output_type -> ateapi.MintWorkerActorCertificateResponse + 122, // 277: ateapi.WorkerService.MintWorkerActorJwt:output_type -> ateapi.MintWorkerActorJwtResponse + 124, // 278: ateapi.WorkerService.RequestActorSuspend:output_type -> ateapi.RequestActorSuspendResponse + 126, // 279: ateapi.WorkerService.ReportActorSnapshotState:output_type -> ateapi.ReportActorSnapshotStateResponse + 232, // [232:280] is the sub-list for method output_type + 184, // [184:232] is the sub-list for method input_type + 182, // [182:184] is the sub-list for extension type_name + 180, // [180:182] is the sub-list for extension extendee + 0, // [0:180] is the sub-list for field type_name } func init() { file_ateapi_proto_init() } @@ -10487,8 +10441,8 @@ func file_ateapi_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_ateapi_proto_rawDesc), len(file_ateapi_proto_rawDesc)), - NumEnums: 9, - NumMessages: 135, + NumEnums: 10, + NumMessages: 133, NumExtensions: 2, NumServices: 2, }, diff --git a/pkg/proto/ateapipb/ateapi.proto b/pkg/proto/ateapipb/ateapi.proto index 0ba0d4a3be..8eefdd647e 100644 --- a/pkg/proto/ateapipb/ateapi.proto +++ b/pkg/proto/ateapipb/ateapi.proto @@ -39,26 +39,20 @@ service Control { } // Suspend a given actor to a new snapshot. A running actor is checkpointed - // on its worker; a paused actor's node-local snapshot is uploaded, narrowed - // to the template's commit scope where required (Full capture, Data commit). + // on its worker. A SUSPENDED actor is returned as it is. rpc SuspendActor(SuspendActorRequest) returns (SuspendActorResponse) { option (method).resource = "Actor"; } - // Pause a given actor: checkpoint it to its worker's node and leave it - // SUSPENDED with that node-local snapshot, pinned to the node. A paused - // actor is told from a suspended one by its snapshots, not its state. - rpc PauseActor(PauseActorRequest) returns (PauseActorResponse) { - option (method).resource = "Actor"; - } - // Resume an actor from its latest snapshot. rpc ResumeActor(ResumeActorRequest) returns (ResumeActorResponse) { option (method).resource = "Actor"; } // Revert an actor to SUSPENDED state at its durable snapshot. - // Only crashed, running or paused actors can be reverted. + // Only crashed or running actors, or suspended actors with a node-local + // snapshot, can be reverted. A suspended actor without a node-local snapshot + // is already at its durable snapshot, so there is nothing to revert. rpc RevertActor(RevertActorRequest) returns (RevertActorResponse) { option (method).resource = "Actor"; } @@ -231,7 +225,7 @@ service Control { } // Delete an ActorTemplate together with its golden actor and golden - // tag in the reserved ate-golden atespace. + // tag in the template's atespace. rpc DeleteActorTemplate(DeleteActorTemplateRequest) returns (ActorTemplate) { option (method).resource = "ActorTemplate"; } @@ -369,7 +363,9 @@ message Snapshot { string locality = 6; // durable_snapshot holds the durable storage representation of this snapshot. - // Unset when the snapshot only exists locally. + // PENDING while the snapshot is uploaded, including a pause's snapshot that + // so far exists only on the node named by locality. COMPLETED once the + // upload is recorded. // // +k8s:optional SnapshotStorage durable_snapshot = 7; @@ -389,8 +385,7 @@ enum SnapshotFidelity { // from the OCI image with the volumes restored. SNAPSHOT_FIDELITY_VOLUMES = 1; // Volumes plus the root filesystem changes made since boot. The actor - // cold-boots on resume with its rootfs writes back in place. Served by the - // micro-VM sandbox class only. + // cold-boots on resume with its rootfs writes back in place. SNAPSHOT_FIDELITY_ROOTFS = 2; // Volumes, root filesystem changes, and process memory. The actor resumes // where it left off. @@ -549,7 +544,8 @@ message Actor { // worker_selector is the per-actor placement constraint. The scheduler // evaluates the AND of this selector and the template's workerSelector to - // find eligible pools. + // find eligible pools. It narrows the template's selection; pinning a + // sandbox class belongs on the template's worker_selector. // Changes take effect on the next ResumeActor call. // // +k8s:optional @@ -1125,7 +1121,13 @@ message ActorTemplate { ResourceMetadata metadata = 1; // worker_selector restricts which worker pools actors from this template - // may use. + // may use, by matching the pools' labels. Unset, every pool is eligible. + // + // The pool decides the sandbox class an actor runs in, so this selector is + // also how a template pins one: if the cluster runs pools of more than one + // sandbox class and the workload needs a particular one, select only pools + // of that class. An actor booting with no snapshot and no selector may land + // on a pool of any class, and keeps that class from its first checkpoint on. // // +k8s:optional Selector worker_selector = 2; @@ -1145,11 +1147,6 @@ message ActorTemplate { // +k8s:required SnapshotConfig snapshot_config = 5; - // sandbox_config selects the sandbox runtime this version's actors run on. - // - // +k8s:required - SandboxConfig sandbox_config = 6; - // resources is the actor's compute size. Both cpu and memory limits are // required: the scheduler books them against worker capacity, and a // micro-VM is booted at that size. @@ -1214,24 +1211,6 @@ message ActorTemplateStatus { GoldenSnapshotStatus golden_snapshot_status = 1; } -// SandboxConfig selects the sandbox runtime for an ActorTemplate. -message SandboxConfig { - // sandbox_class selects the sandbox runtime family: "gvisor" or "microvm". - // Snapshots are not portable across classes. - // - // +k8s:required - // +k8s:maxLength=63 - // +k8s:customValidation # must be "gvisor" or "microvm" - string sandbox_class = 1; - - // config_name names the cluster-scoped SandboxConfig Kubernetes object - // supplying the sandbox binaries. Required; must match sandbox_class. - // - // +k8s:required - // +k8s:format=k8s-long-name - string config_name = 2; -} - // SnapshotConfig selects what actor snapshots capture and where they are // stored. message SnapshotConfig { @@ -1249,9 +1228,7 @@ message SnapshotConfig { // more than this level, and it may package less when a layer cannot be // captured or is too expensive to capture at the time, for example memory // under host pressure. A process resumed in place on its worker keeps its - // memory regardless of this setting. ROOTFS requires - // sandbox_config.sandbox_class "microvm". Defaults to MEMORY - // when unset. + // memory regardless of this setting. Defaults to MEMORY when unset. // // +k8s:required // +k8s:minimum=1 @@ -1676,7 +1653,7 @@ message CreateActorTemplateRequest { // version, timestamps) is ignored, as are the status fields. // // +k8s:required - // +k8s:customValidation # volume_mounts must reference declared volumes; ROOTFS fidelity requires the micro-VM sandbox class + // +k8s:customValidation # volume_mounts must reference declared volumes ActorTemplate actor_template = 1; } @@ -1754,20 +1731,43 @@ message SuspendActorRequest { // +k8s:required // +k8s:subfield(atespace)=+k8s:required ObjectRef actor = 1; + + // options controls how SuspendActor persists the actor's snapshot. + // + // +k8s:optional + SuspendOptions options = 2; } message SuspendActorResponse { Actor actor = 1; } -message PauseActorRequest { - // +k8s:required - // +k8s:subfield(atespace)=+k8s:required - ObjectRef actor = 1; +// SuspendOptions tunes how SuspendActor persists the actor's snapshot. +message SuspendOptions { + // mode selects when SuspendActor returns relative to the snapshot reaching + // durable storage. If unspecified, defaults to DURABILITY_MODE_ASYNCHRONOUS. + // + // +k8s:optional + // +k8s:minimum=1 + // +k8s:maximum=2 # keep this in sync with the DurabilityMode enum + DurabilityMode mode = 1; } -message PauseActorResponse { - Actor actor = 1; +// DurabilityMode selects when a suspend is acknowledged relative to its +// snapshot being stored durably. +enum DurabilityMode { + DURABILITY_MODE_UNSPECIFIED = 0; + + // Return only after the snapshot is complete in durable storage. Applies + // only when the call checkpoints a running actor: an actor that is already + // SUSPENDED is returned with the snapshot it holds, which may still be on + // its node. + DURABILITY_MODE_SYNCHRONOUS = 1; + + // Return once the snapshot is captured on the node. The upload to durable + // storage continues in the background. + DURABILITY_MODE_ASYNCHRONOUS = 2; + // Keep this in sync with SuspendOptions.mode's maximum. } message ResumeActorRequest { @@ -2435,7 +2435,7 @@ message VersionedSandboxCompat { string schema_version = 1; // Key-value compatibility attributes defined by this schema version, covering - // both runtime assets (e.g. "sandbox_runtime_hash") and effective guest + // both runtime assets (e.g. "sandbox_bundle_hash") and effective guest // hardware features after masking (e.g. "architecture", "cpu_features"). // Matched against a snapshot under the same schema_version: every attribute // on the snapshot must be present on the worker with an equal value (except @@ -2456,7 +2456,7 @@ message VersionedSandboxCompat { // AttributeEntry is a single key-value compatibility attribute. message AttributeEntry { - // Attribute name (e.g. "architecture", "cpu_features", "sandbox_runtime_hash"). + // Attribute name (e.g. "architecture", "cpu_features", "sandbox_bundle_hash"). // // +k8s:required // +k8s:maxLength=128 @@ -2729,6 +2729,9 @@ message MintWorkerActorJwtResponse { google.protobuf.Timestamp expire_time = 2; } +// TODO: add a durability mode, carried through atelet's +// RequestActorSuspendRequest, so the Worker can choose how the snapshot is +// made durable. Until then the suspend uses SuspendActor's default mode. message RequestActorSuspendRequest { // The Worker making the request. atespace is always empty; Workers are // global-scoped. atelet fills this in from its caller's certificate, so it diff --git a/pkg/proto/ateapipb/ateapi_grpc.pb.go b/pkg/proto/ateapipb/ateapi_grpc.pb.go index ea0042a580..a40dd7f2f2 100644 --- a/pkg/proto/ateapipb/ateapi_grpc.pb.go +++ b/pkg/proto/ateapipb/ateapi_grpc.pb.go @@ -37,7 +37,6 @@ const ( Control_CreateActor_FullMethodName = "/ateapi.Control/CreateActor" Control_UpdateActor_FullMethodName = "/ateapi.Control/UpdateActor" Control_SuspendActor_FullMethodName = "/ateapi.Control/SuspendActor" - Control_PauseActor_FullMethodName = "/ateapi.Control/PauseActor" Control_ResumeActor_FullMethodName = "/ateapi.Control/ResumeActor" Control_RevertActor_FullMethodName = "/ateapi.Control/RevertActor" Control_DeleteActor_FullMethodName = "/ateapi.Control/DeleteActor" @@ -92,17 +91,14 @@ type ControlClient interface { // Update mutable fields on an existing Actor. UpdateActor(ctx context.Context, in *UpdateActorRequest, opts ...grpc.CallOption) (*Actor, error) // Suspend a given actor to a new snapshot. A running actor is checkpointed - // on its worker; a paused actor's node-local snapshot is uploaded, narrowed - // to the template's commit scope where required (Full capture, Data commit). + // on its worker. A SUSPENDED actor is returned as it is. SuspendActor(ctx context.Context, in *SuspendActorRequest, opts ...grpc.CallOption) (*SuspendActorResponse, error) - // Pause a given actor: checkpoint it to its worker's node and leave it - // SUSPENDED with that node-local snapshot, pinned to the node. A paused - // actor is told from a suspended one by its snapshots, not its state. - PauseActor(ctx context.Context, in *PauseActorRequest, opts ...grpc.CallOption) (*PauseActorResponse, error) // Resume an actor from its latest snapshot. ResumeActor(ctx context.Context, in *ResumeActorRequest, opts ...grpc.CallOption) (*ResumeActorResponse, error) // Revert an actor to SUSPENDED state at its durable snapshot. - // Only crashed, running or paused actors can be reverted. + // Only crashed or running actors, or suspended actors with a node-local + // snapshot, can be reverted. A suspended actor without a node-local snapshot + // is already at its durable snapshot, so there is nothing to revert. RevertActor(ctx context.Context, in *RevertActorRequest, opts ...grpc.CallOption) (*RevertActorResponse, error) // Delete an actor. Only suspended or crashed actors can be deleted unless // any_state is set. @@ -185,7 +181,7 @@ type ControlClient interface { GetActorTemplate(ctx context.Context, in *GetActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error) ListActorTemplates(ctx context.Context, in *ListActorTemplatesRequest, opts ...grpc.CallOption) (*ListActorTemplatesResponse, error) // Delete an ActorTemplate together with its golden actor and golden - // tag in the reserved ate-golden atespace. + // tag in the template's atespace. DeleteActorTemplate(ctx context.Context, in *DeleteActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error) // Get the deployment-wide global access policy singleton. Returns NotFound // until the policy has been created. @@ -256,16 +252,6 @@ func (c *controlClient) SuspendActor(ctx context.Context, in *SuspendActorReques return out, nil } -func (c *controlClient) PauseActor(ctx context.Context, in *PauseActorRequest, opts ...grpc.CallOption) (*PauseActorResponse, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - out := new(PauseActorResponse) - err := c.cc.Invoke(ctx, Control_PauseActor_FullMethodName, in, out, cOpts...) - if err != nil { - return nil, err - } - return out, nil -} - func (c *controlClient) ResumeActor(ctx context.Context, in *ResumeActorRequest, opts ...grpc.CallOption) (*ResumeActorResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(ResumeActorResponse) @@ -669,17 +655,14 @@ type ControlServer interface { // Update mutable fields on an existing Actor. UpdateActor(context.Context, *UpdateActorRequest) (*Actor, error) // Suspend a given actor to a new snapshot. A running actor is checkpointed - // on its worker; a paused actor's node-local snapshot is uploaded, narrowed - // to the template's commit scope where required (Full capture, Data commit). + // on its worker. A SUSPENDED actor is returned as it is. SuspendActor(context.Context, *SuspendActorRequest) (*SuspendActorResponse, error) - // Pause a given actor: checkpoint it to its worker's node and leave it - // SUSPENDED with that node-local snapshot, pinned to the node. A paused - // actor is told from a suspended one by its snapshots, not its state. - PauseActor(context.Context, *PauseActorRequest) (*PauseActorResponse, error) // Resume an actor from its latest snapshot. ResumeActor(context.Context, *ResumeActorRequest) (*ResumeActorResponse, error) // Revert an actor to SUSPENDED state at its durable snapshot. - // Only crashed, running or paused actors can be reverted. + // Only crashed or running actors, or suspended actors with a node-local + // snapshot, can be reverted. A suspended actor without a node-local snapshot + // is already at its durable snapshot, so there is nothing to revert. RevertActor(context.Context, *RevertActorRequest) (*RevertActorResponse, error) // Delete an actor. Only suspended or crashed actors can be deleted unless // any_state is set. @@ -762,7 +745,7 @@ type ControlServer interface { GetActorTemplate(context.Context, *GetActorTemplateRequest) (*ActorTemplate, error) ListActorTemplates(context.Context, *ListActorTemplatesRequest) (*ListActorTemplatesResponse, error) // Delete an ActorTemplate together with its golden actor and golden - // tag in the reserved ate-golden atespace. + // tag in the template's atespace. DeleteActorTemplate(context.Context, *DeleteActorTemplateRequest) (*ActorTemplate, error) // Get the deployment-wide global access policy singleton. Returns NotFound // until the policy has been created. @@ -805,9 +788,6 @@ func (UnimplementedControlServer) UpdateActor(context.Context, *UpdateActorReque func (UnimplementedControlServer) SuspendActor(context.Context, *SuspendActorRequest) (*SuspendActorResponse, error) { return nil, status.Error(codes.Unimplemented, "method SuspendActor not implemented") } -func (UnimplementedControlServer) PauseActor(context.Context, *PauseActorRequest) (*PauseActorResponse, error) { - return nil, status.Error(codes.Unimplemented, "method PauseActor not implemented") -} func (UnimplementedControlServer) ResumeActor(context.Context, *ResumeActorRequest) (*ResumeActorResponse, error) { return nil, status.Error(codes.Unimplemented, "method ResumeActor not implemented") } @@ -1018,24 +998,6 @@ func _Control_SuspendActor_Handler(srv interface{}, ctx context.Context, dec fun return interceptor(ctx, in, info, handler) } -func _Control_PauseActor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(PauseActorRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(ControlServer).PauseActor(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: Control_PauseActor_FullMethodName, - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ControlServer).PauseActor(ctx, req.(*PauseActorRequest)) - } - return interceptor(ctx, in, info, handler) -} - func _Control_ResumeActor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ResumeActorRequest) if err := dec(in); err != nil { @@ -1761,10 +1723,6 @@ var Control_ServiceDesc = grpc.ServiceDesc{ MethodName: "SuspendActor", Handler: _Control_SuspendActor_Handler, }, - { - MethodName: "PauseActor", - Handler: _Control_PauseActor_Handler, - }, { MethodName: "ResumeActor", Handler: _Control_ResumeActor_Handler, diff --git a/pkg/sandboxclass/sandboxclass.go b/pkg/sandboxclass/sandboxclass.go index 6be60e944e..22a57992de 100644 --- a/pkg/sandboxclass/sandboxclass.go +++ b/pkg/sandboxclass/sandboxclass.go @@ -13,7 +13,7 @@ // limitations under the License. // Package sandboxclass names the sandbox runtime families: the values a -// WorkerPool, a SandboxConfig, and an ActorTemplate can select. It imports +// WorkerPool and a SandboxConfig can select. It imports // nothing from Kubernetes, so any package can name a class without pulling in // the ate.dev/v1alpha1 API types. package sandboxclass diff --git a/tools/apitool/exemptions.json b/tools/apitool/exemptions.json index 1664fa468b..d1283b1cec 100644 --- a/tools/apitool/exemptions.json +++ b/tools/apitool/exemptions.json @@ -269,26 +269,6 @@ "subject": "ateapi.ListWorkersResponse", "message": "message has no doc comment" }, - { - "rule": "documented", - "subject": "ateapi.PauseActorRequest", - "message": "message has no doc comment" - }, - { - "rule": "documented", - "subject": "ateapi.PauseActorRequest.actor", - "message": "field has no doc comment" - }, - { - "rule": "documented", - "subject": "ateapi.PauseActorResponse", - "message": "message has no doc comment" - }, - { - "rule": "documented", - "subject": "ateapi.PauseActorResponse.actor", - "message": "field has no doc comment" - }, { "rule": "documented", "subject": "ateapi.Resources", diff --git a/tools/junittool/junit.go b/tools/junittool/junit.go index 135bb0fdd4..a5390f04fe 100644 --- a/tools/junittool/junit.go +++ b/tools/junittool/junit.go @@ -64,10 +64,10 @@ func parse(path string) (totals, error) { return out, nil } -// expand resolves each argument to JUnit files. A directory contributes the -// *.xml inside it; anything else passes through literally, so a missing file -// reaches parse() and is reported. -func expand(args []string) ([]string, error) { +// expand resolves each argument to files. A directory contributes the files +// inside it with extension ext; anything else passes through literally, so a +// missing file is reported by whatever reads it. +func expand(args []string, ext string) ([]string, error) { var out []string for _, a := range args { info, err := os.Stat(a) @@ -75,7 +75,7 @@ func expand(args []string) ([]string, error) { out = append(out, a) continue } - matches, err := filepath.Glob(filepath.Join(a, "*.xml")) + matches, err := filepath.Glob(filepath.Join(a, "*"+ext)) if err != nil { return nil, fmt.Errorf("expanding %s: %w", a, err) } diff --git a/tools/junittool/junit_test.go b/tools/junittool/junit_test.go index 0c915aa21d..c4e14a6cc9 100644 --- a/tools/junittool/junit_test.go +++ b/tools/junittool/junit_test.go @@ -95,7 +95,7 @@ func TestExpand(t *testing.T) { } t.Run("directory expands to its xml, sorted", func(t *testing.T) { - got, err := expand([]string{dir}) + got, err := expand([]string{dir}, ".xml") if err != nil { t.Fatalf("expand() error = %v", err) } @@ -105,7 +105,7 @@ func TestExpand(t *testing.T) { }) t.Run("non-xml files are not picked up", func(t *testing.T) { - got, _ := expand([]string{dir}) + got, _ := expand([]string{dir}, ".xml") for _, p := range got { if filepath.Ext(p) != ".xml" { t.Errorf("expand() included %s, want only .xml", p) @@ -114,7 +114,7 @@ func TestExpand(t *testing.T) { }) t.Run("empty directory contributes nothing", func(t *testing.T) { - got, err := expand([]string{empty}) + got, err := expand([]string{empty}, ".xml") if err != nil || len(got) != 0 { t.Errorf("expand(empty dir) = %v, %v; want no paths and no error", got, err) } @@ -123,7 +123,7 @@ func TestExpand(t *testing.T) { // Passed through, not dropped, so evaluate() reports it missing. t.Run("missing path passes through literally", func(t *testing.T) { missing := filepath.Join(dir, "absent.xml") - got, err := expand([]string{missing}) + got, err := expand([]string{missing}, ".xml") if err != nil || len(got) != 1 || got[0] != missing { t.Errorf("expand(missing) = %v, %v; want [%s]", got, err, missing) } diff --git a/tools/junittool/main.go b/tools/junittool/main.go index 770c807b2f..dfb67021ed 100644 --- a/tools/junittool/main.go +++ b/tools/junittool/main.go @@ -17,6 +17,7 @@ // // junittool register record a file this run will write // junittool verify -manifest require every recorded file to hold tests +// junittool report summarize failures from the go test -json files // // An exit code cannot distinguish "all tests passed" from "no test ran". // `go test ./...` passes the e2e suites with no cluster, and a package filter @@ -38,6 +39,7 @@ import ( const usage = `usage: junittool register [-manifest FILE] [-reject-duplicate] junittool verify (-manifest FILE | ...) [-min N] + junittool report [-summary FILE] [-annotate] [-root DIR] ... ` func main() { @@ -50,6 +52,8 @@ func main() { os.Exit(runRegister(os.Args[2:])) case "verify": os.Exit(runVerify(os.Args[2:])) + case "report": + os.Exit(runReport(os.Args[2:])) case "-h", "--help", "help": fmt.Print(usage) os.Exit(0) @@ -114,7 +118,7 @@ func runVerify(args []string) int { } case fs.NArg() > 0: var err error - if paths, err = expand(fs.Args()); err != nil { + if paths, err = expand(fs.Args(), ".xml"); err != nil { fmt.Fprintf(os.Stderr, "junittool verify: %v\n", err) return 1 } @@ -130,3 +134,51 @@ func runVerify(args []string) int { } return 0 } + +// runReport lists each failed test with its t.Error and t.Fatal output. It +// only reports, the test steps and verify decide whether the job fails. +// +// NOTE: the go test -json files must come from runs without -v, which +// replaces the -test.v=test2json that marks error output. +func runReport(args []string) int { + fs := flag.NewFlagSet("report", flag.ExitOnError) + summary := fs.String("summary", os.Getenv("GITHUB_STEP_SUMMARY"), + "file to append a markdown summary to, defaults to $GITHUB_STEP_SUMMARY") + annotate := fs.Bool("annotate", os.Getenv("GITHUB_ACTIONS") == "true", + "print GitHub error annotations for failures (defaults true in GitHub Actions)") + root := fs.String("root", os.Getenv("GITHUB_WORKSPACE"), + "repository root, for annotation file paths, defaults to $GITHUB_WORKSPACE") + _ = fs.Parse(args) + + if fs.NArg() == 0 { + fmt.Fprint(os.Stderr, usage) + return 2 + } + paths, err := expand(fs.Args(), ".json") + if err != nil { + fmt.Fprintf(os.Stderr, "junittool report: %v\n", err) + return 1 + } + var runs []run + for _, p := range paths { + r := parseRun(p) + r.writeText(os.Stdout) + runs = append(runs, r) + } + if *annotate { + writeAnnotations(os.Stdout, runs, *root) + } + if *summary != "" { + f, err := os.OpenFile(*summary, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + fmt.Fprintf(os.Stderr, "junittool report: %v\n", err) + return 1 + } + defer f.Close() + if err := writeMarkdown(f, runs, summaryBudget); err != nil { + fmt.Fprintf(os.Stderr, "junittool report: failed to write summary: %v\n", err) + return 1 + } + } + return 0 +} diff --git a/tools/junittool/report.go b/tools/junittool/report.go new file mode 100644 index 0000000000..25f46a4e4b --- /dev/null +++ b/tools/junittool/report.go @@ -0,0 +1,571 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "html" + "io" + "os" + "path" + "path/filepath" + "regexp" + "slices" + "sort" + "strconv" + "strings" + "unicode/utf8" +) + +const ( + maxMessageLines = 20 + maxContextLines = 60 + maxStackLines = 200 + maxLineLen = 1000 + // GitHub rejects a step summary over 1MiB + summaryBudget = 900 << 10 + // GitHub drops error annotations past 10 per step + maxAnnotations = 10 +) + +// event is one line of `go test -json`, see `go doc cmd/test2json` +type event struct { + Action string + Package string + Test string + Elapsed float64 + Output string + OutputType string + ImportPath string + FailedBuild string +} + +type testResult struct { + action string // "" if the test never finished + elapsed float64 + output []string // everything but framing + errors []string // from t.Error and t.Fatal +} + +type pkgResult struct { + action string + failedBuild string + output []string // not attributed to a test + all []string // all output in order, test or not + tests map[string]*testResult +} + +// failure is a failed test, or a package that failed with no failed test to +// blame (build failure, timeout, crash) +type failure struct { + pkg string + test string + label string // in place of test for a package failure + elapsed float64 + message []string + context []string +} + +func (f failure) title() string { + if f.test != "" { + return f.test + } + return f.label +} + +// run is the result of one `go test -json` stream +type run struct { + name string + passed, failed, skipped int + failures []failure + err error +} + +func parseRun(path string) run { + r := run{name: strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))} + f, err := os.Open(path) + if err != nil { + r.err = err + return r + } + defer f.Close() + + pkgs := map[string]*pkgResult{} + build := map[string][]string{} + getPkg := func(name string) *pkgResult { + p, ok := pkgs[name] + if !ok { + p = &pkgResult{tests: map[string]*testResult{}} + pkgs[name] = p + } + return p + } + + br := bufio.NewReader(f) + for { + raw, readErr := br.ReadBytes('\n') + var e event + // gotestsum only writes events, but don't let a stray line hide the rest + if len(raw) > 0 && json.Unmarshal(raw, &e) == nil { + line := strings.TrimSuffix(e.Output, "\n") + switch { + case e.Action == "build-output": + build[e.ImportPath] = append(build[e.ImportPath], line) + case e.Test == "": + p := getPkg(e.Package) + switch e.Action { + case "output": + if e.OutputType != "frame" { + p.output = append(p.output, line) + p.all = append(p.all, line) + } + case "pass", "fail", "skip": + p.action = e.Action + p.failedBuild = e.FailedBuild + } + default: + p := getPkg(e.Package) + t, ok := p.tests[e.Test] + if !ok { + t = &testResult{} + p.tests[e.Test] = t + } + switch e.Action { + case "output": + if e.OutputType == "frame" { + break + } + t.output = append(t.output, line) + p.all = append(p.all, line) + if e.OutputType == "error" || e.OutputType == "error-continue" { + t.errors = append(t.errors, line) + } + case "pass", "fail", "skip": + r.count(e.Action) + // a failure sticks if the test runs again, e.g. with -count + if t.action != "fail" { + t.action = e.Action + } + t.elapsed = e.Elapsed + } + } + } + if readErr == io.EOF { + break + } + if readErr != nil { + r.err = readErr + return r + } + } + + for _, name := range sortedKeys(pkgs) { + r.addPackage(name, pkgs[name], build) + } + return r +} + +func (r *run) count(action string) { + switch action { + case "pass": + r.passed++ + case "fail": + r.failed++ + case "skip": + r.skipped++ + } +} + +func (r *run) addPackage(name string, p *pkgResult, build map[string][]string) { + var failed, unfinished []string + for _, tn := range sortedKeys(p.tests) { + switch p.tests[tn].action { + case "fail": + failed = append(failed, tn) + case "": + unfinished = append(unfinished, tn) + } + } + // a panic kills the whole test binary, and test2json files it under + // whichever test printed a framing line last: the parent of a panicking + // subtest, or a test that already passed before a timeout + var crashed []string + if i := crashIndex(p.all); i >= 0 { + crashed = p.all[i:] + } + for _, tn := range failed { + t := p.tests[tn] + // the failed subtests already say why + if len(t.errors) == 0 && hasSubtest(failed, tn) { + continue + } + f := failure{pkg: name, test: tn, elapsed: t.elapsed} + // a panic or race report says more than the error testing adds for it + if f.message = crash(t.output, maxMessageLines); f.message == nil { + f.message = head(t.errors, maxMessageLines) + } + if len(f.message) == 0 { + f.message = tail(t.output, maxMessageLines) + } + if len(f.message) == 0 { + f.message = head(crashed, maxMessageLines) + } + if ctx := tail(t.output, maxContextLines); !slices.Equal(ctx, f.message) { + f.context = ctx + } + r.failures = append(r.failures, f) + } + if p.action != "fail" { + return + } + + f := failure{pkg: name} + switch { + case p.failedBuild != "": + r.failed++ + f.label = "build failed" + f.message = head(build[p.failedBuild], maxMessageLines) + if len(f.message) == 0 { + f.message = excerpt(p.output, maxMessageLines) + } + case len(unfinished) > 0: + // on a timeout the running tests never report + r.failed += len(unfinished) + var leaves, rest []string + for _, tn := range unfinished { + rest = append(rest, p.tests[tn].output...) + if !hasSubtest(unfinished, tn) { + leaves = append(leaves, tn) + } + } + f.label = "did not finish: " + strings.Join(head(leaves, 5), ", ") + if len(leaves) > 5 { + f.label += fmt.Sprintf(" and %d more", len(leaves)-5) + } + if crashed != nil { + f.message = head(crashed, maxMessageLines) + f.context = head(stacksOf(crashed, leaves), maxStackLines) + } else { + f.message = tail(slices.Concat(p.output, rest), maxMessageLines) + } + case len(failed) == 0: + r.failed++ + f.label = "failed outside of a test" + f.message = excerpt(p.output, maxMessageLines) + default: + return + } + r.failures = append(r.failures, f) +} + +func hasSubtest(tests []string, test string) bool { + for _, tn := range tests { + if strings.HasPrefix(tn, test+"/") { + return true + } + } + return false +} + +// excerpt picks up to n lines likely to explain a failure +func excerpt(lines []string, n int) []string { + if c := crash(lines, n); c != nil { + return c + } + return tail(lines, n) +} + +// crash returns up to n lines from the first panic or race report, nil if +// there is none +func crash(lines []string, n int) []string { + if i := crashIndex(lines); i >= 0 { + return head(lines[i:], n) + } + return nil +} + +// crashIndex finds the first panic or race report. The runtime prints these +// unindented, unlike test logs, which may quote a panic from somewhere else. +func crashIndex(lines []string) int { + for i, l := range lines { + if strings.HasPrefix(l, "panic: ") || strings.HasPrefix(l, "fatal error: ") || + strings.HasPrefix(l, "WARNING: DATA RACE") { + return i + } + } + return -1 +} + +// stacksOf picks the goroutines running one of tests out of a stack dump +func stacksOf(lines, tests []string) []string { + var out, block []string + keep := false + flush := func() { + if keep { + out = append(out, block...) + } + block, keep = nil, false + } + for _, l := range lines { + if strings.HasPrefix(l, "goroutine ") { + flush() + block = []string{} + } + if block == nil { + continue + } + block = append(block, l) + for _, tn := range tests { + // subtests and their closures run as TestX.funcN + top, _, _ := strings.Cut(tn, "/") + if strings.Contains(l, "."+top+"(") || strings.Contains(l, "."+top+".func") { + keep = true + } + } + } + flush() + return out +} + +func head(lines []string, n int) []string { + if len(lines) > n { + return lines[:n] + } + return lines +} + +func tail(lines []string, n int) []string { + if len(lines) > n { + return lines[len(lines)-n:] + } + return lines +} + +func sortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// dedent strips the indent go test puts on every line of test output +func dedent(lines []string) []string { + out := make([]string, len(lines)) + for i, l := range lines { + l = strings.TrimPrefix(l, " ") + if len(l) > maxLineLen { + n := maxLineLen + for n > 0 && !utf8.RuneStart(l[n]) { + n-- + } + l = l[:n] + " ..." + } + out[i] = l + } + return out +} + +func (r run) writeText(w io.Writer) { + if r.err != nil { + fmt.Fprintf(w, "%s: no results: %v\n", r.name, r.err) + return + } + fmt.Fprintf(w, "%s: %d passed, %d failed, %d skipped\n", r.name, r.passed, r.failed, r.skipped) + for _, f := range r.failures { + fmt.Fprintf(w, "--- FAIL: %s %s\n", f.pkg, f.title()) + for _, l := range dedent(f.message) { + fmt.Fprintf(w, " %s\n", l) + } + } +} + +// writeMarkdown renders runs for the GitHub job summary, leaving out failures +// once the output would pass budget bytes +func writeMarkdown(w io.Writer, runs []run, budget int) error { + var b strings.Builder + b.WriteString("| run | passed | failed | skipped |\n|:--|--:|--:|--:|\n") + for _, r := range runs { + if r.err != nil { + fmt.Fprintf(&b, "| %s | - | - | - |\n", r.name) + continue + } + fmt.Fprintf(&b, "| %s | %d | %d | %d |\n", r.name, r.passed, r.failed, r.skipped) + } + omitted := 0 + for _, r := range runs { + if len(r.failures) == 0 { + continue + } + fmt.Fprintf(&b, "\n#### %s\n", html.EscapeString(r.name)) + for _, f := range r.failures { + entry := f.markdown() + if b.Len()+len(entry) > budget { + omitted++ + continue + } + b.WriteString(entry) + } + } + if omitted > 0 { + fmt.Fprintf(&b, "\n%d more failures did not fit, see the step logs.\n", omitted) + } + _, err := io.WriteString(w, b.String()) + return err +} + +func (f failure) markdown() string { + var b strings.Builder + fmt.Fprintf(&b, "\n%s in %s", html.EscapeString(f.title()), html.EscapeString(f.pkg)) + if f.test != "" { + fmt.Fprintf(&b, " (%.2fs)", f.elapsed) + } + b.WriteString("\n\n") + writeCodeBlock(&b, dedent(f.message)) + if len(f.context) > 0 { + b.WriteString("\n
output\n\n") + writeCodeBlock(&b, dedent(f.context)) + b.WriteString("\n
\n") + } + return b.String() +} + +var backticks = regexp.MustCompile("`+") + +// writeCodeBlock fences lines with more backticks than any run inside them, +// so test output can't end the block early +func writeCodeBlock(b *strings.Builder, lines []string) { + n := 3 + for _, l := range lines { + for _, m := range backticks.FindAllString(l, -1) { + n = max(n, len(m)+1) + } + } + fence := strings.Repeat("`", n) + fmt.Fprintf(b, "%stext\n", fence) + for _, l := range lines { + b.WriteString(l) + b.WriteByte('\n') + } + fmt.Fprintf(b, "%s\n", fence) +} + +var ( + errorLine = regexp.MustCompile(`^\s*([^\s:]+\.go):(\d+): `) + stackLine = regexp.MustCompile(`^\s*(/\S+\.go):(\d+)`) +) + +// writeAnnotations prints a GitHub error annotation per failure, pointing at +// the line that failed when it is in the test's own package under root +func writeAnnotations(w io.Writer, runs []run, root string) { + module := modulePath(root) + total := 0 + for _, r := range runs { + total += len(r.failures) + } + n := 0 + for _, r := range runs { + for _, f := range r.failures { + if n == maxAnnotations-1 && total > maxAnnotations { + fmt.Fprintf(w, "::error::%d more test failures, see the job summary\n", total-n) + return + } + n++ + title := fmt.Sprintf("%s: %s.%s", r.name, path.Base(f.pkg), f.test) + if f.test == "" { + title = fmt.Sprintf("%s: %s %s", r.name, path.Base(f.pkg), f.label) + } + props := "title=" + escapeProperty(title) + if file, line := location(root, module, f); file != "" { + props = fmt.Sprintf("file=%s,line=%s,%s", escapeProperty(file), line, props) + } + fmt.Fprintf(w, "::error %s::%s\n", props, escapeData(strings.Join(dedent(f.message), "\n"))) + } + } +} + +// location finds where f failed under root: the file of the first error line +// if it is in the test's own package, else the first stack frame under root, +// which for a timeout is in the context +func location(root, module string, f failure) (file, line string) { + if root == "" || len(f.message) == 0 { + return "", "" + } + if m := errorLine.FindStringSubmatch(f.message[0]); m != nil { + if file := sourcePath(root, module, f.pkg, m[1]); file != "" { + return file, m[2] + } + return "", "" + } + for _, l := range slices.Concat(f.message, f.context) { + m := stackLine.FindStringSubmatch(l) + if m == nil { + continue + } + if rel, err := filepath.Rel(root, m[1]); err == nil && !strings.HasPrefix(rel, "..") { + return filepath.ToSlash(rel), m[2] + } + } + return "", "" +} + +func modulePath(root string) string { + if root == "" { + return "" + } + raw, err := os.ReadFile(filepath.Join(root, "go.mod")) + if err != nil { + return "" + } + for l := range strings.SplitSeq(string(raw), "\n") { + if m, ok := strings.CutPrefix(strings.TrimSpace(l), "module "); ok { + if u, err := strconv.Unquote(m); err == nil { + return u + } + return strings.TrimSpace(m) + } + } + return "" +} + +// sourcePath returns file's path relative to root if it is in pkg's directory. +// Nested modules here have paths under the root module, so the import path +// maps to a directory either way. +func sourcePath(root, module, pkg, file string) string { + if module == "" { + return "" + } + rel, ok := strings.CutPrefix(pkg, module) + if !ok || (rel != "" && !strings.HasPrefix(rel, "/")) { + return "" + } + p := path.Join(strings.TrimPrefix(rel, "/"), file) + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(p))); err != nil { + return "" + } + return p +} + +// https://github.com/actions/toolkit/blob/7e25f67e8fd17369aa21e6e816ac99654106e0dc/packages/core/src/command.ts#L103-L117 +func escapeData(s string) string { + return strings.NewReplacer("%", "%25", "\r", "%0D", "\n", "%0A").Replace(s) +} + +func escapeProperty(s string) string { + return strings.NewReplacer("%", "%25", "\r", "%0D", "\n", "%0A", ":", "%3A", ",", "%2C").Replace(s) +} diff --git a/tools/junittool/report_test.go b/tools/junittool/report_test.go new file mode 100644 index 0000000000..4ded66dbd2 --- /dev/null +++ b/tools/junittool/report_test.go @@ -0,0 +1,267 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +const mod = "example.com/m" + +// stream renders events as go test -json output. +func stream(t *testing.T, events ...event) string { + t.Helper() + var b strings.Builder + for _, e := range events { + raw, err := json.Marshal(e) + if err != nil { + t.Fatal(err) + } + b.Write(raw) + b.WriteByte('\n') + } + return b.String() +} + +func out(pkg, test, typ, line string) event { + return event{Action: "output", Package: pkg, Test: test, OutputType: typ, Output: line + "\n"} +} + +func end(action, pkg, test string) event { + return event{Action: action, Package: pkg, Test: test} +} + +func TestParseRun(t *testing.T) { + a, b, c, d, e := mod+"/a", mod+"/b", mod+"/c", mod+"/d", mod+"/e" + content := stream(t, + // passing test, its logs are not reported + out(a, "TestPass", "frame", "=== RUN TestPass"), + out(a, "TestPass", "", " a_test.go:5: fine"), + end("pass", a, "TestPass"), + end("skip", a, "TestSkip"), + // failed subtests; the parent only has logs + out(a, "TestParent", "", " a_test.go:10: setting up"), + out(a, "TestParent/sub", "frame", "=== RUN TestParent/sub"), + out(a, "TestParent/sub", "", " a_test.go:12: trying"), + out(a, "TestParent/sub", "error", " a_test.go:13: expected 1"), + out(a, "TestParent/sub", "error-continue", " but got 2"), + out(a, "TestParent/sub", "frame", " --- FAIL: TestParent/sub (0.00s)"), + end("fail", a, "TestParent/sub"), + end("fail", a, "TestParent"), + // a failed parent with an error of its own is still reported + out(a, "TestOwn/sub", "error", " a_test.go:20: sub broke"), + end("fail", a, "TestOwn/sub"), + out(a, "TestOwn", "error", " a_test.go:22: cleanup broke"), + end("fail", a, "TestOwn"), + // a panic has no error lines + out(a, "TestPanic", "", " a_test.go:30: before"), + out(a, "TestPanic", "frame", "--- FAIL: TestPanic (0.00s)"), + out(a, "TestPanic", "", "panic: boom [recovered]"), + out(a, "TestPanic", "", ""), + out(a, "TestPanic", "", "a.TestPanic(...)"), + end("fail", a, "TestPanic"), + // a logged panic is not a crash + out(a, "TestQuotesPanic", "", " a_test.go:41: pod logs:"), + out(a, "TestQuotesPanic", "", " panic: not ours"), + out(a, "TestQuotesPanic", "error", " a_test.go:42: pod crashed"), + end("fail", a, "TestQuotesPanic"), + // the race report says more than the error testing adds for it + out(a, "TestRace", "", "WARNING: DATA RACE"), + out(a, "TestRace", "", "Write at 0x1 by goroutine 7:"), + out(a, "TestRace", "error", " testing.go:1865: race detected during execution of test"), + end("fail", a, "TestRace"), + end("fail", a, ""), + // a timeout leaves tests unfinished, the panic lands in whichever test + // printed a framing line last + event{Action: "run", Package: b, Test: "TestHang"}, + event{Action: "run", Package: b, Test: "TestHang2"}, + event{Action: "run", Package: b, Test: "TestHang2/sub"}, + out(b, "TestQuick", "frame", "--- PASS: TestQuick (0.00s)"), + out(b, "TestQuick", "", "panic: test timed out after 2s"), + out(b, "TestQuick", "", "\trunning tests:"), + out(b, "TestQuick", "", ""), + out(b, "TestQuick", "", "goroutine 1 [chan receive]:"), + out(b, "TestQuick", "", "testing.(*M).Run(0x1)"), + out(b, "TestQuick", "", ""), + out(b, "TestQuick", "", "goroutine 7 [sleep]:"), + out(b, "TestQuick", "", "b.TestHang2.func1(0x2)"), + out(b, "TestQuick", "", "\t/src/b/b_test.go:9 +0x1"), + end("pass", b, "TestQuick"), + end("fail", b, ""), + // build failure + event{Action: "build-output", ImportPath: c + " [" + c + ".test]", Output: "c/c_test.go:5:28: undefined: x\n"}, + event{Action: "fail", Package: c, FailedBuild: c + " [" + c + ".test]"}, + // TestMain exits non-zero + out(d, "", "", "TestMain setup failed"), + end("fail", d, ""), + // a panicking subtest's panic is filed under its parent + out(e, "TestParent", "", " e_test.go:13: parent setup"), + out(e, "TestParent/sub", "frame", "--- FAIL: TestParent/sub (0.00s)"), + end("fail", e, "TestParent/sub"), + out(e, "TestParent", "frame", "--- FAIL: TestParent (0.00s)"), + out(e, "TestParent", "", "panic: boom [recovered, repanicked]"), + end("fail", e, "TestParent"), + end("fail", e, ""), + ) + "not json\n" + path := write(t, t.TempDir(), "unit.json", content) + + got := parseRun(path) + if got.err != nil { + t.Fatalf("parseRun() error = %v", got.err) + } + if got.name != "unit" || got.passed != 2 || got.failed != 14 || got.skipped != 1 { + t.Errorf("parseRun() = %s %d passed, %d failed, %d skipped; want unit 2, 14, 1", + got.name, got.passed, got.failed, got.skipped) + } + + want := []failure{ + {pkg: a, test: "TestOwn", message: []string{" a_test.go:22: cleanup broke"}}, + {pkg: a, test: "TestOwn/sub", message: []string{" a_test.go:20: sub broke"}}, + {pkg: a, test: "TestPanic", + message: []string{"panic: boom [recovered]", "", "a.TestPanic(...)"}, + context: []string{" a_test.go:30: before", "panic: boom [recovered]", "", "a.TestPanic(...)"}}, + {pkg: a, test: "TestParent/sub", + message: []string{" a_test.go:13: expected 1", " but got 2"}, + context: []string{" a_test.go:12: trying", " a_test.go:13: expected 1", " but got 2"}}, + {pkg: a, test: "TestQuotesPanic", + message: []string{" a_test.go:42: pod crashed"}, + context: []string{" a_test.go:41: pod logs:", " panic: not ours", " a_test.go:42: pod crashed"}}, + {pkg: a, test: "TestRace", message: []string{"WARNING: DATA RACE", "Write at 0x1 by goroutine 7:", + " testing.go:1865: race detected during execution of test"}}, + {pkg: b, label: "did not finish: TestHang, TestHang2/sub", + message: []string{"panic: test timed out after 2s", "\trunning tests:", "", + "goroutine 1 [chan receive]:", "testing.(*M).Run(0x1)", "", + "goroutine 7 [sleep]:", "b.TestHang2.func1(0x2)", "\t/src/b/b_test.go:9 +0x1"}, + context: []string{"goroutine 7 [sleep]:", "b.TestHang2.func1(0x2)", "\t/src/b/b_test.go:9 +0x1"}}, + {pkg: c, label: "build failed", message: []string{"c/c_test.go:5:28: undefined: x"}}, + {pkg: d, label: "failed outside of a test", message: []string{"TestMain setup failed"}}, + {pkg: e, test: "TestParent/sub", message: []string{"panic: boom [recovered, repanicked]"}}, + } + if !reflect.DeepEqual(got.failures, want) { + t.Errorf("failures:\n%s\nwant:\n%s", dump(got.failures), dump(want)) + } +} + +func dump(fs []failure) string { + var b strings.Builder + for _, f := range fs { + fmt.Fprintf(&b, " %+v\n", f) + } + return b.String() +} + +func TestParseRunMissing(t *testing.T) { + got := parseRun(filepath.Join(t.TempDir(), "e2e-gvisor.json")) + if got.err == nil || got.name != "e2e-gvisor" { + t.Errorf("parseRun(missing) = %+v, want an error for e2e-gvisor", got) + } +} + +func TestExcerpt(t *testing.T) { + lines := []string{"1", "2", "3", "4"} + if got := excerpt(lines, 2); !reflect.DeepEqual(got, []string{"3", "4"}) { + t.Errorf("excerpt() = %v, want the last 2 lines", got) + } + race := []string{"log", "WARNING: DATA RACE", "Write at", "by goroutine 7", "more"} + if got := excerpt(race, 3); !reflect.DeepEqual(got, race[1:4]) { + t.Errorf("excerpt() = %v, want the race report", got) + } +} + +func TestWriteAnnotations(t *testing.T) { + root := t.TempDir() + write(t, root, "go.mod", "module "+mod+"\n\ngo 1.27.0\n") + if err := os.MkdirAll(filepath.Join(root, "a"), 0o755); err != nil { + t.Fatal(err) + } + write(t, filepath.Join(root, "a"), "a_test.go", "package a\n") + + runs := []run{{name: "unit", failures: []failure{ + {pkg: mod + "/a", test: "TestX", message: []string{" a_test.go:13: 50% off, a: b", " next"}}, + // the file is in another package, so no location + {pkg: mod + "/a", test: "TestY", message: []string{" helper.go:7: bad"}}, + {pkg: mod + "/a", test: "TestZ", message: []string{"panic: boom", + "\t/usr/lib/go/src/testing/testing.go:2123 +0x2b8", + "\t" + filepath.Join(root, "a", "a_test.go") + ":31 +0x3c"}}, + // a timeout's hung test is in the context + {pkg: mod + "/a", label: "did not finish: TestW", message: []string{"panic: test timed out after 2s"}, + context: []string{"a.TestW(0x1)", "\t" + filepath.Join(root, "a", "a_test.go") + ":40 +0x3c"}}, + }}} + var b strings.Builder + writeAnnotations(&b, runs, root) + want := "::error file=a/a_test.go,line=13,title=unit%3A a.TestX::a_test.go:13: 50%25 off, a: b%0A next\n" + + "::error title=unit%3A a.TestY::helper.go:7: bad\n" + + "::error file=a/a_test.go,line=31,title=unit%3A a.TestZ::panic: boom%0A\t/usr/lib/go/src/testing/testing.go:2123 +0x2b8%0A\t" + + filepath.Join(root, "a", "a_test.go") + ":31 +0x3c\n" + + "::error file=a/a_test.go,line=40,title=unit%3A a did not finish%3A TestW::panic: test timed out after 2s\n" + if b.String() != want { + t.Errorf("writeAnnotations() =\n%s\nwant:\n%s", b.String(), want) + } + + t.Run("capped", func(t *testing.T) { + r := run{name: "e2e"} + for i := range 12 { + r.failures = append(r.failures, failure{pkg: mod + "/a", test: fmt.Sprintf("Test%d", i)}) + } + var b strings.Builder + writeAnnotations(&b, []run{r}, "") + lines := strings.Split(strings.TrimSpace(b.String()), "\n") + if len(lines) != maxAnnotations { + t.Fatalf("writeAnnotations() printed %d lines, want %d", len(lines), maxAnnotations) + } + if last := lines[len(lines)-1]; last != "::error::3 more test failures, see the job summary" { + t.Errorf("last annotation = %q", last) + } + }) +} + +func TestWriteMarkdown(t *testing.T) { + runs := []run{ + {name: "unit", passed: 3, failed: 1, failures: []failure{ + {pkg: mod + "/a", test: "TestX", elapsed: 1.5, message: []string{" a_test.go:1: has ``` in it"}}, + }}, + {name: "root", err: os.ErrNotExist}, + } + var b strings.Builder + if err := writeMarkdown(&b, runs, summaryBudget); err != nil { + t.Fatal(err) + } + for _, want := range []string{ + "| unit | 3 | 1 | 0 |", + "| root | - | - | - |", + "TestX<y> in " + mod + "/a (1.50s)", + "````text\na_test.go:1: has ``` in it\n````\n", + } { + if !strings.Contains(b.String(), want) { + t.Errorf("writeMarkdown() missing %q in:\n%s", want, b.String()) + } + } + + t.Run("over budget", func(t *testing.T) { + var b strings.Builder + if err := writeMarkdown(&b, runs, 200); err != nil { + t.Fatal(err) + } + if strings.Contains(b.String(), "TestX") || !strings.Contains(b.String(), "1 more failures did not fit") { + t.Errorf("writeMarkdown() over budget =\n%s", b.String()) + } + }) +} diff --git a/tools/setup-gcp/README.md b/tools/setup-gcp/README.md index 042e77836d..81803576db 100644 --- a/tools/setup-gcp/README.md +++ b/tools/setup-gcp/README.md @@ -116,6 +116,77 @@ go run ./tools/setup-gcp create cluster [flags] `NODE_MACHINE_TYPE` was named `GVISOR_NODE_MACHINE_TYPE`. The old name still works and warns; rename it in your `.ate-dev-env.sh`. +**Node sysctls for high sandbox density (>800 sandboxes/node):** each awake +actor sandbox creates its own network namespace, veth/tap pair, and inotify +watches on the host kernel (even in multi-actor worker pods). Default COS +limits (`net.ipv4.neigh.default.gc_thresh3 = 1024`, +`fs.inotify.max_user_instances = 128`, `fs.inotify.max_user_watches = 8192`) +fail around ~800–1,000 live sandboxes per node: in single-node density +benchmarks on `c3-standard-192-metal` (50–100 multi-actor worker pods), +inotify watches exhausted at ~800 live sandboxes and the host ARP cache +overflowed on the 1,000-sandbox wave after passing 850 (`dmesg`: `neighbour: +arp_cache: neighbor table overflow!`), while raising these sysctls sustained +1,500 concurrent sandboxes per node. +`create cluster` sets these sysctls on `substrate-node-pool` **when creating a +new cluster**; existing clusters keep their original node-pool sysctls +(`create cluster` logs a warning if they are missing) and must be updated via +one of the options below. + +* **When creating or updating node pools** (e.g., bare-metal pools), pass + `--system-config-from-file=systemconfig.yaml` to `gcloud container node-pools + create` or `update`. Note that `node-pools update --system-config-from-file` + **replaces the pool's entire system config** (so carry over any existing + sysctls/kubelet/cgroup settings) and **recreates the pool's nodes**: + + ```yaml + # systemconfig.yaml + linuxConfig: + sysctls: + net.ipv4.neigh.default.gc_thresh1: '4096' + net.ipv4.neigh.default.gc_thresh2: '16384' + net.ipv4.neigh.default.gc_thresh3: '32768' + fs.inotify.max_user_instances: '65536' + fs.inotify.max_user_watches: '1048576' + ``` + +* **On an already-running node pool (without rolling/recreating nodes)**, apply + a privileged `DaemonSet` to tune the host sysctls in place: + + ```yaml + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: substrate-sysctl-tuner + namespace: kube-system + spec: + selector: + matchLabels: + app: substrate-sysctl-tuner + template: + metadata: + labels: + app: substrate-sysctl-tuner + spec: + hostNetwork: true + tolerations: + - operator: Exists + containers: + - name: sysctl-tuner + image: busybox:1.36 + securityContext: + privileged: true + command: + - sh + - -ce + - | + sysctl -w net.ipv4.neigh.default.gc_thresh1=4096 + sysctl -w net.ipv4.neigh.default.gc_thresh2=16384 + sysctl -w net.ipv4.neigh.default.gc_thresh3=32768 + sysctl -w fs.inotify.max_user_instances=65536 + sysctl -w fs.inotify.max_user_watches=1048576 + sleep infinity + ``` + ### 3. Create Bucket Creates a GCS bucket for storing snapshots. diff --git a/tools/setup-gcp/cmd/cluster.go b/tools/setup-gcp/cmd/cluster.go index d39a997fcc..c3386272de 100644 --- a/tools/setup-gcp/cmd/cluster.go +++ b/tools/setup-gcp/cmd/cluster.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "log/slog" + "maps" "slices" "strings" "time" @@ -37,6 +38,14 @@ var requiredBetaAPIs = []string{ "certificates.k8s.io/v1beta1/clustertrustbundles", } +var requiredNodeSysctls = map[string]string{ + "net.ipv4.neigh.default.gc_thresh1": "4096", + "net.ipv4.neigh.default.gc_thresh2": "16384", + "net.ipv4.neigh.default.gc_thresh3": "32768", + "fs.inotify.max_user_instances": "65536", + "fs.inotify.max_user_watches": "1048576", +} + func deleteCluster(ctx context.Context, cfg *Config) error { client, err := container.NewClusterManagerClient(ctx) if err != nil { @@ -61,6 +70,9 @@ func buildCreateClusterRequest(parent string, cfg *Config) *containerpb.CreateCl } nodeConfig := &containerpb.NodeConfig{ MachineType: cfg.MachineType, + LinuxNodeConfig: &containerpb.LinuxNodeConfig{ + Sysctls: maps.Clone(requiredNodeSysctls), + }, } if cfg.EnableNestedVirtualization { nodeConfig.AdvancedMachineFeatures = &containerpb.AdvancedMachineFeatures{ @@ -120,6 +132,28 @@ func nestedVirtualizationEnabled(cluster *containerpb.Cluster) bool { return false } +// nodeSysctlsConfigured reports whether any node pool carries all +// requiredNodeSysctls keys in its LinuxNodeConfig. +func nodeSysctlsConfigured(cluster *containerpb.Cluster) bool { + for _, pool := range cluster.GetNodePools() { + sysctls := pool.GetConfig().GetLinuxNodeConfig().GetSysctls() + if len(sysctls) == 0 { + continue + } + allPresent := true + for k := range requiredNodeSysctls { + if _, ok := sysctls[k]; !ok { + allPresent = false + break + } + } + if allPresent { + return true + } + } + return false +} + func createClusterInternal(ctx context.Context, cfg *Config, client *container.ClusterManagerClient, parent string) error { slog.Info("Cluster does not exist. Creating...", slog.String("cluster", cfg.ClusterName)) req := buildCreateClusterRequest(parent, cfg) @@ -305,6 +339,12 @@ func createClusterIdempotent(ctx context.Context, cfg *Config) error { slog.String("remedy", "recreate the cluster, or add a node pool with nested virtualization")) } + if !nodeSysctlsConfigured(cluster) { + slog.Warn("Cluster has no node pool with high-density node sysctls; create cluster only sets them on a cluster it creates", + slog.String("cluster", cfg.ClusterName), + slog.String("remedy", "run gcloud container node-pools update --system-config-from-file (see tools/setup-gcp/README.md; this warning is expected to remain if using the in-place DaemonSet)")) + } + return nil } diff --git a/tools/setup-gcp/cmd/cluster_test.go b/tools/setup-gcp/cmd/cluster_test.go index 6ded0a9e83..ab46f4451a 100644 --- a/tools/setup-gcp/cmd/cluster_test.go +++ b/tools/setup-gcp/cmd/cluster_test.go @@ -15,6 +15,8 @@ package cmd import ( + "maps" + "strconv" "strings" "testing" @@ -106,6 +108,36 @@ func TestBuildCreateClusterRequest_NodeConfig(t *testing.T) { if nodeConfig.DiskType != tt.wantDiskType { t.Errorf("DiskType = %q, want %q", nodeConfig.DiskType, tt.wantDiskType) } + + sysctls := nodeConfig.GetLinuxNodeConfig().GetSysctls() + wantKeys := []string{ + "net.ipv4.neigh.default.gc_thresh1", + "net.ipv4.neigh.default.gc_thresh2", + "net.ipv4.neigh.default.gc_thresh3", + "fs.inotify.max_user_instances", + "fs.inotify.max_user_watches", + } + parsed := make(map[string]int, len(wantKeys)) + for _, k := range wantKeys { + raw, ok := sysctls[k] + if !ok { + t.Fatalf("missing required sysctl key %q", k) + } + v, err := strconv.Atoi(raw) + if err != nil || v <= 0 { + t.Fatalf("sysctl %q = %q, want positive integer: %v", k, raw, err) + } + parsed[k] = v + } + t1 := parsed["net.ipv4.neigh.default.gc_thresh1"] + t2 := parsed["net.ipv4.neigh.default.gc_thresh2"] + t3 := parsed["net.ipv4.neigh.default.gc_thresh3"] + if !(1024 < t1 && t1 < t2 && t2 < t3) { + t.Errorf("expected 1024 < gc_thresh1 (%d) < gc_thresh2 (%d) < gc_thresh3 (%d)", t1, t2, t3) + } + if watches := parsed["fs.inotify.max_user_watches"]; watches > 1048576 { + t.Errorf("fs.inotify.max_user_watches = %d, exceeds GKE cap of 1048576", watches) + } }) } } @@ -232,6 +264,77 @@ func TestNestedVirtualizationEnabled(t *testing.T) { } } +func TestNodeSysctlsConfigured(t *testing.T) { + poolWith := func(sysctls map[string]string) *containerpb.NodePool { + return &containerpb.NodePool{ + Config: &containerpb.NodeConfig{ + LinuxNodeConfig: &containerpb.LinuxNodeConfig{ + Sysctls: sysctls, + }, + }, + } + } + + partialSysctls := maps.Clone(requiredNodeSysctls) + delete(partialSysctls, "fs.inotify.max_user_watches") + + tests := []struct { + name string + cluster *containerpb.Cluster + want bool + }{ + { + name: "nil cluster", + cluster: nil, + want: false, + }, + { + name: "no node pools", + cluster: &containerpb.Cluster{}, + want: false, + }, + { + name: "pool without linux node config", + cluster: &containerpb.Cluster{ + NodePools: []*containerpb.NodePool{{Config: &containerpb.NodeConfig{}}}, + }, + want: false, + }, + { + name: "pool missing required sysctl", + cluster: &containerpb.Cluster{ + NodePools: []*containerpb.NodePool{poolWith(partialSysctls)}, + }, + want: false, + }, + { + name: "single pool with required sysctls", + cluster: &containerpb.Cluster{ + NodePools: []*containerpb.NodePool{poolWith(maps.Clone(requiredNodeSysctls))}, + }, + want: true, + }, + { + name: "second pool carries required sysctls", + cluster: &containerpb.Cluster{ + NodePools: []*containerpb.NodePool{ + poolWith(nil), + poolWith(maps.Clone(requiredNodeSysctls)), + }, + }, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := nodeSysctlsConfigured(tt.cluster); got != tt.want { + t.Errorf("nodeSysctlsConfigured() = %v, want %v", got, tt.want) + } + }) + } +} + func TestFilestoreCsiDriverEnabled(t *testing.T) { tests := []struct { name string