diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 37393f7087..17f21ffe51 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -144,7 +144,7 @@ jobs: for arch in amd64 arm64; do CGO_ENABLED=0 GOOS="${os}" GOARCH="${arch}" go build \ -trimpath \ - -ldflags="-s -w -X=github.com/agent-substrate/substrate/internal/version.Version=${VERSION}" \ + -ldflags="-s -w -X=go.ate.dev/substrate/internal/version.Version=${VERSION}" \ -o "dist/kubectl-ate-${os}-${arch}" \ ./cmd/kubectl-ate done diff --git a/.golangci.yaml b/.golangci.yaml index 96c1f42822..3d70dd8b22 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -43,7 +43,7 @@ linters: - "!**/cmd/ateworker-*/**" - "!**/internal/ateworker/**" deny: - - pkg: github.com/agent-substrate/substrate/internal/ateworker + - pkg: go.ate.dev/substrate/internal/ateworker desc: only the ateworker binaries may import internal/ateworker errcheck: diff --git a/.ko.yaml b/.ko.yaml index 7dd90c96e7..a16adb0c23 100644 --- a/.ko.yaml +++ b/.ko.yaml @@ -19,7 +19,7 @@ defaultPlatforms: - linux/arm64 baseImageOverrides: - github.com/agent-substrate/substrate/demos/sandbox: alpine:3.24.2@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6 + go.ate.dev/substrate/demos/sandbox: alpine:3.24.2@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6 # atelet needs glibc to run the node's kubelet credential provider, which # some platforms ship dynamically linked (e.g. AKS, GKE Ubuntu). - github.com/agent-substrate/substrate/cmd/atelet: gcr.io/distroless/base-debian13:latest@sha256:389cad21f73e4c37b94ffe5b13736d5a92bd5bd3c6c6b38c2be1c881e14ba2bd + go.ate.dev/substrate/cmd/atelet: gcr.io/distroless/base-debian13:latest@sha256:389cad21f73e4c37b94ffe5b13736d5a92bd5bd3c6c6b38c2be1c881e14ba2bd diff --git a/AGENTS.md b/AGENTS.md index ab3eaf7953..59a2e4a269 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -104,5 +104,6 @@ Keep this up to date when updating AGENTS.md. - **Workload Isolation**: The project uses `gVisor` (`runsc`) for sandboxing and security isolation of workloads on pods. - **JWTs and JWKs**: Signing (`internal/localjwtauthority`), verification (`internal/oidcjwt`), and JWK set publishing (`internal/oidcdiscovery`) are built on the standard library's crypto packages. Extend those packages rather than adding go-jose or another JOSE library, so the code that decides whether a token is valid stays small and supports only the algorithms Substrate uses. - **Ingress Authentication**: `atenet-router --ingress-auth-mode=static-mtls` requires clients to present a SPIFFE certificate signed by `--ingress-client-ca-file` whose ID is in `--ingress-allowed-spiffe-ids`; that list is the whole authorization policy, so any listed client may reach any Actor. The default, `deprecated-insecure`, checks a certificate only when a client presents one, and also serves plaintext listeners that accept any client. Install with it via `ate-setup --ingress-auth-mode=static-mtls` (`manifests/ate-install/components/router-static-mtls`); the allowlist lives in `manifests/ate-install/atenet-router.yaml`. +- **Control API Authorization**: `ate-api-server` checks every Control RPC against an embedded OpenFGA model (`cmd/ateapi/internal/authz/model.fga`), on by default. Every served unary RPC needs an entry in `defaultRPCPermissions` (`registry.go`): with enforcement on, an RPC without one is denied, and a test fails the build. Access is granted per global scope and atespace through AccessPolicy (`kubectl ate access-policy`); the install's bootstrap owner is the shared `ate-client` ServiceAccount that `kubectl ate` uses. See `docs/authorization.md` for roles and known gaps. For future plans for security, reference `docs/roadmap.md`. diff --git a/README.md b/README.md index 41b9cf60bf..ccf43e918d 100644 --- a/README.md +++ b/README.md @@ -7,34 +7,33 @@ # Agent Substrate -**A secure-by-default runtime for running millions of agent sandboxes on Kubernetes.** +**Zero-trust sandboxing for millions of agents on Kubernetes.** [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -[Quickstart](#quickstart-development) · [Architecture](docs/architecture.md) · [Roadmap](docs/roadmap.md) · [Contributing](CONTRIBUTING.md) +[Quickstart](#quickstart-development) · [Security](docs/security.md) · [Architecture](docs/architecture.md) · [Roadmap](docs/roadmap.md) · [Contributing](CONTRIBUTING.md) ## What is Agent Substrate? -Agent Substrate is a secure-by-default agent execution runtime engineered to run millions of sandboxes with 10x higher density than standard container runtimes. Purpose-built for the era of autonomous agents, Substrate delivers sub-500ms resume operations at over 500 suspend/resume activations per second with native zero-trust kernel and network isolation. It supports multiple sandbox technologies including microVMs and gVisor, enabling consistent lifecycle operations for all sandbox types. +Agent Substrate is a security-hardened, zero-trust runtime for sandboxed agents, engineered to safely run millions of agents and tools with 10x higher density than standard containers. Purpose-built for the era of autonomous agents, Substrate delivers fast resume operations in less than 400ms, with more than 1500 activations per second. Agents get isolated compute, identity, networking, and storage. Sensitive credentials are kept out of reach from untrusted code. It supports multiple sandbox technologies, including microVMs and gVisor, while enabling a consistent lifecycle and management of all sandboxes. -At its core, Agent Substrate maps a larger set of “actors” (applications such as agents) onto a smaller set of ready “workers”, relying on the fact that agent-like applications tend to be idle most of the time to achieve heavy multiplexing. It provides functionality to manage an actor’s lifecycle (e.g. create/destroy, suspend/resume), to assign actors to workers in real time, and to route incoming traffic to them. +At its core, Agent Substrate maps a larger set of “actors” (stateful workloads such as agents) onto a smaller pool of ready “workers”. Because agents spend most of their lives waiting on models, tools, or humans, Substrate snapshots idle actors out of memory and restores them when a request arrives. -Agent Substrate is intended to be a low-opinion system. The workloads it manages don't have to be literal AI agents, but those are the best example of the kind of applications it is designed for. It is not an SDK for building agents, but rather a system for running them at scale. +Substrate is a low-opinion runtime, not an agent SDK. The workloads it runs do not have to be AI agents, though autonomous, untrusted, mostly-idle agents are the target for its isolation and scheduling model. It runs on standard Kubernetes: Pods, autoscaling, NetworkPolicies. It adds agent-specific scheduling, snapshotting, identity, and network mediation. This allows agentic, inference, training, and reinforcement learning workloads to share one infrastructure plane. -Agent Substrate leverages Kubernetes for the infrastructure provisioning and worker lifecycle management (Kubernetes Pods). It builds on top of Kubernetes features like Pods and Pod autoscaling, while Agent Substrate provides agent-specific scheduling and control to achieve lower latency. Using Kubernetes as the underlying system enables consistent infrastructure management across all workloads types that are required for end to end agentic deployments and allows holistic infrastructure optimizations for RL scenarios that span agentic, inference and training cycles. +## Security Model -## Star History +Substrate treats every actor as untrusted and mutually hostile, so tenants can share physical nodes without sharing a security boundary. Every control operates within the cold-start and resume budget. See the [Security Guide](docs/security.md) for the full list of security principles and threat model. - - - - Star history chart for agent-substrate/substrate - - +* **Always sandboxed, VM-grade boundaries:** Every actor runs inside a dedicated [gVisor](https://gvisor.dev/) (`runsc`) or [Cloud Hypervisor](https://www.cloudhypervisor.org/) microVM sandbox with segregated compute, network, and storage. Plain Linux containers are not supported. +* **Per-agent cryptographic identity:** Every active actor receives a short-lived [SPIFFE](https://spiffe.io/) X.509 certificate by default, and the egress gateway can mint per-actor JWTs on demand when an egress policy calls for one. Both carry the actor identity and atespace (the Substrate namespace grouping), and are designed to federate with external identity systems. +* **Zero credentials in the sandbox:** Actors never hold actor identity JWTs/certs, API keys, or cloud tokens. The built-in egress gateway ([`atenet-egress`](docs/egress-credential-injection.md)) injects credentials into approved outbound requests outside the sandbox, so a prompt-injected or compromised agent has nothing to exfiltrate. +* **Default-deny networking:** Actors start with a private network stack, no inbound path except through the ingress router (`atenet-router`), no lateral actor-to-actor or control-plane access, and outbound traffic [restricted by the egress gateway](docs/egress-traffic.md). +* **Pluggable policy at the edge:** The egress gateway acts as an external Policy Enforcement Point (PEP) that is fully replaceable, allowing you to attach your own authorization, DLP, prompt-injection scanning, or cloud IAM policy engine without placing policy code inside the untrusted sandbox. -## Demo +## Performance & Multiplexing [![Agent Substrate Demo](https://img.youtube.com/vi/ZEzkCFJkzjY/hq1.jpg)](https://www.youtube.com/watch?v=ZEzkCFJkzjY) @@ -42,22 +41,29 @@ Agent Substrate leverages Kubernetes for the infrastructure provisioning and wor This demo highlights the core developer experience and "Agentic Infrastructure" capabilities of Substrate: -1. **Actor Teleport:** High-performance suspend and resume of actors onto any available worker in the pool with sub-second activation. -2. **State Persistence:** Persistent working memory (volatile RAM) and filesystem state preserved perfectly across hibernation cycles via full-state snapshots. -3. **Agent Multiplexing:** Demonstrates 30x+ oversubscription by "juggling" a large registry of stateful actors onto a small pool of shared physical pods. +1. **Actor Teleport:** Suspend and resume actors onto any available worker in the pool, when needed. +2. **Full-State Persistence:** Working memory (RAM) and filesystem state are captured in snapshots and restored intact across hibernation cycles, without losing in-flight sessions or tool state. +3. **10x–30x Oversubscription:** Multiplex a large registry of stateful actors onto a small pool of warm worker pods, paying for compute only while an agent is actively thinking or executing a tool. -To reproduce this demo in your own cluster, please refer to the detailed walkthrough in the **[Counter Demo](demos/counter/README.md)**. +To reproduce this demo in your own cluster, see the **[Counter Demo](demos/counter/README.md)**. For more walkthroughs, visit the **[agent-substrate YouTube channel](https://www.youtube.com/channel/UCN9PPqlTtVxlcpbQ-NWpfZQ)**. -For more videos and walkthroughs, visit our YouTube channel: **[agent-substrate](https://www.youtube.com/channel/UCN9PPqlTtVxlcpbQ-NWpfZQ)**. +## Star History + + + + + Star history chart for agent-substrate/substrate + + ## Framework Agnostic & Compatibility -Agent Substrate is designed to be **framework and agent harness agnostic**. Because it manages standard OCI containers at the kernel level (via gVisor), it can host agents built on any stack. +Agent Substrate is **framework and agent harness agnostic**. Because it runs standard OCI container images inside gVisor or microVM sandboxes and enforces security outside the workload, it hosts agents built on any stack without SDK changes: -* **Agent Development Kit (ADK):** Support for ADK agents with session state preservation across invocations as actor state. Ideal for all types of agents and stateful tool or subagent calls. -* **LangChain:** Ideal execution environment for LangChain agents and tool calls. -* **Claude Code, CodeX, and Antigravity:** Support for high-density, stateful coding environments that preserve system state and filesystem state across sessions. -* **Model Context Protocol (MCP):** Support for deploying secure, sandboxed MCP servers as Substrate Actors to provide durable tools for any model. +* **Agent Development Kit (ADK):** Preserves session state across invocations as actor state. +* **LangChain:** Drop-in execution environment for LangChain agents and tool calls. +* **Claude Code, Codex, and Antigravity:** High-density, stateful coding environments that keep system and filesystem state across sessions while isolating untrusted code execution. +* **Model Context Protocol (MCP):** Deploy sandboxed MCP servers as Substrate Actors with per-actor identity and egress controls. ## Ecosystem & Examples diff --git a/benchmarking/README.md b/benchmarking/README.md index 393e0c33e1..d64187df9c 100644 --- a/benchmarking/README.md +++ b/benchmarking/README.md @@ -20,6 +20,19 @@ scenario ladder, read [observability.md](observability.md). > export BUCKET_NAME="ate-snapshots" > ``` +> [!IMPORTANT] +> The ate-api-server checks every call against +> [authorization](../docs/authorization.md), and Locust calls it as the +> `benchmarking/default` ServiceAccount. Install Substrate with that identity +> as a bootstrap owner, alongside the default `ate-client` owner, or every +> benchmark call is denied: +> ```bash +> ATE_API_AUTHZ_BOOTSTRAP_OWNERS=system:serviceaccount:ate-system:ate-client,spiffe://cluster.local/ns/benchmarking/sa/default \ +> ./hack/install-ate.sh --deploy-ate-system +> ``` +> The nightly automation sets this itself; see +> [automation/README.md](automation/README.md). + Note that deploying the benchmarks does not run them. You must visit Locust's web UI to start a test. @@ -157,7 +170,36 @@ and state restoration latency when a durable directory is attached to the actor. The sweperf benchmark replays a recorded SWE-Perf task inside an actor, suspending and resuming between cycles to measure the cost of actor state transitions under a realistic -agent workload. One task is four cycles by default. +agent workload. One task is four cycles by default. Each task runs on a fresh actor: once +the last cycle finishes, the actor is deleted and the user creates a new one for the next +task. + +#### Sweperf Tasks + +Each task is one entry (name, digest-pinned image, step count) in +`workloads/manifests/sweperf/sweperf-images.json`. Its ActorTemplate, `sweperf-`, +is generated into `workloads/manifests/sweperf/generated/` and checked in. To add or change +a task, edit the catalog and regenerate: + +```bash +SWEPERF=benchmarking/workloads/manifests/sweperf +python3 -m venv "${SWEPERF}/venv" && "${SWEPERF}/venv/bin/pip" install -r "${SWEPERF}/requirements.txt" +"${SWEPERF}/venv/bin/python" "${SWEPERF}/generate_sweperf_templates.py" '*' +"${SWEPERF}/venv/bin/python" "${SWEPERF}/generate_sweperf_templates.py" --check # fail if generated/ is missing or stale +hack/verify/benchmarking-python.sh # unit tests, as CI runs them +``` + +Deploy a task and the sweperf user class: + +```bash +WORKLOAD_TEMPLATES=sweperf-django-11099 ./benchmarking/workloads/deploy.sh --deploy +./benchmarking/locust/deploy.sh --deploy --user-class sweperf +``` + +Then start the run with `--sweperf-template sweperf-django-11099` and +`--sweperf-total-steps 15`, set in the Locust web UI or through `LOCUST_SWEPERF_TEMPLATE` +and `LOCUST_SWEPERF_TOTAL_STEPS`. The client takes the step count from that option, not +from the template, so it must match the catalog. #### Sweperf Reported Metrics @@ -167,7 +209,8 @@ ran inside the sandbox, as reported by `replay.py`. * `ResumeToFirstExec`: Resume RPC start until the sandbox accepts the cycle's `/execute`. * `CycleCEL`: CEL for one cycle. * `TaskCEL`: CEL summed over one task. -* `TaskWallClock`: Client wall clock for one task, excluding inter-cycle think time. +* `TaskWallClock`: Client wall clock for one task, excluding inter-cycle think time and actor + startup. * `CreateAtespace`, `CreateActor`, `ResumeActor`, `SuspendActor`, `DeleteActor`: Server-side elapsed time for each control-plane RPC from the response trailer, or client time without one. * `_rtt`: Client round trip for the RPC of the same name, recorded only when the trailer is @@ -176,9 +219,26 @@ ran inside the sandbox, as reported by `replay.py`. the job finishes. Polled every `--sweperf-poll-interval-ms` (default 100), so this row sits up to one interval above the job's actual end. -The liveness check at session start already has the actor running, so the first cycle's -resume is a no-op. Its successful `ResumeActor`, `ResumeActor_rtt` and `ResumeToFirstExec` -samples are not recorded; failures still are. +The liveness check on a new actor already has it running, so the first cycle of every task +runs on a freshly started actor and its resume is a no-op. Its successful `ResumeActor`, +`ResumeActor_rtt` and `ResumeToFirstExec` samples are not recorded; failures still are. + +The last cycle of a task is not suspended: the actor is deleted straight from running. A +task of `n` cycles therefore records `n - 1` `SuspendActor` samples, and `TaskWallClock` +does not include a final suspend. `DeleteActor` therefore times deleting a running actor, +including its sandbox teardown, so it is not comparable with older runs that suspended the +actor before deleting it. + +`locust_users` counts live actors rather than virtual users: it rises when a user starts an +actor and falls when that actor is deleted, so it dips between tasks while a user waits to +start its next actor. + +`CreateAtespace` is sent once per worker, not per task, and again only if `CreateActor` +reports the atespace missing. + +When the run ends, shutdown cancels cycles still in flight. Calls cut short this way record +nothing, and the interrupted task records no `TaskCEL` or `TaskWallClock` row, so the end of +a run does not show up as a burst of failures. ### Agent-Session Benchmark @@ -547,6 +607,22 @@ map holds the raw facts and the derived numbers side by side. whole cluster, so a separate infrastructure pool is not counted. They are recorded so the ratios below can be re-derived later, or recomputed against a different denominator. +* `kubernetes_version`: the worker nodes' Kubernetes version (kubelet version). +* `worker_actor_capacity`, `worker_memory_limit_gb` (GiB): the capacity each + worker pod declares, its ateworker `--max-actors` (1000 when unset) and its + container memory limit (null when unset, as the node bounds it instead). + `sandbox_class`: `gvisor` or `microvm`, from the worker image. A mixed pool + gives a sorted comma-joined list for each. An unreadable value is skipped + without affecting the others, and logged once. +* `multi_actor_worker`: `true` when `actors_per_pod_p50` is above 1. +* `run_start`, `run_end`: Unix seconds bracketing the run, for finding its + server-side metrics after the cluster is deleted. +* `metadata_`: one per key of the free-form `--metadata ''` object, + for facts only the caller knows. Non-string values are stored as JSON, and + null values are skipped. The orchestrator passes `cluster_name`, + `cluster_location` and `project_id` from the target cluster, plus any keys in + the `BENCHMARK_METADATA` environment variable (a JSON object) and in the + test's own `--metadata` flag, whose keys win on conflict. * `actors_per_node`, `actors_per_vcpu`, `actors_per_gb_ram`: the most actors Locust reported running, over the matching capacity. The `-u` flag only stands in when no sample was read. @@ -572,7 +648,9 @@ comparing numbers across runs: count rather than a resident one. * **The denominators are read once, after the run.** A cluster that autoscaled mid-run is measured at its final size, so the ratio pairs a peak from one - moment with a capacity from another. + moment with a capacity from another. The same holds for every worker and + node fact above, including `kubernetes_version` and the worker capacity, and + so for `multi_actor_worker`, which is derived from `actors_per_pod`. * **The peak assumes every actor is alive at once.** A workload that creates and deletes actors as it goes never holds them all at the same time, so its real density is lower than reported. diff --git a/benchmarking/automation/README.md b/benchmarking/automation/README.md index 9a152df4f0..3849e6ce53 100644 --- a/benchmarking/automation/README.md +++ b/benchmarking/automation/README.md @@ -23,7 +23,9 @@ the router capacity benchmark — see `ateArgs`) + `benchmarking/workloads/deploy.sh --deploy --sandbox-class ` (these build & push substrate / workload images via `ko` as part of their deploy steps — there's no separate - image build step). For a `microvm` test the orchestrator also + image build step). The install sets `ATE_API_AUTHZ_BOOTSTRAP_OWNERS` to + the `ate-client` and `benchmarking/benchmark-runner` ServiceAccounts so the + runner's calls pass [authorization](../../docs/authorization.md). For a `microvm` test the orchestrator also runs `hack/install-microvm-deps.sh --install` between the two, which stages kata + cloud-hypervisor + virtiofsd assets to the cluster's object store bucket and applies the cluster-wide `microvm` SandboxConfig. @@ -38,6 +40,10 @@ the router capacity benchmark — see - Submits a Job using the just-built image for the test's type (`runner-job.yaml.tmpl` for locust, `nighthawk-ingress-runner-job.yaml.tmpl` for nighthawk-ingress). + A locust Job also gets `--metadata` with the target cluster's name, + location and project, merged with the `BENCHMARK_METADATA` environment + variable (a JSON object) if set. A `--metadata` in the test's own + `flags` is merged in too, its keys winning on conflict. - Polls until complete/failed/timeout; tails logs; deletes the Job. - Tears down workloads + micro-VM deps (if any) + substrate. - If not the last test, redeploys them so the next run starts clean. diff --git a/benchmarking/automation/orchestrator.py b/benchmarking/automation/orchestrator.py index 2539b2e33a..f6ed296e92 100644 --- a/benchmarking/automation/orchestrator.py +++ b/benchmarking/automation/orchestrator.py @@ -185,6 +185,63 @@ def test_type(test: dict[str, Any]) -> str: return test["type"] +def _json_object(raw: str, what: str) -> dict[str, Any]: + """Parses raw as a JSON object, naming `what` in the error otherwise.""" + try: + value = json.loads(raw) + except json.JSONDecodeError as e: + raise ValueError(f"{what} is not valid JSON: {e}") from e + if not isinstance(value, dict): + raise ValueError(f"{what} must be a JSON object") + return value + + +def run_metadata() -> dict[str, Any]: + """Free-form run facts handed to the runner as --metadata. + + Read per test, after apply_target_cluster has reset os.environ; + BENCHMARK_METADATA survives that reset because it is part of the startup + env (_ORIG_ENV). + """ + md: dict[str, Any] = { + k.lower(): os.environ[k] + for k in ("CLUSTER_NAME", "CLUSTER_LOCATION", "PROJECT_ID") + if os.environ.get(k) + } + raw = os.environ.get("BENCHMARK_METADATA") + if raw: + md.update(_json_object(raw, "BENCHMARK_METADATA")) + return md + + +def split_metadata_flag(test: dict[str, Any]) -> tuple[list[Any], dict[str, Any]]: + """Separates a test's own --metadata from the rest of its flags. + + Accepts `--metadata ` and `--metadata=`. With more than one, + later keys win. Raises ValueError when the value is missing or is not a + JSON object, rather than dropping it. + """ + name = test.get("name") + rest: list[Any] = [] + md: dict[str, Any] = {} + flags = list(test.get("flags", [])) + i = 0 + while i < len(flags): + flag = str(flags[i]) + if flag == "--metadata": + if i + 1 >= len(flags): + raise ValueError(f"test {name!r}: --metadata needs a JSON value") + raw, i = str(flags[i + 1]), i + 2 + elif flag.startswith("--metadata="): + raw, i = flag.split("=", 1)[1], i + 1 + else: + rest.append(flags[i]) + i += 1 + continue + md.update(_json_object(raw, f"test {name!r}: --metadata")) + return rest, md + + def wait_for_docker(timeout: int = 120) -> None: print("Waiting for DIND sidecar...", flush=True) start = time.time() @@ -326,6 +383,9 @@ def validate_and_normalize_tests(tests: list[dict[str, Any]]) -> None: f"(want one of {list(TEST_TYPES)})" ) TYPES[ttype].validate(t) + if ttype == "locust": + # A malformed --metadata fails here rather than mid-sweep. + split_metadata_flag(t) isolate_block(t) @@ -337,8 +397,21 @@ def substrate_args(test: dict[str, Any]) -> list[str]: return [*test.get("ateArgs", []), *isolate_install_args(test)] +# ate-api-server enforces authorization, and the runner Job calls it with its +# pod certificate as BENCHMARK_RUNNER_PRINCIPAL. Each test reinstalls +# Substrate, so making the runner a bootstrap owner on every install grants it +# global owner for exactly the life of that install. kubectl-ate's principal +# stays an owner for the workload and teardown steps. +BENCHMARK_RUNNER_PRINCIPAL = "spiffe://cluster.local/ns/benchmarking/sa/benchmark-runner" +CLIENT_PRINCIPAL = "system:serviceaccount:ate-system:ate-client" + + def deploy_substrate(ate_args: Iterable[str] = ()) -> None: - run(["hack/install-ate.sh", "--deploy-ate-system", *(str(a) for a in ate_args)]) + env = { + **os.environ, + "ATE_API_AUTHZ_BOOTSTRAP_OWNERS": f"{CLIENT_PRINCIPAL},{BENCHMARK_RUNNER_PRINCIPAL}", + } + run(["hack/install-ate.sh", "--deploy-ate-system", *(str(a) for a in ate_args)], env=env) def teardown_substrate() -> None: @@ -366,6 +439,7 @@ def deploy_workloads( wait_timeout_secs: int | str = "", worker_memory: str = "", fake_atelet_args: Iterable[str] = (), + actor_cpu: str = "", ) -> None: cmd = [ "benchmarking/workloads/deploy.sh", @@ -379,6 +453,11 @@ def deploy_workloads( # minimum); RAM-consuming suites set actorMemory in tests.yaml. if actor_memory: cmd += ["--actor-memory", actor_memory] + # Empty keeps the default in workloads/deploy.sh (1 CPU). The scheduler + # books the limit against worker CPU capacity, so dense suites set + # actorCPU in tests.yaml to fit more actors per worker. + if actor_cpu: + cmd += ["--actor-cpu", actor_cpu] # Empty leaves the worker pods unsized; set workerMemory in tests.yaml to # control how many workers the scheduler packs onto a node. if worker_memory: @@ -416,7 +495,14 @@ def run_test( mod = TYPES[test_type(test)] tmpl = mod.job_tmpl(manifests_dir) subs.update(mod.job_subs(test)) - manifest = render_template(tmpl, subs, test.get("flags", [])) + flags = list(test.get("flags", [])) + # Only the locust runner knows --metadata. It keeps only the last one it + # is given, so a test's own --metadata is folded into a single merged + # flag; the test's keys win over the run's. + if test_type(test) == "locust": + flags, own = split_metadata_flag(test) + flags += ["--metadata", json.dumps({**run_metadata(), **own})] + manifest = render_template(tmpl, subs, flags) wait_for_no_active_runners() print(f"Submitting Job {job}", flush=True) subprocess.run( @@ -498,6 +584,8 @@ def main() -> None: print(f"Running {len(tests)} test(s)", flush=True) try: validate_and_normalize_tests(tests) + # Checks BENCHMARK_METADATA only; cluster vars may not be set yet. + run_metadata() except ValueError as e: sys.exit(str(e)) @@ -566,6 +654,7 @@ def main() -> None: test.get("workerWaitTimeout", ""), test.get("workerMemory", ""), isolate_workload_args(test), + test.get("actorCPU", ""), ) try: status = run_test( diff --git a/benchmarking/automation/test_orchestrator.py b/benchmarking/automation/test_orchestrator.py index b85c368107..004785f011 100644 --- a/benchmarking/automation/test_orchestrator.py +++ b/benchmarking/automation/test_orchestrator.py @@ -14,6 +14,7 @@ """Unit tests for orchestrator.py: python3 benchmarking/automation/test_orchestrator.py""" +import json import os import unittest from unittest import mock @@ -59,6 +60,30 @@ def test_bare_isolate_key(self): self.assertEqual(orchestrator.substrate_args({"isolate": None}), ["--fake-data-plane"]) +class DeploySubstrateTest(unittest.TestCase): + # The runner Job authenticates with its pod certificate, so ate-api-server + # denies it unless the install makes it an owner. + @mock.patch("orchestrator.run") + def test_runner_and_client_are_bootstrap_owners(self, run): + orchestrator.deploy_substrate(["--credential-provider={}"]) + cmd = run.call_args.args[0] + self.assertEqual(cmd, ["hack/install-ate.sh", "--deploy-ate-system", "--credential-provider={}"]) + owners = run.call_args.kwargs["env"]["ATE_API_AUTHZ_BOOTSTRAP_OWNERS"].split(",") + self.assertEqual( + owners, + [ + "system:serviceaccount:ate-system:ate-client", + "spiffe://cluster.local/ns/benchmarking/sa/benchmark-runner", + ], + ) + + @mock.patch("orchestrator.run") + def test_keeps_the_rest_of_the_environment(self, run): + with mock.patch.dict(os.environ, {"KO_DOCKER_REPO": "example"}): + orchestrator.deploy_substrate() + self.assertEqual(run.call_args.kwargs["env"]["KO_DOCKER_REPO"], "example") + + class DeployWorkloadsTest(unittest.TestCase): @mock.patch("orchestrator.run") def test_defaults_omit_optional_flags(self, run): @@ -89,6 +114,7 @@ def test_all_options(self, run): actor_memory="1536Mi", wait_timeout_secs=600, worker_memory="5Gi", + actor_cpu="250m", ) run.assert_called_once_with( [ @@ -100,6 +126,8 @@ def test_all_options(self, run): "microvm", "--actor-memory", "1536Mi", + "--actor-cpu", + "250m", "--worker-memory", "5Gi", "--wait-timeout", @@ -199,5 +227,88 @@ def test_two_runs_of_the_same_long_test_differ(self): ) +CLUSTER_ENV = {"CLUSTER_NAME": "c", "CLUSTER_LOCATION": "l", "PROJECT_ID": "p"} + + +class RunMetadataTest(unittest.TestCase): + def test_cluster_env(self): + with mock.patch.dict(os.environ, CLUSTER_ENV, clear=True): + self.assertEqual( + orchestrator.run_metadata(), + {"cluster_name": "c", "cluster_location": "l", "project_id": "p"}, + ) + + def test_benchmark_metadata_is_merged(self): + env = {**CLUSTER_ENV, "BENCHMARK_METADATA": '{"build_id": "1"}'} + with mock.patch.dict(os.environ, env, clear=True): + self.assertEqual(orchestrator.run_metadata()["build_id"], "1") + + def test_bad_benchmark_metadata(self): + for bad in ("not json", "[1]", '"text"'): + env = {**CLUSTER_ENV, "BENCHMARK_METADATA": bad} + with self.subTest(bad=bad), mock.patch.dict(os.environ, env, clear=True), \ + self.assertRaisesRegex(ValueError, "BENCHMARK_METADATA"): + orchestrator.run_metadata() + + def submitted_args(self, ttype, flags=("--x",)): + """The extra args run_test hands to render_template for a test type.""" + + class Rendered(Exception): + pass + + fake = mock.Mock(job_tmpl=mock.Mock(return_value="t"), job_subs=mock.Mock(return_value={})) + test = {"name": "n", "type": ttype, "duration": "1m", "flags": list(flags)} + with mock.patch.dict(orchestrator.TYPES, {ttype: fake}), \ + mock.patch.dict(os.environ, CLUSTER_ENV, clear=True), \ + mock.patch.object(orchestrator, "render_template", side_effect=Rendered) as render, \ + self.assertRaises(Rendered): + orchestrator.run_test(test, "img", "dest", "abcdef0123") + return render.call_args.args[2] + + def test_locust_job_gets_metadata(self): + args = self.submitted_args("locust") + self.assertEqual(args[:2], ["--x", "--metadata"]) + self.assertEqual(json.loads(args[2])["cluster_name"], "c") + + def test_other_types_do_not(self): + self.assertEqual(self.submitted_args("nighthawk-ingress"), ["--x"]) + + def test_test_metadata_is_merged_not_overridden(self): + for own in (["--metadata", '{"cluster_name": "mine", "suite": "s"}'], + ['--metadata={"cluster_name": "mine", "suite": "s"}']): + with self.subTest(own=own): + args = self.submitted_args("locust", ["--x", *own, "--y"]) + # One merged --metadata, after the test's other flags. + self.assertEqual(args.count("--metadata"), 1) + self.assertFalse([a for a in args if a.startswith("--metadata=")]) + self.assertEqual(args[:3], ["--x", "--y", "--metadata"]) + md = json.loads(args[3]) + # Test keys win; run keys the test does not set are kept. + self.assertEqual(md["cluster_name"], "mine") + self.assertEqual(md["suite"], "s") + self.assertEqual(md["project_id"], "p") + + def test_bad_test_metadata_fails_clearly(self): + for own in (["--metadata", "not json"], ["--metadata=[1]"], ["--metadata"]): + with self.subTest(own=own), \ + self.assertRaisesRegex(ValueError, "test 'n': --metadata"): + orchestrator.split_metadata_flag({"name": "n", "flags": own}) + + # Caught by validation, before any cluster work. + test = {"name": "n", "type": "locust", "targetCluster": "t", + "file": "f", "duration": "1m", "users": 1, + "flags": ["--metadata", "{bad"]} + with self.assertRaisesRegex(ValueError, "not valid JSON"): + orchestrator.validate_and_normalize_tests([test]) + + def test_metadata_survives_yaml(self): + md = json.dumps({"cluster_name": "c: d", "q": "\"'"}) + subs = {"JOB_NAME": "j", "IMAGE": "i", "TAG": "t", "NAME": "n", "DEST": "d"} + subs.update(locust.job_subs({"file": "f", "duration": "1m", "users": 1})) + text = orchestrator.render_template(RunnerSizingTest.TMPL, subs, ["--metadata", md]) + job = next(d for d in yaml.safe_load_all(text) if d and d.get("kind") == "Job") + self.assertEqual(job["spec"]["template"]["spec"]["containers"][0]["args"][-1], md) + + if __name__ == "__main__": unittest.main() diff --git a/benchmarking/automation/tests.yaml b/benchmarking/automation/tests.yaml index b59f3734a8..6bc63d6495 100644 --- a/benchmarking/automation/tests.yaml +++ b/benchmarking/automation/tests.yaml @@ -35,6 +35,10 @@ # actorMemory Optional (default "256Mi", the smallest size microvm # admits). Memory limit on the benchmark ActorTemplates; # raise it for suites that exercise glutton's WriteRAM path. +# actorCPU Optional (default "1"). CPU limit on the benchmark +# ActorTemplates. The scheduler books it against each +# worker's CPU capacity, so it caps the actors a worker +# hosts at once; lower it for dense suites. # workerMemory Optional (default unset: the worker pods have no requests # or limits). Memory request and limit on each WorkerPool # pod. Use it to fix how many workers share a node: one per diff --git a/benchmarking/deploy_locust.sh b/benchmarking/deploy_locust.sh index a431870a76..fb4d335bdb 100755 --- a/benchmarking/deploy_locust.sh +++ b/benchmarking/deploy_locust.sh @@ -31,6 +31,8 @@ SKIP_BUILD=0 OTLP_ENDPOINT="" # Empty keeps the default in workloads/deploy.sh (256Mi, the microvm minimum). ACTOR_MEMORY="" +# Empty keeps the default in workloads/deploy.sh (1 CPU). +ACTOR_CPU="" # Empty leaves the WorkerPool pods unsized. WORKER_MEMORY="" STORAGE_CLASS_NAME="" @@ -49,6 +51,8 @@ usage() { echo " instrumented actor container sends telemetry." echo " --actor-memory SIZE Forwarded to workloads/deploy.sh. Memory limit for the" echo " benchmark ActorTemplates (default: 256Mi, the microvm minimum)." + echo " --actor-cpu CPU Forwarded to workloads/deploy.sh. CPU limit for the benchmark" + echo " ActorTemplates (default: 1)." echo " --worker-memory SIZE Forwarded to workloads/deploy.sh. Memory request and limit" echo " for each WorkerPool pod (default: unset, the pod is unsized)." echo " --storage-class-name S Forwarded to workloads/deploy.sh. StorageClass for the glutton-storage" @@ -81,6 +85,8 @@ while [[ "$#" -gt 0 ]]; do --otlp-endpoint=*) OTLP_ENDPOINT="${1#*=}" ;; --actor-memory) shift; ACTOR_MEMORY="$1" ;; --actor-memory=*) ACTOR_MEMORY="${1#*=}" ;; + --actor-cpu) shift; ACTOR_CPU="$1" ;; + --actor-cpu=*) ACTOR_CPU="${1#*=}" ;; --worker-memory) shift; WORKER_MEMORY="$1" ;; --worker-memory=*) WORKER_MEMORY="${1#*=}" ;; --storage-class-name) shift; STORAGE_CLASS_NAME="$1" ;; @@ -123,6 +129,9 @@ if [[ "${action}" == "deploy" ]]; then if [[ -n "${ACTOR_MEMORY}" ]]; then workload_args+=(--actor-memory "${ACTOR_MEMORY}") fi + if [[ -n "${ACTOR_CPU}" ]]; then + workload_args+=(--actor-cpu "${ACTOR_CPU}") + fi if [[ -n "${WORKER_MEMORY}" ]]; then workload_args+=(--worker-memory "${WORKER_MEMORY}") fi diff --git a/benchmarking/locust/cluster_facts.py b/benchmarking/locust/cluster_facts.py index 449b31f8ee..8bb03d9a82 100644 --- a/benchmarking/locust/cluster_facts.py +++ b/benchmarking/locust/cluster_facts.py @@ -17,7 +17,9 @@ Reads allocatable CPU/RAM, node count and worker pod count from the Kubernetes API, then derives the actor-density frontiers (actors per node / vCPU / GB RAM -and the actors-per-pod percentiles) for a completed trial. +and the actors-per-pod percentiles) for a completed trial. Also records the +nodes' Kubernetes version, the worker pods' declared capacity and sandbox +class, the run window, and any caller-supplied --metadata. """ import argparse @@ -36,15 +38,26 @@ LIVE_POD_PHASES = ("Running", "Pending") MACHINE_TYPE_LABEL = "node.kubernetes.io/instance-type" +# The worker pod's container, as named by the atecontroller. +WORKER_CONTAINER = "worker" +# ateworker's own --max-actors default (cmd/ateworker-gvisor, cmd/ateworker-microvm). +ATEWORKER_DEFAULT_MAX_ACTORS = 1000 +# The worker image is ateworker- (benchmarking/automation/README.md). +SANDBOX_CLASS_IMAGE = re.compile(r"ateworker-(gvisor|microvm)") + # Shape returned when the cluster cannot be read, or when discovery is skipped # with --no-cluster-facts. Keeping one definition means a trial_summary row has -# the same hardware keys either way, so consumers never have to special-case it. +# the same fact keys either way, so consumers never have to special-case it. EMPTY_FACTS: dict[str, Any] = { "machine_type": None, "node_count": None, "allocatable_cores": None, "allocatable_ram_gb": None, "worker_pod_count": None, + "kubernetes_version": None, + "worker_actor_capacity": None, + "worker_memory_limit_gb": None, + "sandbox_class": None, } @@ -101,8 +114,80 @@ def _list_worker_pods( return None +def _joined(values: set[Any]) -> str | None: + """A single value, or a mixed pool's values sorted and comma-joined.""" + return ",".join(str(v) for v in sorted(values)) or None + + +def _max_actors(args: list[str]) -> int | None: + """The --max-actors value on an ateworker command line, or ateworker's default. + + Returns None when the flag is present but its value is not an integer. + """ + for i, arg in enumerate(args): + value = None + if arg.startswith("--max-actors="): + value = arg.split("=", 1)[1] + elif arg == "--max-actors" and i + 1 < len(args): + value = args[i + 1] + if value is not None: + try: + return int(value) + except ValueError: + return None + return ATEWORKER_DEFAULT_MAX_ACTORS + + +def _worker_container_facts( + pods: list[Any], logs: TextIO | None = None +) -> dict[str, str | None]: + """Reads the capacity and sandbox class each worker pod declares. + + A worker reports its capacity as --max-actors and its container memory + limit. With no memory limit set the worker is bounded by its node instead, + so the limit stays None. + + Each fact is read on its own, so a malformed value skips only that value + and leaves the other facts intact. + """ + capacities: set[int] = set() + memory_gb: set[float] = set() + sandbox_classes: set[str] = set() + # Logged once rather than per pod, so a large pool does not flood the log. + unreadable: set[str] = set() + for p in pods: + for c in p.spec.containers or []: + if c.name != WORKER_CONTAINER: + continue + capacity = _max_actors(c.args or []) + if capacity is None: + unreadable.add("--max-actors") + else: + capacities.add(capacity) + limits = (c.resources.limits if c.resources else None) or {} + if "memory" in limits: + try: + # GiB, the same as allocatable_ram_gb. + memory_gb.add(round( + int(parse_quantity(limits["memory"])) / (1024**3), 2)) + except ValueError: + unreadable.add(f"memory limit {limits['memory']!r}") + match = SANDBOX_CLASS_IMAGE.search(c.image or "") + if match: + sandbox_classes.add(match.group(1)) + if unreadable: + _log(logs, "Notice: skipped unreadable worker container values: " + + ", ".join(sorted(unreadable))) + return { + "worker_actor_capacity": _joined(capacities), + "worker_memory_limit_gb": _joined(memory_gb), + "sandbox_class": _joined(sandbox_classes), + } + + def get_cluster_hardware_facts(logs: TextIO | None = None) -> dict[str, Any]: - """Reads the worker pool size and the capacity of the nodes it runs on. + """Reads the worker pool size, the capacity and Kubernetes version of the + nodes it runs on, and what each worker pod declares. Capacity is scoped to the nodes carrying worker pods, so a cluster that keeps its infrastructure on a separate pool does not count that pool's @@ -126,6 +211,12 @@ def get_cluster_hardware_facts(logs: TextIO | None = None) -> dict[str, Any]: facts["worker_pod_count"] = len(pods) + # Malformed values are skipped inside; this guards an unexpected pod shape. + try: + facts.update(_worker_container_facts(pods, logs)) + except Exception as e: + _log(logs, f"Notice: could not read worker container specs: {e}") + try: # A Pending pod may not be scheduled yet, so it counts toward the pool # size without contributing a node. @@ -139,6 +230,7 @@ def get_cluster_hardware_facts(logs: TextIO | None = None) -> dict[str, Any]: total_cores = 0.0 total_ram_bytes = 0 machine_types = set() + versions = set() for node in nodes: metadata = node.metadata if metadata is None or metadata.name not in worker_nodes: @@ -150,13 +242,17 @@ def get_cluster_hardware_facts(logs: TextIO | None = None) -> dict[str, Any]: machine_type = (metadata.labels or {}).get(MACHINE_TYPE_LABEL) if machine_type: machine_types.add(machine_type) + if node.status.node_info and node.status.node_info.kubelet_version: + versions.add(node.status.node_info.kubelet_version) facts["node_count"] = node_count facts["allocatable_cores"] = round(total_cores, 2) # GiB, as the apiserver and kubectl quote it. facts["allocatable_ram_gb"] = round(total_ram_bytes / (1024**3), 2) # Kept so results stay comparable across hardware changes. A mixed pool # is a sorted comma-joined list rather than one node picked at random. - facts["machine_type"] = ",".join(sorted(machine_types)) or None + facts["machine_type"] = _joined(machine_types) + # The worker nodes' Kubernetes version, e.g. v1.33.5-gke.1080000. + facts["kubernetes_version"] = _joined(versions) except Exception as e: reason = getattr(e, "reason", e) _log(logs, f"Notice: could not read node capacity: {reason}") @@ -172,6 +268,8 @@ def append_trial_summary( data_ts: str, facts: dict[str, Any], logs: TextIO | None = None, + run_start: int | None = None, + run_end: int | None = None, ) -> None: # Locust's own User Count samples. The -u flag is a request; under a custom # load shape what actually ran is whatever the shape asked for. @@ -249,8 +347,26 @@ def append_trial_summary( else: _log(logs, f"Notice: {stats_csv} not found; failure ratios unknown") + # Whether workers hosted more than one actor at once (a multi-actor + # worker), going by the median actors per pod. + multi_actor_worker = ( + None if actors_per_pod_p50 is None + else str(actors_per_pod_p50 > 1).lower() + ) + measurements = { **{k: facts.get(k) for k in EMPTY_FACTS}, + # Unix seconds bracketing the run, for finding its server-side metrics + # in Cloud Monitoring. + "run_start": run_start, + "run_end": run_end, + # Caller-supplied --metadata, one key each so values stay plain strings. + **{ + f"metadata_{k}": v if isinstance(v, str) else json.dumps(v) + for k, v in (getattr(args, "metadata", None) or {}).items() + if v is not None + }, + "multi_actor_worker": multi_actor_worker, "actors_per_node": actors_per_node, "actors_per_vcpu": actors_per_vcpu, "actors_per_gb_ram": actors_per_gb_ram, diff --git a/benchmarking/locust/common/sweperf_config.py b/benchmarking/locust/common/sweperf_config.py index ddb90dea2a..e4810ca870 100644 --- a/benchmarking/locust/common/sweperf_config.py +++ b/benchmarking/locust/common/sweperf_config.py @@ -23,9 +23,9 @@ def add_sweperf_arguments(parser: LocustArgumentParser) -> None: parser.add_argument( "--sweperf-template", type=str, - default="swebench-astropy-7336", + default="sweperf-astropy-7336", env_var="LOCUST_SWEPERF_TEMPLATE", - help="ActorTemplate name for the sweperf workload (default: swebench-astropy-7336)", + help="ActorTemplate name for the sweperf workload (default: sweperf-astropy-7336)", include_in_web_ui=True, ) parser.add_argument( diff --git a/benchmarking/locust/runner.py b/benchmarking/locust/runner.py index 42cdcc8ddb..5f94e78dc2 100644 --- a/benchmarking/locust/runner.py +++ b/benchmarking/locust/runner.py @@ -80,6 +80,17 @@ ) +def json_object(value: str) -> dict[str, Any]: + """argparse type for --metadata: a JSON object, checked before the run.""" + try: + obj = json.loads(value) + except json.JSONDecodeError as e: + raise argparse.ArgumentTypeError(f"not valid JSON: {e}") from e + if not isinstance(obj, dict): + raise argparse.ArgumentTypeError("must be a JSON object") + return obj + + def parse_args() -> argparse.Namespace: p = argparse.ArgumentParser(description=__doc__) p.add_argument("-f", required=True, dest="file", help="Locust test file (-f)") @@ -147,6 +158,15 @@ def parse_args() -> argparse.Namespace: "--no-cluster-facts to skip Kubernetes API discovery" ), ) + p.add_argument( + "--metadata", + type=json_object, + default={}, + help=( + "Free-form JSON object describing the run, such as where it ran. " + "Each key is recorded in trial_summary as metadata_" + ), + ) p.add_argument( "--prometheus-url", default=DEFAULT_PROMETHEUS_URL, @@ -490,7 +510,7 @@ def upload(src: Path, dest: str) -> None: def collect_cluster_facts( args: argparse.Namespace, logs: TextIO ) -> dict[str, Any]: - """Returns cluster hardware facts, or empty facts when discovery is off.""" + """Returns cluster facts, or empty facts when discovery is off.""" if not args.cluster_facts: tee(logs, "Skipping cluster hardware discovery (--no-cluster-facts)") return dict(EMPTY_FACTS) @@ -576,6 +596,8 @@ def main() -> None: data_ts, facts, logs, + run_start=run_ts, + run_end=run_end_ts, ) except Exception as e: tee(logs, f"Warning: Failed to record cluster facts: {e}") diff --git a/benchmarking/locust/unit_tests/test_cluster_facts.py b/benchmarking/locust/unit_tests/test_cluster_facts.py index 21ef770291..713748b009 100644 --- a/benchmarking/locust/unit_tests/test_cluster_facts.py +++ b/benchmarking/locust/unit_tests/test_cluster_facts.py @@ -51,17 +51,30 @@ "--tag", "unit", "--name", "unit-run", "--dest", "/tmp/unit"] +KUBERNETES_VERSION = "v1.33.5-gke.1080000" + + def node(machine_type="c3-standard-4", name="node-a"): labels = {} if machine_type is None else { cluster_facts.MACHINE_TYPE_LABEL: machine_type} return SimpleNamespace( metadata=SimpleNamespace(labels=labels, name=name), - status=SimpleNamespace(allocatable={"cpu": NODE_CPU, "memory": NODE_MEMORY})) + status=SimpleNamespace( + allocatable={"cpu": NODE_CPU, "memory": NODE_MEMORY}, + node_info=SimpleNamespace(kubelet_version=KUBERNETES_VERSION))) + +def container(name="worker", args=(), memory=None, + image="registry/ateworker-gvisor-0123abcd@sha256:00"): + limits = None if memory is None else {"memory": memory} + return SimpleNamespace(name=name, args=list(args), image=image, + resources=SimpleNamespace(limits=limits)) -def pod(phase="Running", node_name="node-a"): - return SimpleNamespace(spec=SimpleNamespace(node_name=node_name), - status=SimpleNamespace(phase=phase)) + +def pod(phase="Running", node_name="node-a", containers=()): + return SimpleNamespace( + spec=SimpleNamespace(node_name=node_name, containers=list(containers)), + status=SimpleNamespace(phase=phase)) def fake_api(nodes=None, ns_pods=None): @@ -86,7 +99,8 @@ def discover(api): def summarize(facts, directory, stats=STATS_HEADER + ",Aggregated,100,25\n", - users=10, user_counts=None, actors_per_user=None): + users=10, user_counts=None, actors_per_user=None, + run_start=None, run_end=None, metadata=None): """Writes CSV inputs and returns the emitted trial_summary row.""" d = Path(directory) if stats is not None: @@ -102,8 +116,9 @@ def summarize(facts, directory, stats=STATS_HEADER + ",Aggregated,100,25\n", cluster_facts.append_trial_summary( out, d / "stats.csv", d / "stats_history.csv", argparse.Namespace(users=users, tag="unit", name="unit-run", - actors_per_user=actors_per_user), - "2026-01-01", facts) + actors_per_user=actors_per_user, + metadata=metadata), + "2026-01-01", facts, run_start=run_start, run_end=run_end) return json.loads(out.read_text().splitlines()[0]) @@ -195,8 +210,127 @@ def tripwire(*_args, **_kwargs): with mock.patch.object(runner, "get_cluster_hardware_facts", return_value={"node_count": 1}) as discovery, \ contextlib.redirect_stdout(io.StringIO()): - runner.collect_cluster_facts(parse(), io.StringIO()) + facts = runner.collect_cluster_facts(parse(), io.StringIO()) discovery.assert_called_once() + self.assertEqual(facts, {"node_count": 1}) + + def test_worker_container_facts(self): + facts = discover(fake_api(nodes=[node()], ns_pods=[ + pod(containers=[container(args=["--pod-uid=x", "--max-actors=50"], + memory="16Gi"), + container(name="sidecar", args=["--max-actors=7"])]), + pod(containers=[container(args=["--max-actors", "50"], + memory="17179869184")]), + ])) + self.assertEqual(facts["worker_actor_capacity"], "50") + self.assertEqual(facts["worker_memory_limit_gb"], "16.0") # GiB + self.assertEqual(facts["sandbox_class"], "gvisor") + self.assertEqual(facts["kubernetes_version"], KUBERNETES_VERSION) + + # Unset flag is ateworker's default; no memory limit is unmeasured; a mixed + # pool lists every value. + facts = discover(fake_api(nodes=[node()], ns_pods=[ + pod(containers=[container()]), + pod(containers=[container( + args=["--max-actors=1"], + image="registry/ateworker-microvm-4567@sha256:00")]), + ])) + self.assertEqual(facts["worker_actor_capacity"], "1,1000") + self.assertIsNone(facts["worker_memory_limit_gb"]) + self.assertEqual(facts["sandbox_class"], "gvisor,microvm") + + # No worker pods -> nothing to read. + facts = discover(fake_api(nodes=[node()], ns_pods=[])) + self.assertIsNone(facts["worker_actor_capacity"]) + self.assertIsNone(facts["sandbox_class"]) + + def test_worker_facts_fail_independently(self): + # A bad --max-actors still yields the memory limit, sandbox class and + # node capacity. + for args in (["--max-actors=lots"], ["--max-actors", "lots"]): + with self.subTest(args=args): + facts = discover(fake_api(nodes=[node()], ns_pods=[ + pod(containers=[container(args=args, memory="16Gi")])])) + self.assertIsNone(facts["worker_actor_capacity"]) + self.assertEqual(facts["worker_memory_limit_gb"], "16.0") + self.assertEqual(facts["sandbox_class"], "gvisor") + self.assertEqual(facts["node_count"], 1) + + # A bad memory quantity still yields the capacity and sandbox class. + facts = discover(fake_api(nodes=[node()], ns_pods=[ + pod(containers=[container(args=["--max-actors=50"], + memory="lots")])])) + self.assertEqual(facts["worker_actor_capacity"], "50") + self.assertIsNone(facts["worker_memory_limit_gb"]) + self.assertEqual(facts["sandbox_class"], "gvisor") + + # Only the bad pod's value is skipped; a healthy pod still counts. + facts = discover(fake_api(nodes=[node()], ns_pods=[ + pod(containers=[container(args=["--max-actors=lots"], + memory="lots")]), + pod(containers=[container(args=["--max-actors=50"], + memory="16Gi")])])) + self.assertEqual(facts["worker_actor_capacity"], "50") + self.assertEqual(facts["worker_memory_limit_gb"], "16.0") + + def test_unreadable_worker_values_are_logged_once(self): + logs = io.StringIO() + bad = pod(containers=[container(args=["--max-actors=lots"], + memory="lots")]) + with contextlib.redirect_stdout(io.StringIO()): + cluster_facts._worker_container_facts([bad, bad, bad], logs) + notices = logs.getvalue().splitlines() + self.assertEqual(len(notices), 1) + self.assertIn("--max-actors", notices[0]) + self.assertIn("'lots'", notices[0]) + + def test_metadata_flag(self): + self.assertEqual(parse().metadata, {}) + got = parse("--metadata", '{"cluster_name": "c", "nodes": 3}') + self.assertEqual(got.metadata, {"cluster_name": "c", "nodes": 3}) + self.assertNotIn("--metadata", got.locust_extra) + + # Rejected before the run starts, not after it. + for bad in ("not json", "[1, 2]", '"text"'): + with self.subTest(bad=bad), \ + contextlib.redirect_stderr(io.StringIO()), \ + self.assertRaises(SystemExit): + parse("--metadata", bad) + + def test_metadata_in_trial_summary(self): + md = {"cluster_name": "sweperf-c3hm176nested", "nodes": 3, + "labels": {"a": 1}, "unset": None} + with tempfile.TemporaryDirectory() as td: + m = summarize(FACTS, td, metadata=md)["measurements"] + self.assertEqual(m["metadata_cluster_name"], "sweperf-c3hm176nested") + # Non-strings are stored as JSON so BigQuery can parse them. + self.assertEqual(m["metadata_nodes"], "3") + self.assertEqual(json.loads(m["metadata_labels"]), {"a": 1}) + self.assertNotIn("metadata_unset", m) + + # No metadata, no metadata_* keys. + with tempfile.TemporaryDirectory() as td: + m = summarize(FACTS, td)["measurements"] + self.assertFalse([k for k in m if k.startswith("metadata_")]) + + def test_run_window_and_multi_actor_worker(self): + facts = {**FACTS, "worker_actor_capacity": 1000} + with tempfile.TemporaryDirectory() as td: + m = summarize(facts, td, run_start=1788914580, + run_end=1788915180)["measurements"] + self.assertEqual(m["run_start"], "1788914580") + self.assertEqual(m["run_end"], "1788915180") + self.assertEqual(m["worker_actor_capacity"], "1000") + self.assertEqual(m["multi_actor_worker"], "true") # 2 actors per pod + + # One actor per pod is not multi-actor; no pod count is unknown. + with tempfile.TemporaryDirectory() as td: + m = summarize({**FACTS, "worker_pod_count": 10}, td)["measurements"] + self.assertEqual(m["multi_actor_worker"], "false") # 10 users / 10 pods + self.assertIsNone(m["run_start"]) + with tempfile.TemporaryDirectory() as td: + m = summarize(dict(cluster_facts.EMPTY_FACTS), td)["measurements"] + self.assertIsNone(m["multi_actor_worker"]) def test_trial_summary(self): with tempfile.TemporaryDirectory() as td: diff --git a/benchmarking/nighthawk-egress/traffic-driver.yaml.tmpl b/benchmarking/nighthawk-egress/traffic-driver.yaml.tmpl index 3e08db74af..478dd935eb 100644 --- a/benchmarking/nighthawk-egress/traffic-driver.yaml.tmpl +++ b/benchmarking/nighthawk-egress/traffic-driver.yaml.tmpl @@ -33,6 +33,6 @@ metadata: workload: traffic-driver spec: replicas: 1 - workerImage: ko://github.com/agent-substrate/substrate/cmd/ateworker-gvisor + workerImage: ko://go.ate.dev/substrate/cmd/ateworker-gvisor sandboxClasses: - name: gvisor diff --git a/benchmarking/workloads/deploy.sh b/benchmarking/workloads/deploy.sh index 9fe68389bc..59c4d15b21 100755 --- a/benchmarking/workloads/deploy.sh +++ b/benchmarking/workloads/deploy.sh @@ -31,14 +31,25 @@ fi MANIFEST_DIR="benchmarking/workloads/manifests" POOL_MANIFEST="${MANIFEST_DIR}/workloads.yaml.tmpl" +# Generated sweperf ActorTemplates; see manifests/sweperf/generate_sweperf_templates.py. +SWEPERF_TEMPLATE_DIR="${MANIFEST_DIR}/sweperf/generated" # The benchmark ActorTemplates: -template.yaml.tmpl each, created -# through the ate API in the benchmark-workloads atespace. WORKLOAD_TEMPLATES +# through the ate API in the benchmark-workloads atespace. sweperf-* names are +# read from SWEPERF_TEMPLATE_DIR, the rest from MANIFEST_DIR. WORKLOAD_TEMPLATES # overrides the default set — the usermem and kernelmem templates (for the # matching locust tests) are not deployed by default. The glutton-storage # template is added when a StorageClass is set, because it needs a CSI driver # that not every cluster has. read -r -a TEMPLATES <<<"${WORKLOAD_TEMPLATES:-sleep glutton glutton-durdir-data glutton-durdir-full}" +# template_manifest prints the manifest path for ActorTemplate NAME. +template_manifest() { + case "$1" in + sweperf-*) echo "${SWEPERF_TEMPLATE_DIR}/$1-template.yaml.tmpl" ;; + *) echo "${MANIFEST_DIR}/$1-template.yaml.tmpl" ;; + esac +} + if [[ ! -f "${POOL_MANIFEST}" ]]; then echo "Error: ${POOL_MANIFEST} not found in $(pwd)" >&2 exit 1 @@ -51,6 +62,10 @@ SANDBOX_CLASS="gvisor" # since a bigger guest drags more page cache into every memory snapshot. Raise # it for RAM-consuming suites. ACTOR_MEMORY="256Mi" +# Actor CPU limit (ActorTemplate resources.limits.cpu). The scheduler books +# it against the worker's CPU capacity, so it bounds how many actors a worker +# hosts at once as much as the memory limit does. +ACTOR_CPU="1" # Worker pod memory (WorkerPool spec.template.resources), set as both request # and limit. Empty leaves the pod unsized. The limit is also the memory the # worker reports as its actor capacity. @@ -80,6 +95,7 @@ usage() { echo " microvm requires hack/install-microvm-deps.sh --install to have run." echo " --actor-memory SIZE Memory limit for the benchmark ActorTemplates (default: 256Mi," echo " the smallest size microvm admits)" + echo " --actor-cpu CPU CPU limit for the benchmark ActorTemplates (default: 1)" echo " --worker-memory SIZE Memory request and limit for each WorkerPool pod" echo " (default: unset, the pod is unsized)" echo " --storage-class-name NAME StorageClass for the glutton-storage external volume. Setting it" @@ -155,8 +171,8 @@ substitute() { -e "s|\${SANDBOX_CONFIG_NAME}|${sandbox_config_name}|g" \ -e "s|\${OTLP_ENDPOINT}|${OTLP_ENDPOINT}|g" \ -e "s|\${ACTOR_MEMORY}|${ACTOR_MEMORY}|g" \ + -e "s|\${ACTOR_CPU}|${ACTOR_CPU}|g" \ -e "s|\${WORKER_RESOURCES}|${worker_resources}|g" \ - -e "s|\${SWEPERF_IMAGE}|${SWEPERF_IMAGE:-}|g" \ -e "s|\${STORAGE_CLASS_NAME}|${STORAGE_CLASS_NAME}|g" \ "${manifest}" } @@ -241,6 +257,21 @@ tune_fake_atelet() { } deploy() { + # Check every requested template has a manifest before anything on the + # cluster is deleted or created. + local template manifest + for template in "${TEMPLATES[@]}"; do + manifest="$(template_manifest "${template}")" + if [[ ! -f "${manifest}" ]]; then + echo "Error: no manifest for template ${template}: ${manifest} not found" >&2 + if [[ "${template}" == sweperf-* ]]; then + echo "sweperf templates come from ${MANIFEST_DIR}/sweperf/sweperf-images.json; add the task" >&2 + echo "there and run generate_sweperf_templates.py (see \"Sweperf Tasks\" in benchmarking/README.md)" >&2 + fi + exit 1 + fi + done + resolve_otlp_endpoint # Before the pool, so its fake Workers are served by the tuned fake-atelet. if fake_data_plane; then @@ -249,7 +280,7 @@ deploy() { echo "Error: --fake-atelet-env needs substrate installed with --fake-data-plane" >&2 exit 1 fi - echo "Deploying workloads (worker_count=${WORKER_COUNT}, actor_memory=${ACTOR_MEMORY}, worker_memory=${WORKER_MEMORY:-unset}, storage_class=${STORAGE_CLASS_NAME:-none}, otlp_endpoint=${OTLP_ENDPOINT})..." + echo "Deploying workloads (worker_count=${WORKER_COUNT}, actor_memory=${ACTOR_MEMORY}, actor_cpu=${ACTOR_CPU}, worker_memory=${WORKER_MEMORY:-unset}, storage_class=${STORAGE_CLASS_NAME:-none}, otlp_endpoint=${OTLP_ENDPOINT})..." substitute "${POOL_MANIFEST}" | hack/run-tool.sh ko apply -f - echo "Waiting for worker pool to be ready (timeout: ${WAIT_TIMEOUT_SECS}s)..." if fake_data_plane; then @@ -276,13 +307,12 @@ deploy() { # the old template first. This removal is safe, because the benchmark # automation deletes the actors between the tests; it also removes the old # golden actor and snapshot server-side. - local template for template in "${TEMPLATES[@]}"; do run_kubectl_ate delete actor-template "${template}" -a benchmark-workloads \ >/dev/null 2>&1 || true # ko resolve builds the ko:// image references and replaces them with # pushed digests before the manifest reaches kubectl-ate. - substitute "${MANIFEST_DIR}/${template}-template.yaml.tmpl" \ + substitute "$(template_manifest "${template}")" \ | hack/run-tool.sh ko resolve -f - \ | run_kubectl_ate create actor-template -f - done @@ -346,6 +376,13 @@ while [[ "$#" -gt 0 ]]; do --actor-memory=*) ACTOR_MEMORY="${1#*=}" ;; + --actor-cpu) + shift + ACTOR_CPU="$1" + ;; + --actor-cpu=*) + ACTOR_CPU="${1#*=}" + ;; --worker-memory) shift WORKER_MEMORY="$1" diff --git a/benchmarking/workloads/manifests/glutton-durdir-data-template.yaml.tmpl b/benchmarking/workloads/manifests/glutton-durdir-data-template.yaml.tmpl index d77db84139..02ad6c98d7 100644 --- a/benchmarking/workloads/manifests/glutton-durdir-data-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/glutton-durdir-data-template.yaml.tmpl @@ -24,7 +24,7 @@ workerSelector: workload: benchmark-worker containers: - name: glutton - image: ko://github.com/agent-substrate/substrate/cmd/benchmarking/glutton + image: ko://go.ate.dev/substrate/cmd/benchmarking/glutton command: - "/ko-app/glutton" - "--grpc-listen-addr=:80" @@ -47,9 +47,9 @@ volumes: resources: limits: - name: cpu - quantity: "1" + quantity: "${ACTOR_CPU}" - name: memory - quantity: ${ACTOR_MEMORY} + quantity: "${ACTOR_MEMORY}" snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_VOLUMES storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/glutton-durdir-data/ diff --git a/benchmarking/workloads/manifests/glutton-durdir-full-template.yaml.tmpl b/benchmarking/workloads/manifests/glutton-durdir-full-template.yaml.tmpl index a4860bfc47..6e738624ab 100644 --- a/benchmarking/workloads/manifests/glutton-durdir-full-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/glutton-durdir-full-template.yaml.tmpl @@ -23,7 +23,7 @@ workerSelector: workload: benchmark-worker containers: - name: glutton - image: ko://github.com/agent-substrate/substrate/cmd/benchmarking/glutton + image: ko://go.ate.dev/substrate/cmd/benchmarking/glutton command: - "/ko-app/glutton" - "--grpc-listen-addr=:80" @@ -44,9 +44,9 @@ volumes: resources: limits: - name: cpu - quantity: "1" + quantity: "${ACTOR_CPU}" - name: memory - quantity: ${ACTOR_MEMORY} + quantity: "${ACTOR_MEMORY}" snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/glutton-durdir-full/ diff --git a/benchmarking/workloads/manifests/glutton-storage-template.yaml.tmpl b/benchmarking/workloads/manifests/glutton-storage-template.yaml.tmpl index fee943693e..c64e36e26a 100644 --- a/benchmarking/workloads/manifests/glutton-storage-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/glutton-storage-template.yaml.tmpl @@ -29,7 +29,7 @@ volumes: storageClassName: ${STORAGE_CLASS_NAME} containers: - name: glutton-storage - image: ko://github.com/agent-substrate/substrate/cmd/benchmarking/glutton + image: ko://go.ate.dev/substrate/cmd/benchmarking/glutton command: - "/ko-app/glutton" - "--grpc-listen-addr=:80" @@ -53,9 +53,9 @@ containers: resources: limits: - name: cpu - quantity: "1" + quantity: "${ACTOR_CPU}" - name: memory - quantity: ${ACTOR_MEMORY} + quantity: "${ACTOR_MEMORY}" snapshotConfig: onCommit: SNAPSHOT_CONTENT_SCOPE_FULL storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/glutton-storage/ diff --git a/benchmarking/workloads/manifests/glutton-template.yaml.tmpl b/benchmarking/workloads/manifests/glutton-template.yaml.tmpl index dc249366db..8d26c6c643 100644 --- a/benchmarking/workloads/manifests/glutton-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/glutton-template.yaml.tmpl @@ -24,7 +24,7 @@ workerSelector: workload: benchmark-worker containers: - name: glutton - image: ko://github.com/agent-substrate/substrate/cmd/benchmarking/glutton + image: ko://go.ate.dev/substrate/cmd/benchmarking/glutton # Port 80 is the only inbound port nftables forwards from the worker pod # into the actor sandbox (see cmd/ateworker-gvisor/main.go). HTTP/1.1 mode # lets the atenet router (which forwards HTTP/1.1 upstream) reach Ping. @@ -58,9 +58,9 @@ containers: resources: limits: - name: cpu - quantity: "1" + quantity: "${ACTOR_CPU}" - name: memory - quantity: ${ACTOR_MEMORY} + quantity: "${ACTOR_MEMORY}" snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/glutton/ diff --git a/benchmarking/workloads/manifests/sleep-template.yaml.tmpl b/benchmarking/workloads/manifests/sleep-template.yaml.tmpl index 72becd7126..0a30855682 100644 --- a/benchmarking/workloads/manifests/sleep-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/sleep-template.yaml.tmpl @@ -33,9 +33,9 @@ containers: resources: limits: - name: cpu - quantity: "1" + quantity: "${ACTOR_CPU}" - name: memory - quantity: ${ACTOR_MEMORY} + quantity: "${ACTOR_MEMORY}" snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/sleep/ diff --git a/benchmarking/workloads/manifests/swebench-astropy-7336-template.yaml.tmpl b/benchmarking/workloads/manifests/swebench-astropy-7336-template.yaml.tmpl deleted file mode 100644 index 5f8455216e..0000000000 --- a/benchmarking/workloads/manifests/swebench-astropy-7336-template.yaml.tmpl +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -metadata: - atespace: benchmark-workloads - name: swebench-astropy-7336 -workerSelector: - matchLabels: - workload: benchmark-worker -containers: -- name: astropy - # The task image is built from the SWE-Perf repo - # https://github.com/gke-labs/sweperf, which supplies both the server - # entrypoint and the trajectory under test; the command below runs that - # entrypoint against that trace, so the image has to be the one built for - # this task. - # - # Supplied via the SWEPERF_IMAGE environment variable, substituted by - # benchmarking/workloads/deploy.sh. Pin by digest rather than tag: these - # images are commonly rebuilt in place under the same tag. - # - # Build the image from sweperf c30c0d6 (2026-09-21) or later. The - # asynchronous HTTP execution API this benchmark drives was added on - # 2026-08-31; images older than that ship a replay.py with no HTTP server at - # all and fail on the client's first GET /status. c30c0d6 adds - # execution_duration_ms to the job status, the in-sandbox time of each cycle. - image: "${SWEPERF_IMAGE}" - # generate.py writes the replay server to /replay.py and the trajectory to - # /trace.json at the image root; these are also the image's own ENTRYPOINT. - # Nothing in that repo produces /opt/swebench/. - command: - - "python3" - - "/replay.py" - - "/trace.json" - env: - - name: SWEPERF_DISABLE_INTERNAL_SLEEP - value: "1" -# Sandbox size: enough for the astropy test runs the trace replays. The cpu -# limit caps the measured execution time, so keep it fixed across runs. -resources: - limits: - - name: cpu - quantity: "2" - - name: memory - quantity: 4Gi -sandboxConfig: - sandboxClass: ${SANDBOX_CLASS} - configName: ${SANDBOX_CONFIG_NAME} -snapshotConfig: - preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/swebench-astropy-7336/ diff --git a/benchmarking/workloads/manifests/sweperf/generate_sweperf_templates.py b/benchmarking/workloads/manifests/sweperf/generate_sweperf_templates.py new file mode 100644 index 0000000000..b96a955d81 --- /dev/null +++ b/benchmarking/workloads/manifests/sweperf/generate_sweperf_templates.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Renders sweperf ActorTemplates from sweperf-images.json. + +Usage: + generate_sweperf_templates.py '*' # every task + generate_sweperf_templates.py astropy-7336 # one task + generate_sweperf_templates.py 'django*' -o /tmp/out # glob, custom output directory + generate_sweperf_templates.py --check # fail if generated/ is stale + +Writes sweperf--template.yaml.tmpl per task into generated/ next to +this script, where deploy.sh looks for every sweperf-* template. The +generated files are checked in: regenerate after editing the catalog or the +.j2, and use --check to find files that are missing or stale. Images in +sweperf-images.json must be pinned by digest. +""" + +import argparse +import collections +import fnmatch +import json +import pathlib +import sys + +import jinja2 + +HERE = pathlib.Path(__file__).resolve().parent + + +def load_catalog(path: pathlib.Path) -> list[dict]: + """Reads the catalog and rejects entries deploy.sh or the client can't use.""" + tasks = json.loads(path.read_text()) + for t in tasks: + missing = {"name", "image", "steps"} - t.keys() + if missing: + raise ValueError(f"{path}: entry {t} is missing {sorted(missing)}") + if not isinstance(t["steps"], int) or t["steps"] <= 0: + raise ValueError(f"{path}: {t['name']}: steps must be a positive integer") + if "@sha256:" not in t["image"]: + raise ValueError(f"{path}: {t['name']}: image must be pinned by digest (repo@sha256:...)") + # render() keys its output by file name, so a duplicate would silently + # overwrite the earlier entry. + names = collections.Counter(t["name"] for t in tasks) + duplicates = sorted(n for n, c in names.items() if c > 1) + if duplicates: + raise ValueError(f"{path}: duplicate task names: {duplicates}") + return tasks + + +def render(tasks: list[dict]) -> dict[str, str]: + """Returns the generated file name and contents for each task.""" + template = jinja2.Environment( + loader=jinja2.FileSystemLoader(HERE), + undefined=jinja2.StrictUndefined, + keep_trailing_newline=True, + ).get_template("sweperf-template.yaml.j2") + out = {} + for t in tasks: + name = f"sweperf-{t['name']}" + out[f"{name}-template.yaml.tmpl"] = template.render( + name=name, + container=t["name"].split("-")[0], # astropy-7336 -> astropy + image=t["image"], + steps=t["steps"], + ) + return out + + +def check(rendered: dict[str, str], directory: pathlib.Path) -> list[str]: + """Returns how directory differs from what the catalog renders.""" + problems = [] + for fname, content in rendered.items(): + path = directory / fname + if not path.exists(): + problems.append(f"missing: {path}") + elif path.read_text() != content: + problems.append(f"stale: {path}") + for path in sorted(directory.glob("*-template.yaml.tmpl")): + if path.name not in rendered: + problems.append(f"not in catalog: {path}") + return problems + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("task", nargs="?", help="task name to render, or a glob such as '*'") + parser.add_argument("-o", "--out", type=pathlib.Path, default=HERE / "generated", + help="output directory (default: %(default)s)") + parser.add_argument("--catalog", type=pathlib.Path, default=HERE / "sweperf-images.json") + parser.add_argument("--check", action="store_true", + help="fail if the output directory is missing, stale or has extra templates") + args = parser.parse_args() + + try: + tasks = load_catalog(args.catalog) + except ValueError as e: + print(e, file=sys.stderr) + return 1 + + if args.check: + problems = check(render(tasks), args.out) + for p in problems: + print(p, file=sys.stderr) + if problems: + print(f"regenerate with: {sys.argv[0]} '*'", file=sys.stderr) + return 1 + return 0 + + if not args.task: + parser.error("give a task name or glob, or --check") + selected = [t for t in tasks if fnmatch.fnmatch(t["name"], args.task)] + if not selected: + names = ", ".join(t["name"] for t in tasks) + print(f"no task matches {args.task!r}; known: {names}", file=sys.stderr) + return 1 + + args.out.mkdir(parents=True, exist_ok=True) + for fname, content in render(selected).items(): + path = args.out / fname + path.write_text(content) + print(path) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarking/workloads/manifests/sweperf/generated/sweperf-astropy-7336-template.yaml.tmpl b/benchmarking/workloads/manifests/sweperf/generated/sweperf-astropy-7336-template.yaml.tmpl new file mode 100644 index 0000000000..050f1d9676 --- /dev/null +++ b/benchmarking/workloads/manifests/sweperf/generated/sweperf-astropy-7336-template.yaml.tmpl @@ -0,0 +1,55 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generated by benchmarking/workloads/manifests/sweperf/generate_sweperf_templates.py +# from sweperf-images.json and sweperf-template.yaml.j2; edit those, not this +# file. The ${...} values are filled in by benchmarking/workloads/deploy.sh. +metadata: + atespace: benchmark-workloads + name: sweperf-astropy-7336 +workerSelector: + matchLabels: + workload: benchmark-worker +containers: +- name: astropy + # Built from https://github.com/gke-labs/sweperf c30c0d6 or later (older + # images have no HTTP server and fail the client's first GET /status), pinned + # by digest because these images are rebuilt in place under the same tag. + image: "us-docker.pkg.dev/gke-substrate-release/benchmark/sweperf@sha256:c265bfba3bea40c99e7baad734579b75d02dab6dbe6910cb1f9ed71c8b08f5b1" + command: + - "python3" + - "/replay.py" + - "/trace.json" + env: + - name: SWEPERF_DISABLE_INTERNAL_SLEEP + value: "1" + # The trajectory's step count, for reference only: the sweperf client takes + # it from --sweperf-total-steps, so run this template with + # --sweperf-total-steps=21. + - name: SWEPERF_TOTAL_STEPS + value: "21" +# Sandbox size: enough for the test runs the traces replay. The cpu limit caps +# the measured execution time, so keep it fixed across runs. +resources: + limits: + - name: cpu + quantity: "2" + - name: memory + quantity: 4Gi +sandboxConfig: + sandboxClass: ${SANDBOX_CLASS} + configName: ${SANDBOX_CONFIG_NAME} +snapshotConfig: + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY + storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/sweperf-astropy-7336/ diff --git a/benchmarking/workloads/manifests/sweperf/generated/sweperf-django-11099-template.yaml.tmpl b/benchmarking/workloads/manifests/sweperf/generated/sweperf-django-11099-template.yaml.tmpl new file mode 100644 index 0000000000..43ce70e1ff --- /dev/null +++ b/benchmarking/workloads/manifests/sweperf/generated/sweperf-django-11099-template.yaml.tmpl @@ -0,0 +1,55 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generated by benchmarking/workloads/manifests/sweperf/generate_sweperf_templates.py +# from sweperf-images.json and sweperf-template.yaml.j2; edit those, not this +# file. The ${...} values are filled in by benchmarking/workloads/deploy.sh. +metadata: + atespace: benchmark-workloads + name: sweperf-django-11099 +workerSelector: + matchLabels: + workload: benchmark-worker +containers: +- name: django + # Built from https://github.com/gke-labs/sweperf c30c0d6 or later (older + # images have no HTTP server and fail the client's first GET /status), pinned + # by digest because these images are rebuilt in place under the same tag. + image: "us-docker.pkg.dev/gke-substrate-release/benchmark/sweperf@sha256:7e4baaf61a7482d6d09e2fcccecd1529f47488aa52821d507cc8d6ad0ba31eea" + command: + - "python3" + - "/replay.py" + - "/trace.json" + env: + - name: SWEPERF_DISABLE_INTERNAL_SLEEP + value: "1" + # The trajectory's step count, for reference only: the sweperf client takes + # it from --sweperf-total-steps, so run this template with + # --sweperf-total-steps=15. + - name: SWEPERF_TOTAL_STEPS + value: "15" +# Sandbox size: enough for the test runs the traces replay. The cpu limit caps +# the measured execution time, so keep it fixed across runs. +resources: + limits: + - name: cpu + quantity: "2" + - name: memory + quantity: 4Gi +sandboxConfig: + sandboxClass: ${SANDBOX_CLASS} + configName: ${SANDBOX_CONFIG_NAME} +snapshotConfig: + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY + storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/sweperf-django-11099/ diff --git a/benchmarking/workloads/manifests/sweperf/generated/sweperf-matplotlib-20826-template.yaml.tmpl b/benchmarking/workloads/manifests/sweperf/generated/sweperf-matplotlib-20826-template.yaml.tmpl new file mode 100644 index 0000000000..315add0315 --- /dev/null +++ b/benchmarking/workloads/manifests/sweperf/generated/sweperf-matplotlib-20826-template.yaml.tmpl @@ -0,0 +1,55 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generated by benchmarking/workloads/manifests/sweperf/generate_sweperf_templates.py +# from sweperf-images.json and sweperf-template.yaml.j2; edit those, not this +# file. The ${...} values are filled in by benchmarking/workloads/deploy.sh. +metadata: + atespace: benchmark-workloads + name: sweperf-matplotlib-20826 +workerSelector: + matchLabels: + workload: benchmark-worker +containers: +- name: matplotlib + # Built from https://github.com/gke-labs/sweperf c30c0d6 or later (older + # images have no HTTP server and fail the client's first GET /status), pinned + # by digest because these images are rebuilt in place under the same tag. + image: "us-docker.pkg.dev/gke-substrate-release/benchmark/sweperf@sha256:74989c4921831f9fbf3fb95924e210bf6616ede00cf30cc9cf0bad945342174b" + command: + - "python3" + - "/replay.py" + - "/trace.json" + env: + - name: SWEPERF_DISABLE_INTERNAL_SLEEP + value: "1" + # The trajectory's step count, for reference only: the sweperf client takes + # it from --sweperf-total-steps, so run this template with + # --sweperf-total-steps=49. + - name: SWEPERF_TOTAL_STEPS + value: "49" +# Sandbox size: enough for the test runs the traces replay. The cpu limit caps +# the measured execution time, so keep it fixed across runs. +resources: + limits: + - name: cpu + quantity: "2" + - name: memory + quantity: 4Gi +sandboxConfig: + sandboxClass: ${SANDBOX_CLASS} + configName: ${SANDBOX_CONFIG_NAME} +snapshotConfig: + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY + storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/sweperf-matplotlib-20826/ diff --git a/benchmarking/workloads/manifests/sweperf/generated/sweperf-sphinx-8638-template.yaml.tmpl b/benchmarking/workloads/manifests/sweperf/generated/sweperf-sphinx-8638-template.yaml.tmpl new file mode 100644 index 0000000000..df415cc62e --- /dev/null +++ b/benchmarking/workloads/manifests/sweperf/generated/sweperf-sphinx-8638-template.yaml.tmpl @@ -0,0 +1,55 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generated by benchmarking/workloads/manifests/sweperf/generate_sweperf_templates.py +# from sweperf-images.json and sweperf-template.yaml.j2; edit those, not this +# file. The ${...} values are filled in by benchmarking/workloads/deploy.sh. +metadata: + atespace: benchmark-workloads + name: sweperf-sphinx-8638 +workerSelector: + matchLabels: + workload: benchmark-worker +containers: +- name: sphinx + # Built from https://github.com/gke-labs/sweperf c30c0d6 or later (older + # images have no HTTP server and fail the client's first GET /status), pinned + # by digest because these images are rebuilt in place under the same tag. + image: "us-docker.pkg.dev/gke-substrate-release/benchmark/sweperf@sha256:ca801ef886f07f92598f576b875e6f5b5ce4d7d8ed4da607d210e7f0caa23210" + command: + - "python3" + - "/replay.py" + - "/trace.json" + env: + - name: SWEPERF_DISABLE_INTERNAL_SLEEP + value: "1" + # The trajectory's step count, for reference only: the sweperf client takes + # it from --sweperf-total-steps, so run this template with + # --sweperf-total-steps=64. + - name: SWEPERF_TOTAL_STEPS + value: "64" +# Sandbox size: enough for the test runs the traces replay. The cpu limit caps +# the measured execution time, so keep it fixed across runs. +resources: + limits: + - name: cpu + quantity: "2" + - name: memory + quantity: 4Gi +sandboxConfig: + sandboxClass: ${SANDBOX_CLASS} + configName: ${SANDBOX_CONFIG_NAME} +snapshotConfig: + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY + storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/sweperf-sphinx-8638/ diff --git a/benchmarking/workloads/manifests/sweperf/generated/sweperf-sympy-13647-template.yaml.tmpl b/benchmarking/workloads/manifests/sweperf/generated/sweperf-sympy-13647-template.yaml.tmpl new file mode 100644 index 0000000000..12310ad3fe --- /dev/null +++ b/benchmarking/workloads/manifests/sweperf/generated/sweperf-sympy-13647-template.yaml.tmpl @@ -0,0 +1,55 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generated by benchmarking/workloads/manifests/sweperf/generate_sweperf_templates.py +# from sweperf-images.json and sweperf-template.yaml.j2; edit those, not this +# file. The ${...} values are filled in by benchmarking/workloads/deploy.sh. +metadata: + atespace: benchmark-workloads + name: sweperf-sympy-13647 +workerSelector: + matchLabels: + workload: benchmark-worker +containers: +- name: sympy + # Built from https://github.com/gke-labs/sweperf c30c0d6 or later (older + # images have no HTTP server and fail the client's first GET /status), pinned + # by digest because these images are rebuilt in place under the same tag. + image: "us-docker.pkg.dev/gke-substrate-release/benchmark/sweperf@sha256:89ed518b85efaf063cd3a515c33bc293a4883964d9af7cc21c4e34824ee171f1" + command: + - "python3" + - "/replay.py" + - "/trace.json" + env: + - name: SWEPERF_DISABLE_INTERNAL_SLEEP + value: "1" + # The trajectory's step count, for reference only: the sweperf client takes + # it from --sweperf-total-steps, so run this template with + # --sweperf-total-steps=33. + - name: SWEPERF_TOTAL_STEPS + value: "33" +# Sandbox size: enough for the test runs the traces replay. The cpu limit caps +# the measured execution time, so keep it fixed across runs. +resources: + limits: + - name: cpu + quantity: "2" + - name: memory + quantity: 4Gi +sandboxConfig: + sandboxClass: ${SANDBOX_CLASS} + configName: ${SANDBOX_CONFIG_NAME} +snapshotConfig: + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY + storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/sweperf-sympy-13647/ diff --git a/benchmarking/workloads/manifests/sweperf/requirements.txt b/benchmarking/workloads/manifests/sweperf/requirements.txt new file mode 100644 index 0000000000..96707f9b84 --- /dev/null +++ b/benchmarking/workloads/manifests/sweperf/requirements.txt @@ -0,0 +1,15 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +jinja2>=3.1 diff --git a/benchmarking/workloads/manifests/sweperf/sweperf-images.json b/benchmarking/workloads/manifests/sweperf/sweperf-images.json new file mode 100644 index 0000000000..88037e587d --- /dev/null +++ b/benchmarking/workloads/manifests/sweperf/sweperf-images.json @@ -0,0 +1,27 @@ +[ + { + "name": "astropy-7336", + "image": "us-docker.pkg.dev/gke-substrate-release/benchmark/sweperf@sha256:c265bfba3bea40c99e7baad734579b75d02dab6dbe6910cb1f9ed71c8b08f5b1", + "steps": 21 + }, + { + "name": "django-11099", + "image": "us-docker.pkg.dev/gke-substrate-release/benchmark/sweperf@sha256:7e4baaf61a7482d6d09e2fcccecd1529f47488aa52821d507cc8d6ad0ba31eea", + "steps": 15 + }, + { + "name": "matplotlib-20826", + "image": "us-docker.pkg.dev/gke-substrate-release/benchmark/sweperf@sha256:74989c4921831f9fbf3fb95924e210bf6616ede00cf30cc9cf0bad945342174b", + "steps": 49 + }, + { + "name": "sphinx-8638", + "image": "us-docker.pkg.dev/gke-substrate-release/benchmark/sweperf@sha256:ca801ef886f07f92598f576b875e6f5b5ce4d7d8ed4da607d210e7f0caa23210", + "steps": 64 + }, + { + "name": "sympy-13647", + "image": "us-docker.pkg.dev/gke-substrate-release/benchmark/sweperf@sha256:89ed518b85efaf063cd3a515c33bc293a4883964d9af7cc21c4e34824ee171f1", + "steps": 33 + } +] diff --git a/benchmarking/workloads/manifests/sweperf/sweperf-template.yaml.j2 b/benchmarking/workloads/manifests/sweperf/sweperf-template.yaml.j2 new file mode 100644 index 0000000000..47a259b7ee --- /dev/null +++ b/benchmarking/workloads/manifests/sweperf/sweperf-template.yaml.j2 @@ -0,0 +1,55 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generated by benchmarking/workloads/manifests/sweperf/generate_sweperf_templates.py +# from sweperf-images.json and sweperf-template.yaml.j2; edit those, not this +# file. The ${...} values are filled in by benchmarking/workloads/deploy.sh. +metadata: + atespace: benchmark-workloads + name: {{ name }} +workerSelector: + matchLabels: + workload: benchmark-worker +containers: +- name: {{ container }} + # Built from https://github.com/gke-labs/sweperf c30c0d6 or later (older + # images have no HTTP server and fail the client's first GET /status), pinned + # by digest because these images are rebuilt in place under the same tag. + image: "{{ image }}" + command: + - "python3" + - "/replay.py" + - "/trace.json" + env: + - name: SWEPERF_DISABLE_INTERNAL_SLEEP + value: "1" + # The trajectory's step count, for reference only: the sweperf client takes + # it from --sweperf-total-steps, so run this template with + # --sweperf-total-steps={{ steps }}. + - name: SWEPERF_TOTAL_STEPS + value: "{{ steps }}" +# Sandbox size: enough for the test runs the traces replay. The cpu limit caps +# the measured execution time, so keep it fixed across runs. +resources: + limits: + - name: cpu + quantity: "2" + - name: memory + quantity: 4Gi +sandboxConfig: + sandboxClass: ${SANDBOX_CLASS} + configName: ${SANDBOX_CONFIG_NAME} +snapshotConfig: + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY + storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/{{ name }}/ diff --git a/benchmarking/workloads/manifests/sweperf/test_generate_sweperf_templates.py b/benchmarking/workloads/manifests/sweperf/test_generate_sweperf_templates.py new file mode 100644 index 0000000000..2d67908e01 --- /dev/null +++ b/benchmarking/workloads/manifests/sweperf/test_generate_sweperf_templates.py @@ -0,0 +1,76 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for generate_sweperf_templates.py: +python3 benchmarking/workloads/manifests/sweperf/test_generate_sweperf_templates.py +""" + +import json +import pathlib +import tempfile +import unittest + +import generate_sweperf_templates as gen + +PINNED = "example.com/sweperf@sha256:" + "0" * 64 + + +class GenerateTest(unittest.TestCase): + + def test_generated_matches_catalog(self): + # Fails when sweperf-images.json or the .j2 changed without regenerating. + rendered = gen.render(gen.load_catalog(gen.HERE / "sweperf-images.json")) + self.assertEqual(gen.check(rendered, gen.HERE / "generated"), [], + "run generate_sweperf_templates.py '*'") + + def test_render_fills_task_values(self): + out = gen.render([{"name": "django-11099", "image": PINNED, "steps": 15}]) + content = out["sweperf-django-11099-template.yaml.tmpl"] + self.assertIn("name: sweperf-django-11099", content) + self.assertIn(f'image: "{PINNED}"', content) + self.assertIn('value: "15"', content) + # deploy.sh fills these, so they must survive rendering. + self.assertIn("${BUCKET_NAME}", content) + + def test_check_reports_missing_stale_and_orphaned_files(self): + rendered = gen.render([{"name": n, "image": PINNED, "steps": 1} + for n in ("fresh-1", "stale-1", "missing-1")]) + with tempfile.TemporaryDirectory() as d: + d = pathlib.Path(d) + (d / "sweperf-fresh-1-template.yaml.tmpl").write_text( + rendered["sweperf-fresh-1-template.yaml.tmpl"]) + (d / "sweperf-stale-1-template.yaml.tmpl").write_text("old") + (d / "sweperf-gone-1-template.yaml.tmpl").write_text("old") + problems = gen.check(rendered, d) + kinds = sorted(p.split(":")[0] for p in problems) + self.assertEqual(kinds, ["missing", "not in catalog", "stale"], problems) + + def test_catalog_rejects_bad_entries(self): + cases = { + "missing key": [{"name": "a", "image": PINNED}], + "zero steps": [{"name": "a", "image": PINNED, "steps": 0}], + "unpinned image": [{"name": "a", "image": "example.com/sweperf:a", "steps": 1}], + "duplicate name": [{"name": "a", "image": PINNED, "steps": 1}, + {"name": "a", "image": PINNED, "steps": 2}], + } + for label, catalog in cases.items(): + with self.subTest(label), tempfile.NamedTemporaryFile("w", suffix=".json") as f: + json.dump(catalog, f) + f.flush() + with self.assertRaises(ValueError): + gen.load_catalog(pathlib.Path(f.name)) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarking/workloads/manifests/workloads.yaml.tmpl b/benchmarking/workloads/manifests/workloads.yaml.tmpl index d8043c4d76..e72b93d18a 100644 --- a/benchmarking/workloads/manifests/workloads.yaml.tmpl +++ b/benchmarking/workloads/manifests/workloads.yaml.tmpl @@ -35,7 +35,7 @@ spec: replicas: ${WORKER_COUNT} sandboxClasses: - name: ${SANDBOX_CLASS} - workerImage: ko://github.com/agent-substrate/substrate/cmd/ateworker-${SANDBOX_CLASS} + workerImage: ko://go.ate.dev/substrate/cmd/ateworker-${SANDBOX_CLASS} template: # Lets gvisor and microvm workers share a node tainted ate.dev/sandboxClass=shared. tolerations: diff --git a/charts/substrate-crds/templates/ate.dev_workerpools.yaml b/charts/substrate-crds/templates/ate.dev_workerpools.yaml index 4c32fcb044..c20f14ec2c 100644 --- a/charts/substrate-crds/templates/ate.dev_workerpools.yaml +++ b/charts/substrate-crds/templates/ate.dev_workerpools.yaml @@ -434,8 +434,8 @@ spec: serviceAccountName: description: |- ServiceAccountName for the worker pods. The ServiceAccount must exist - in the WorkerPool's namespace. When omitted, the pods use the - namespace's default ServiceAccount. + in the WorkerPool's namespace. When omitted, the pods run as the + "ateworker" ServiceAccount, which the controller creates. minLength: 1 type: string tolerations: diff --git a/charts/substrate/README.md b/charts/substrate/README.md index d326d71a65..5601339946 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -69,6 +69,9 @@ See `values.yaml` for the full set; the important keys: | `ateApi.actorJWTIssuer` | `https://idp..svc` | Issuer placed in actor JWTs; set a stable URL reachable by relying parties | | `objectStorePlugin.extraEnv` | `[]` | Object storage credentials/settings shared by both object-store plugins | | `objectStorePlugin.control.resources` / `objectStorePlugin.node.resources` | see `values.yaml` | Resources for the API and node object-store sidecars | +| `ateApi.logLevel` | `info` | API server log level in its mounted config; one of `debug`, `info`, `warn`, or `error` | +| `ateApi.authorization.enabled` | `true` | Enforce API authorization | +| `ateApi.authorization.bootstrapOwners` | `[]` | Global owners; empty defaults to the release's shared `ate-client` ServiceAccount | | `ateApi.extraArgs` | `[]` | Additional command-line arguments appended to the ateapi defaults | | `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces, metrics, the actor lifecycle events and the router access log | | `otel.traces.enabled` | `true` | Set to `false` to export no traces from the router; the Go components do not honor this yet | @@ -166,3 +169,21 @@ Create both group roles and all grants before installation, then set roles and table schemas for separate installs sharing one database. Give each install separate logins and grant each login membership only in its install's roles. + +## Upgrading API configuration and authorization + +The API server now requires `--config-file`. The chart mounts its generated +`ate-api-config` ConfigMap and rolls the API pods when `ateApi.logLevel` changes. +Replace `--log-level` in `ateApi.extraArgs` with `ateApi.logLevel`. + +API authorization is enabled by default, matching `ate-setup`. The release's +shared `ate-client` ServiceAccount is a bootstrap global owner so demos and +CLI access work. For shared installs, set `ateApi.authorization.bootstrapOwners` +to your operators' own principals and grant tenant access with `kubectl ate +create access-policy`; anyone who can mint the shared client's token otherwise has +global owner access. See [authorization](../../docs/authorization.md). + +Upgrade clients and components together: upstream now uses the +`go.ate.dev/substrate` Go module, and the worker protocol includes dynamic +runtime registration and the unified suspend/pause workflow. Custom worker +images and integrations must use the matching API version. diff --git a/charts/substrate/templates/ate-api-config.yaml b/charts/substrate/templates/ate-api-config.yaml new file mode 100644 index 0000000000..d34d218ea0 --- /dev/null +++ b/charts/substrate/templates/ate-api-config.yaml @@ -0,0 +1,27 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if not (has .Values.ateApi.logLevel (list "debug" "info" "warn" "error")) }} +{{- fail "ateApi.logLevel must be debug, info, warn, or error" }} +{{- end }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "substrate.fullname" (list "ate-api-config" .) }} + namespace: {{ .Release.Namespace }} +data: + apiconfig.textproto: | + logging: { level: {{ .Values.ateApi.logLevel | quote }} } diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index d1b6f81595..c6a9409149 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -75,6 +75,7 @@ spec: labels: app: ate-api-server annotations: + checksum/api-config: {{ .Values.ateApi.logLevel | sha256sum }} prometheus.io/scrape: "true" prometheus.io/port: "9090" spec: @@ -119,6 +120,9 @@ spec: - --objectstore-plugin-socket=/run/objectstore-plugin/control.sock - "--grpc-listen-addr=0.0.0.0:443" - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" + - --config-file=/run/apiconfig.ate.dev/apiconfig.textproto + - {{ printf "--experimental-enable-authz=%t" .Values.ateApi.authorization.enabled | quote }} + - {{ printf "--authz-bootstrap-owners=%s" (join "," (.Values.ateApi.authorization.bootstrapOwners | default (list (printf "system:serviceaccount:%s:%s" .Release.Namespace (include "substrate.fullname" (list "ate-client" .)))))) | quote }} - "--authentication-config=/etc/ateapi/authentication/authentication.yaml" - "--postgres-read-write-connection-string=@env" - "--postgres-owner-connection-string=@env" @@ -182,6 +186,7 @@ spec: {{- if .Values.postgres.clientCertificates.enabled }} - { name: postgres, mountPath: /run/postgres.podcert.ate.dev, readOnly: true } {{- end }} + - { name: api-config, mountPath: /run/apiconfig.ate.dev, readOnly: true } - { name: authentication-config, mountPath: /etc/ateapi/authentication, readOnly: true } ports: - containerPort: 443 @@ -238,6 +243,9 @@ spec: name: actor-id-ca-pool items: - { key: pool, path: pool.json } + - name: api-config + configMap: + name: {{ include "substrate.fullname" (list "ate-api-config" .) }} - name: authentication-config configMap: name: ate-api-authentication diff --git a/charts/substrate/templates/role.yaml b/charts/substrate/templates/role.yaml index 356f7be94d..5ff03a5ff6 100644 --- a/charts/substrate/templates/role.yaml +++ b/charts/substrate/templates/role.yaml @@ -35,6 +35,13 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - patch - apiGroups: - apps resources: diff --git a/charts/substrate/tests/control_plane_test.yaml b/charts/substrate/tests/control_plane_test.yaml index 44c6938fa5..798ec2cd8b 100644 --- a/charts/substrate/tests/control_plane_test.yaml +++ b/charts/substrate/tests/control_plane_test.yaml @@ -15,6 +15,8 @@ suite: Control plane integration templates: - ate-api-server.yaml +- ate-api-config.yaml +- role.yaml - atelet.yaml - k8s-credential-provider.yaml - ate-system-networkpolicies.yaml @@ -161,3 +163,78 @@ tests: value: - protocol: TCP port: api + +- it: supplies the required API config and bootstraps the release client + template: ate-api-server.yaml + documentSelector: + path: kind + value: Deployment + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --config-file=/run/apiconfig.ate.dev/apiconfig.textproto + - contains: + path: spec.template.spec.containers[0].args + content: --experimental-enable-authz=true + - contains: + path: spec.template.spec.containers[0].args + content: --authz-bootstrap-owners=system:serviceaccount:custom-system:custom-ate-client + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: api-config + mountPath: /run/apiconfig.ate.dev + readOnly: true + - contains: + path: spec.template.spec.volumes + content: + name: api-config + configMap: + name: custom-ate-api-config +- it: configures operator ownership even with runtime enforcement disabled + template: ate-api-server.yaml + set: + ateApi.authorization.enabled: false + ateApi.authorization.bootstrapOwners: + - system:serviceaccount:operators:admin + documentSelector: + path: kind + value: Deployment + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --experimental-enable-authz=false + - contains: + path: spec.template.spec.containers[0].args + content: --authz-bootstrap-owners=system:serviceaccount:operators:admin +- it: configures API logging in textproto + template: ate-api-config.yaml + set: + ateApi.logLevel: debug + asserts: + - equal: + path: metadata.name + value: custom-ate-api-config + - equal: + path: data["apiconfig.textproto"] + value: | + logging: { level: "debug" } +- it: rejects an invalid API log level + template: ate-api-config.yaml + set: + ateApi.logLevel: verbose + asserts: + - failedTemplate: + errorMessage: ateApi.logLevel must be debug, info, warn, or error +- it: lets the controller provision worker service accounts + template: role.yaml + documentSelector: + path: kind + value: ClusterRole + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: [serviceaccounts] + verbs: [create, patch] diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index b4ee86f3fe..8bc5bc47f2 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -107,6 +107,12 @@ objectStorePlugin: # Additional arguments appended to the ateapi defaults. ateApi: + logLevel: info + authorization: + enabled: true + # Empty defaults to the release's shared ate-client principal. Use explicit + # operator principals for shared installs; see docs/authorization.md. + bootstrapOwners: [] # Defaults to the in-cluster IdP URL; set a stable issuer for external consumers. actorJWTIssuer: "" extraArgs: [] diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index b7fde07919..ae716ff032 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -33,6 +33,7 @@ a pre-scan pass, so they may appear anywhere on its command line. | `--fake-data-plane` | `--fake-data-plane` | Install the fake data plane for control-plane benchmarks: fake-atelet takes atelet's place in its DaemonSet, taking its objectstore-plugin's storage settings, ate-controller is scaled to zero, and fake-workersync honors every WorkerPool with fake Workers on the fake-atelet nodes. Reaches `deploy ate-system`, `deploy atelet`, and `deploy ate-controller`; only `deploy ate-system` switches a cluster between the real and fake data planes, and the other two refuse a data plane other than the installed one. `deploy ate-system --fake-data-plane` refuses a cluster whose WorkerPools have worker pods. fake-atelet runs with its defaults; a benchmark tunes it through the optional `fake-atelet-tuning` ConfigMap, whose `FAKE_ATELET_*` variables set its flags, and which no install creates or resets. Actors do not exist; never use it on a cluster that serves real actors. `ATE_INSTALL_FAKE_DATA_PLANE=true` when the flag is absent | | `--experimental-additional-egress-extproc-service NS/SVC:PORT` | `--experimental-additional-egress-extproc-service NS/SVC:PORT` | External processor authorization filter | | `--credential-provider JSON` | `--credential-provider JSON`, or `ATE_CREDENTIAL_PROVIDER` | Required by `deploy ate-system` and `deploy atenet`. A JSON object: `{"name":"k8s.io"}` deploys and uses the bundled Kubernetes Secrets provider, with a NetworkPolicy that admits only the egress gateway; `{"enabled":false}` turns egress credential injection off; `{"name":"","address":":"}` uses a provider you deploy yourself. See [`docs/egress-credential-injection.md`](../../docs/egress-credential-injection.md) | +| `--ateapi-log-level LEVEL` | `--ateapi-log-level LEVEL`, or `ATE_API_LOG_LEVEL=LEVEL` | ate-api-server log level: `debug`, `info`, `warn`, or `error`. Written to the `ate-api-config` ConfigMap on every deploy when set; unset keeps the installed level (`info` on a new install). The server picks up a change without restarting | | `--otlp-endpoint URL` | `--otlp-endpoint URL`, or `ATE_OTLP_ENDPOINT=URL` | Send control plane telemetry to `URL` instead of the cluster default (see [`benchmarking/telemetry/README.md`](../../benchmarking/telemetry/README.md)) | | `--preview GATES` | `--preview GATES`, or `ATE_PREVIEW=GATES` | Preview gates passed as `--preview` to every substrate component: comma-separated gate names, or `ALL` for all. `ATE_PREVIEW` or `config.preview` when the flag is absent. An explicit `--preview=''` turns them all off, even when `ATE_PREVIEW` is set | | `--context NAME` | `KUBECTL_CONTEXT=NAME` | Kubeconfig context; still defaults to `KUBECTL_CONTEXT` | @@ -153,6 +154,7 @@ Individual secrets and config that `deploy ate-system` creates automatically. | `create podcertificate-controller-cas` | `--create-podcertificate-controller-cas` | | `create api-server-env-vars` | `--create-api-server-env-vars` | | `create api-authentication-config` | `--create-api-authentication-config` | +| `create api-config` | `--create-api-config` | ## Setup diff --git a/cmd/ate-setup/internal/cmd/benchmarks.go b/cmd/ate-setup/internal/cmd/benchmarks.go index 5b7d48fba7..d92d1c4b6e 100644 --- a/cmd/ate-setup/internal/cmd/benchmarks.go +++ b/cmd/ate-setup/internal/cmd/benchmarks.go @@ -17,8 +17,8 @@ package cmd import ( "github.com/spf13/cobra" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/steps" ) // The command paths the benchmark settings bind their flags to. Both accept diff --git a/cmd/ate-setup/internal/cmd/benchmarks_test.go b/cmd/ate-setup/internal/cmd/benchmarks_test.go index 00bdc9760a..731148618c 100644 --- a/cmd/ate-setup/internal/cmd/benchmarks_test.go +++ b/cmd/ate-setup/internal/cmd/benchmarks_test.go @@ -18,8 +18,8 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/steps" ) // The benchmark settings reach three channels, so a rejection has to name the diff --git a/cmd/ate-setup/internal/cmd/create.go b/cmd/ate-setup/internal/cmd/create.go index a0b4da52e6..9c5c5c193e 100644 --- a/cmd/ate-setup/internal/cmd/create.go +++ b/cmd/ate-setup/internal/cmd/create.go @@ -19,7 +19,7 @@ import ( "github.com/spf13/cobra" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" + "go.ate.dev/substrate/cmd/ate-setup/internal/steps" ) var createCmd = &cobra.Command{ @@ -82,6 +82,11 @@ func init() { short: "Create the default ate-api-server authentication config", run: (*steps.Env).CreateAPIAuthenticationConfig, }, + { + use: "api-config", + short: "Create the ate-api-server config file, including its log level", + run: (*steps.Env).CreateAPIConfig, + }, } { run := sub.run createCmd.AddCommand(&cobra.Command{ diff --git a/cmd/ate-setup/internal/cmd/csi_test.go b/cmd/ate-setup/internal/cmd/csi_test.go index f2d03d6312..393a4dfc5d 100644 --- a/cmd/ate-setup/internal/cmd/csi_test.go +++ b/cmd/ate-setup/internal/cmd/csi_test.go @@ -21,9 +21,9 @@ import ( "github.com/spf13/cobra" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" + "go.ate.dev/substrate/cmd/ate-setup/internal/steps" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" ) // runRoot executes the command tree with args, quietly. diff --git a/cmd/ate-setup/internal/cmd/delete.go b/cmd/ate-setup/internal/cmd/delete.go index 60c9c03759..7bb9df466e 100644 --- a/cmd/ate-setup/internal/cmd/delete.go +++ b/cmd/ate-setup/internal/cmd/delete.go @@ -17,7 +17,7 @@ package cmd import ( "github.com/spf13/cobra" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos" + "go.ate.dev/substrate/cmd/ate-setup/internal/demos" ) var deleteCmd = &cobra.Command{ diff --git a/cmd/ate-setup/internal/cmd/demo.go b/cmd/ate-setup/internal/cmd/demo.go index e3dc080deb..49b882fe71 100644 --- a/cmd/ate-setup/internal/cmd/demo.go +++ b/cmd/ate-setup/internal/cmd/demo.go @@ -19,10 +19,10 @@ import ( "github.com/spf13/cobra" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/demos" // Registers every bundled demo; see the package doc. - _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/all" + _ "go.ate.dev/substrate/cmd/ate-setup/internal/demos/all" ) // demoArg is the short name a demo is addressed by on the command line: diff --git a/cmd/ate-setup/internal/cmd/deploy.go b/cmd/ate-setup/internal/cmd/deploy.go index 08ab2d7bb6..20cddbe891 100644 --- a/cmd/ate-setup/internal/cmd/deploy.go +++ b/cmd/ate-setup/internal/cmd/deploy.go @@ -17,8 +17,8 @@ package cmd import ( "github.com/spf13/cobra" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/steps" ) var deployCmd = &cobra.Command{ diff --git a/cmd/ate-setup/internal/cmd/record_test.go b/cmd/ate-setup/internal/cmd/record_test.go index e9b2996728..0787e14669 100644 --- a/cmd/ate-setup/internal/cmd/record_test.go +++ b/cmd/ate-setup/internal/cmd/record_test.go @@ -23,8 +23,8 @@ import ( "sigs.k8s.io/yaml" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/steps" ) // A record describes how the system was installed, so only the commands that diff --git a/cmd/ate-setup/internal/cmd/recordname_test.go b/cmd/ate-setup/internal/cmd/recordname_test.go index 25ead4161b..e1b82adca3 100644 --- a/cmd/ate-setup/internal/cmd/recordname_test.go +++ b/cmd/ate-setup/internal/cmd/recordname_test.go @@ -20,8 +20,8 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/steps" ) // recordFor runs the record writer the way PersistentPreRunE does and returns diff --git a/cmd/ate-setup/internal/cmd/root.go b/cmd/ate-setup/internal/cmd/root.go index 33c23026e2..e08a206739 100644 --- a/cmd/ate-setup/internal/cmd/root.go +++ b/cmd/ate-setup/internal/cmd/root.go @@ -22,10 +22,10 @@ import ( "github.com/spf13/cobra" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" - "github.com/agent-substrate/substrate/internal/version" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/cmd/ate-setup/internal/steps" + "go.ate.dev/substrate/internal/version" ) // Inputs that select where configuration comes from, as opposed to the diff --git a/cmd/ate-setup/internal/cmd/setup.go b/cmd/ate-setup/internal/cmd/setup.go index 09506b28c8..6b14b92b4a 100644 --- a/cmd/ate-setup/internal/cmd/setup.go +++ b/cmd/ate-setup/internal/cmd/setup.go @@ -17,7 +17,7 @@ package cmd import ( "github.com/spf13/cobra" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" + "go.ate.dev/substrate/cmd/ate-setup/internal/steps" ) var setupCmd = &cobra.Command{ diff --git a/cmd/ate-setup/internal/config/config.go b/cmd/ate-setup/internal/config/config.go index 6e43606974..126ff83c2e 100644 --- a/cmd/ate-setup/internal/config/config.go +++ b/cmd/ate-setup/internal/config/config.go @@ -32,9 +32,9 @@ import ( "k8s.io/apimachinery/pkg/util/validation" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/images" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/pkg/postgressetup" + "go.ate.dev/substrate/cmd/ate-setup/internal/images" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/pkg/postgressetup" ) // Enumerated values for the install-shaping flags. @@ -122,6 +122,13 @@ type Config struct { // follows neither form. ExpectedJWTIssuer string + // APIExperimentalEnableAuthz turns on ate-api-server's runtime + // authorization (ATE_API_EXPERIMENTAL_ENABLE_AUTHZ). + APIExperimentalEnableAuthz bool + // APIAuthzBootstrapOwners is the comma-separated principal list granted + // owner on global:root (ATE_API_AUTHZ_BOOTSTRAP_OWNERS). + APIAuthzBootstrapOwners string + // ActorJWTAlgorithm is the signing algorithm of the key in a new actor JWT // pool (ACTOR_JWT_ALGORITHM): ES256 or RS256. ActorJWTAlgorithm string @@ -222,9 +229,16 @@ type Config struct { // OtlpEndpoint is where the control plane ships telemetry // (ATE_OTLP_ENDPOINT). Benchmark actors are pointed at it too. OtlpEndpoint string + // PreviewGates is the --preview value (ATE_PREVIEW) every substrate // component is deployed with: gate names, or "ALL" for all of them. PreviewGates []string + + // APILogLevel is the ate-api-server log level: debug, info, warn, or + // error. Empty means none was asked for, which keeps the level an + // existing install already has. + APILogLevel string + // BenchmarkActorMemory is the memory limit for benchmark actors // (BENCHMARK_ACTOR_MEMORY). Empty leaves the workload default in place. BenchmarkActorMemory string diff --git a/cmd/ate-setup/internal/config/config_test.go b/cmd/ate-setup/internal/config/config_test.go index 20a2abd650..e9d8c79dd4 100644 --- a/cmd/ate-setup/internal/config/config_test.go +++ b/cmd/ate-setup/internal/config/config_test.go @@ -26,8 +26,8 @@ import ( "github.com/spf13/pflag" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/images" - "github.com/agent-substrate/substrate/internal/preview" + "go.ate.dev/substrate/cmd/ate-setup/internal/images" + "go.ate.dev/substrate/internal/preview" ) // loadEnv isolates Load from the ambient environment. @@ -46,6 +46,9 @@ func loadEnv(t *testing.T) { "ACTOR_JWT_ISSUER", "ANTHROPIC_API_KEY", "ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE", + "ATE_API_AUTHZ_BOOTSTRAP_OWNERS", + "ATE_API_EXPERIMENTAL_ENABLE_AUTHZ", + "ATE_API_LOG_LEVEL", "ATE_API_POSTGRES_CLOUDSQL_GSA", "ATE_API_POSTGRES_CLOUDSQL_IAM_AUTH", "ATE_API_POSTGRES_CLOUDSQL_IP_TYPE", @@ -557,6 +560,111 @@ func TestLoadExpectedJWTIssuer(t *testing.T) { } } +// An unset level stays empty, so that a deploy keeps the level an existing +// install has rather than resetting it to the default. +func TestLoadAPILogLevel(t *testing.T) { + for _, tt := range []struct { + name string + env string + flag string + want string + wantErr bool + }{ + {name: "unset", want: ""}, + {name: "environment", env: "debug", want: "debug"}, + {name: "flag beats environment", env: "debug", flag: "warn", want: "warn"}, + {name: "unsupported", env: "verbose", wantErr: true}, + } { + t.Run(tt.name, func(t *testing.T) { + loadEnv(t) + t.Setenv("ATE_API_LOG_LEVEL", tt.env) + fs := pflag.NewFlagSet("test", pflag.ContinueOnError) + BindFlags(fs) + if tt.flag != "" { + if err := fs.Set("ateapi-log-level", tt.flag); err != nil { + t.Fatalf("Set(--ateapi-log-level) error = %v", err) + } + } + + cfg, err := LoadFlags(fs, LoadOptions{NoDevEnv: true}) + if tt.wantErr { + if err == nil { + t.Fatalf("LoadFlags() with ATE_API_LOG_LEVEL=%q returned nil error", tt.env) + } + return + } + if err != nil { + t.Fatalf("LoadFlags() error = %v", err) + } + if cfg.APILogLevel != tt.want { + t.Errorf("APILogLevel = %q, want %q", cfg.APILogLevel, tt.want) + } + }) + } +} + +func TestLoadAPIAuthz(t *testing.T) { + const ( + clientOwner = "system:serviceaccount:ate-system:ate-client" + owners = "alice@example.com,bob@example.com" + ) + for _, tc := range []struct { + name string + env map[string]string + wantEnabled bool + wantOwners string + wantErr bool + }{ + { + name: "on by default, owned by the client ServiceAccount", + wantEnabled: true, + wantOwners: clientOwner, + }, + { + name: "default owner follows the namespace", + env: map[string]string{"ATE_NAMESPACE": "custom"}, + wantEnabled: true, + wantOwners: "system:serviceaccount:custom:ate-client", + }, + { + name: "explicit owners replace the default", + env: map[string]string{"ATE_API_AUTHZ_BOOTSTRAP_OWNERS": owners}, + wantEnabled: true, + wantOwners: owners, + }, + { + name: "turned off", + env: map[string]string{"ATE_API_EXPERIMENTAL_ENABLE_AUTHZ": "false"}, + wantOwners: clientOwner, + }, + { + name: "enabled with only separators", + env: map[string]string{"ATE_API_AUTHZ_BOOTSTRAP_OWNERS": " , "}, + wantErr: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + loadEnv(t) + for name, value := range tc.env { + t.Setenv(name, value) + } + cfg, err := Load(Options{}) + if tc.wantErr { + if err == nil || !strings.Contains(err.Error(), "bootstrap owner") { + t.Fatalf("Load() error = %v, want the missing bootstrap owner error", err) + } + return + } + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if cfg.APIExperimentalEnableAuthz != tc.wantEnabled || cfg.APIAuthzBootstrapOwners != tc.wantOwners { + t.Errorf("authz = %v, %q, want %v, %q", cfg.APIExperimentalEnableAuthz, cfg.APIAuthzBootstrapOwners, tc.wantEnabled, tc.wantOwners) + } + }) + } +} + func TestLoadActorJWTAlgorithm(t *testing.T) { for _, tt := range []struct { name string diff --git a/cmd/ate-setup/internal/config/coverage_test.go b/cmd/ate-setup/internal/config/coverage_test.go index f8a45bba42..036b0739f0 100644 --- a/cmd/ate-setup/internal/config/coverage_test.go +++ b/cmd/ate-setup/internal/config/coverage_test.go @@ -102,6 +102,8 @@ func valueFor(key string) string { return "ES256" case "actorJWT.issuer": return "https://idp.example.com" + case "ateapi.logLevel": + return "debug" case "csi.setup": return "none" case "preview": diff --git a/cmd/ate-setup/internal/config/credentials.go b/cmd/ate-setup/internal/config/credentials.go index 33e64132fa..d0c14f601e 100644 --- a/cmd/ate-setup/internal/config/credentials.go +++ b/cmd/ate-setup/internal/config/credentials.go @@ -21,7 +21,7 @@ import ( "os/exec" "time" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" ) // EnsureClusterCredentials fetches GKE credentials when the configuration diff --git a/cmd/ate-setup/internal/config/document.go b/cmd/ate-setup/internal/config/document.go index e2e65fdeeb..e02842386c 100644 --- a/cmd/ate-setup/internal/config/document.go +++ b/cmd/ate-setup/internal/config/document.go @@ -22,7 +22,7 @@ import ( "sigs.k8s.io/yaml" - "github.com/agent-substrate/substrate/internal/version" + "go.ate.dev/substrate/internal/version" ) // Outcome is how the run that produced a document ended. diff --git a/cmd/ate-setup/internal/config/load.go b/cmd/ate-setup/internal/config/load.go index 9ce6e53b47..13b2ad9f29 100644 --- a/cmd/ate-setup/internal/config/load.go +++ b/cmd/ate-setup/internal/config/load.go @@ -23,10 +23,10 @@ import ( "github.com/spf13/pflag" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/images" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/internal/oidcdiscovery" - "github.com/agent-substrate/substrate/internal/preview" + "go.ate.dev/substrate/cmd/ate-setup/internal/images" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/oidcdiscovery" + "go.ate.dev/substrate/internal/preview" ) // ConfigPathEnv names the configuration document when --config is not given. @@ -209,17 +209,29 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro if actorJWTIssuer == "" { actorJWTIssuer = installdefaults.ActorJWTIssuer(r.String("namespace")) } + // kubectl-ate, ate-setup's demo steps and the e2e clients fall back to + // tokens minted from the client ServiceAccount, so making it the default + // bootstrap owner keeps them working with authorization on. The deploy + // warns whenever it is a bootstrap owner. + bootstrapOwners := r.String("ateapi.authz.bootstrapOwners") + if bootstrapOwners == "" { + bootstrapOwners = installdefaults.ClientPrincipal(r.String("namespace")) + } cfg := &Config{ - Root: root, - Kind: r.Bool("kindCluster.enabled"), - Namespace: r.String("namespace"), - Kubeconfig: kubeconfig, - Context: r.String("context"), - ProjectID: r.String("gcp.projectID"), - ClusterName: r.String("gcp.clusterName"), - ClusterLocation: r.String("gcp.clusterLocation"), - ExpectedJWTIssuer: r.String("ateapi.expectedJWTIssuer"), + Root: root, + Kind: r.Bool("kindCluster.enabled"), + Namespace: r.String("namespace"), + Kubeconfig: kubeconfig, + Context: r.String("context"), + ProjectID: r.String("gcp.projectID"), + ClusterName: r.String("gcp.clusterName"), + ClusterLocation: r.String("gcp.clusterLocation"), + ExpectedJWTIssuer: r.String("ateapi.expectedJWTIssuer"), + + APIExperimentalEnableAuthz: r.Bool("ateapi.authz.enabled"), + APIAuthzBootstrapOwners: bootstrapOwners, + BucketName: r.String("storage.bucketName"), KODockerRepo: r.String("ko.dockerRepo"), KODefaultPlatforms: r.String("ko.defaultPlatforms"), @@ -266,6 +278,7 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro CredentialProviderJSON: r.String("atenet.egress.credentialProvider"), AnthropicAPIKey: r.String("demo.anthropicAPIKey"), OtlpEndpoint: r.String("otlpEndpoint"), + APILogLevel: r.String("ateapi.logLevel"), BenchmarkActorMemory: r.String("benchmark.actorMemory"), PreviewGates: splitList(r.String("preview")), kubeconfigEnv: kubeconfigEnv, @@ -403,6 +416,13 @@ func validateResolved(cfg *Config, r *Resolved) error { return &InvalidError{Value: authMode, Want: IngressAuthDeprecatedInsecure + " or " + IngressAuthStaticMTLS} } + switch cfg.APILogLevel { + case "", "debug", "info", "warn", "error": + default: + level, _ := r.Value("ateapi.logLevel") + return &InvalidError{Value: level, Want: "debug, info, warn, or error"} + } + switch cfg.ActorJWTAlgorithm { case "ES256", "RS256": default: @@ -441,5 +461,12 @@ func validateResolved(cfg *Config, r *Resolved) error { return fmt.Errorf("%w (from %s)", err, provider.From.Describe(provider.Setting)) } } + // ate-api-server exits on this combination, so catch it before deploying. + if cfg.APIExperimentalEnableAuthz && strings.Trim(cfg.APIAuthzBootstrapOwners, " ,") == "" { + enabled, _ := r.Value("ateapi.authz.enabled") + owners, _ := r.Value("ateapi.authz.bootstrapOwners") + return &ConflictError{A: enabled, B: owners, + Why: "enforced authorization needs at least one bootstrap owner"} + } return nil } diff --git a/cmd/ate-setup/internal/config/setting.go b/cmd/ate-setup/internal/config/setting.go index 1de019c875..f6d829bc73 100644 --- a/cmd/ate-setup/internal/config/setting.go +++ b/cmd/ate-setup/internal/config/setting.go @@ -24,7 +24,7 @@ import ( "github.com/spf13/pflag" - "github.com/agent-substrate/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/installdefaults" ) // ValueKind is the type of a setting's value. @@ -162,11 +162,28 @@ var Registry = []Setting{ "(requires --atenet-dataplane=envoy)", }, + { + // No default: an unset level keeps the one an existing install has, + // and a new install gets ate-setup's default. + Key: "ateapi.logLevel", Env: "ATE_API_LOG_LEVEL", Flag: "ateapi-log-level", Kind: KindString, + Usage: "ate-api-server log level: debug, info, warn, or error (default: keep the installed level, else info)", + }, { Key: "ateapi.expectedJWTIssuer", Env: "EXPECTED_JWT_ISSUER", Flag: "ateapi-expected-jwt-issuer", Kind: KindString, Usage: "Service account token issuer ate-api-server trusts, overriding derivation and discovery", }, + { + Key: "ateapi.authz.enabled", Env: "ATE_API_EXPERIMENTAL_ENABLE_AUTHZ", + Flag: "ateapi-experimental-enable-authz", Kind: KindBool, Default: "true", + Usage: "Enforce ate-api-server's runtime authorization", + }, + { + Key: "ateapi.authz.bootstrapOwners", Env: "ATE_API_AUTHZ_BOOTSTRAP_OWNERS", + Flag: "ateapi-authz-bootstrap-owners", Kind: KindString, + Usage: "Comma-separated principals that always hold owner on global:root when authorization is enforced. " + + "Unset means kubectl-ate's ServiceAccount, system:serviceaccount::ate-client", + }, { // The egress gateway's credential provider, as a JSON object. Upstream // takes it as one value on one flag; it is carried the same way here diff --git a/cmd/ate-setup/internal/demos/all/all.go b/cmd/ate-setup/internal/demos/all/all.go index 6fa8022b74..49fec9da2c 100644 --- a/cmd/ate-setup/internal/demos/all/all.go +++ b/cmd/ate-setup/internal/demos/all/all.go @@ -21,14 +21,14 @@ package all import ( - _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/autoscaledworkerpool" - _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/claudemultiplex" - _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/counter" - _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/countermicrovm" - _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/egress" - _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/egressmicrovm" - _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/jupyter" - _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/multitemplate" - _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/parking" - _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/sandbox" + _ "go.ate.dev/substrate/cmd/ate-setup/internal/demos/autoscaledworkerpool" + _ "go.ate.dev/substrate/cmd/ate-setup/internal/demos/claudemultiplex" + _ "go.ate.dev/substrate/cmd/ate-setup/internal/demos/counter" + _ "go.ate.dev/substrate/cmd/ate-setup/internal/demos/countermicrovm" + _ "go.ate.dev/substrate/cmd/ate-setup/internal/demos/egress" + _ "go.ate.dev/substrate/cmd/ate-setup/internal/demos/egressmicrovm" + _ "go.ate.dev/substrate/cmd/ate-setup/internal/demos/jupyter" + _ "go.ate.dev/substrate/cmd/ate-setup/internal/demos/multitemplate" + _ "go.ate.dev/substrate/cmd/ate-setup/internal/demos/parking" + _ "go.ate.dev/substrate/cmd/ate-setup/internal/demos/sandbox" ) diff --git a/cmd/ate-setup/internal/demos/all/all_test.go b/cmd/ate-setup/internal/demos/all/all_test.go index f3e640f00b..61ed686895 100644 --- a/cmd/ate-setup/internal/demos/all/all_test.go +++ b/cmd/ate-setup/internal/demos/all/all_test.go @@ -19,9 +19,9 @@ import ( "path/filepath" "testing" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos" - _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/all" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/demos" + _ "go.ate.dev/substrate/cmd/ate-setup/internal/demos/all" ) // nonDemoPackages are the directories under internal/demos that are support diff --git a/cmd/ate-setup/internal/demos/autoscaledworkerpool/autoscaledworkerpool.go b/cmd/ate-setup/internal/demos/autoscaledworkerpool/autoscaledworkerpool.go index 5f20d048a7..4b246f6b16 100644 --- a/cmd/ate-setup/internal/demos/autoscaledworkerpool/autoscaledworkerpool.go +++ b/cmd/ate-setup/internal/demos/autoscaledworkerpool/autoscaledworkerpool.go @@ -23,11 +23,11 @@ package autoscaledworkerpool import ( "context" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/cmd/ate-setup/internal/demos" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/cmd/ate-setup/internal/steps" + "go.ate.dev/substrate/internal/resources" ) // namespace holds the demo workload, its prometheus-adapter, and the HPA; diff --git a/cmd/ate-setup/internal/demos/claudemultiplex/claudemultiplex.go b/cmd/ate-setup/internal/demos/claudemultiplex/claudemultiplex.go index 0555d79417..f8b7db51b3 100644 --- a/cmd/ate-setup/internal/demos/claudemultiplex/claudemultiplex.go +++ b/cmd/ate-setup/internal/demos/claudemultiplex/claudemultiplex.go @@ -24,11 +24,11 @@ import ( "context" "fmt" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/images" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/demos" + "go.ate.dev/substrate/cmd/ate-setup/internal/images" + "go.ate.dev/substrate/cmd/ate-setup/internal/steps" + "go.ate.dev/substrate/internal/resources" ) const ( diff --git a/cmd/ate-setup/internal/demos/claudemultiplex/claudemultiplex_test.go b/cmd/ate-setup/internal/demos/claudemultiplex/claudemultiplex_test.go index fd16cf12d4..a3b15a609e 100644 --- a/cmd/ate-setup/internal/demos/claudemultiplex/claudemultiplex_test.go +++ b/cmd/ate-setup/internal/demos/claudemultiplex/claudemultiplex_test.go @@ -17,8 +17,8 @@ package claudemultiplex import ( "testing" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/demotest" + "go.ate.dev/substrate/cmd/ate-setup/internal/demos" + "go.ate.dev/substrate/cmd/ate-setup/internal/demos/demotest" ) // TestRenderManifests covers this demo's manifests, which the shared render diff --git a/cmd/ate-setup/internal/demos/counter/counter.go b/cmd/ate-setup/internal/demos/counter/counter.go index 39f5848036..a424529a2b 100644 --- a/cmd/ate-setup/internal/demos/counter/counter.go +++ b/cmd/ate-setup/internal/demos/counter/counter.go @@ -20,10 +20,10 @@ package counter import ( "context" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/demos" + "go.ate.dev/substrate/cmd/ate-setup/internal/steps" + "go.ate.dev/substrate/internal/resources" ) const ( diff --git a/cmd/ate-setup/internal/demos/counter/counter_test.go b/cmd/ate-setup/internal/demos/counter/counter_test.go index ada06a8977..05630230c2 100644 --- a/cmd/ate-setup/internal/demos/counter/counter_test.go +++ b/cmd/ate-setup/internal/demos/counter/counter_test.go @@ -19,9 +19,9 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/demotest" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/cmd/ate-setup/internal/demos" + "go.ate.dev/substrate/cmd/ate-setup/internal/demos/demotest" + "go.ate.dev/substrate/internal/resources" ) // TestExternalVolumeRenders covers the substitution branch of the counter diff --git a/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go b/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go index e23225370a..52a5036e62 100644 --- a/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go +++ b/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go @@ -20,9 +20,9 @@ package countermicrovm import ( - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/cmd/ate-setup/internal/demos" + "go.ate.dev/substrate/cmd/ate-setup/internal/steps" + "go.ate.dev/substrate/internal/resources" ) // namespace is the pool's k8s namespace; it doubles as the atespace holding diff --git a/cmd/ate-setup/internal/demos/demos.go b/cmd/ate-setup/internal/demos/demos.go index 4bd90dbd1a..ff99a65f92 100644 --- a/cmd/ate-setup/internal/demos/demos.go +++ b/cmd/ate-setup/internal/demos/demos.go @@ -27,8 +27,8 @@ import ( "sort" "strings" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/steps" ) // Demo is one installable example workload. diff --git a/cmd/ate-setup/internal/demos/demotest/demotest.go b/cmd/ate-setup/internal/demos/demotest/demotest.go index 66395ed54e..9c5550caf4 100644 --- a/cmd/ate-setup/internal/demos/demotest/demotest.go +++ b/cmd/ate-setup/internal/demos/demotest/demotest.go @@ -22,10 +22,10 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/cmd/ate-setup/internal/steps" + "go.ate.dev/substrate/internal/resources" ) // Env builds an Env with no cluster connection. The render paths only read diff --git a/cmd/ate-setup/internal/demos/egress/egress.go b/cmd/ate-setup/internal/demos/egress/egress.go index b4f4e926cd..1766301125 100644 --- a/cmd/ate-setup/internal/demos/egress/egress.go +++ b/cmd/ate-setup/internal/demos/egress/egress.go @@ -22,9 +22,9 @@ package egress import ( - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/cmd/ate-setup/internal/demos" + "go.ate.dev/substrate/cmd/ate-setup/internal/steps" + "go.ate.dev/substrate/internal/resources" ) // namespace is the pool's k8s namespace; it doubles as the atespace holding diff --git a/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go b/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go index 67f8c4342a..7c8146f494 100644 --- a/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go +++ b/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go @@ -23,9 +23,9 @@ package egressmicrovm import ( - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/cmd/ate-setup/internal/demos" + "go.ate.dev/substrate/cmd/ate-setup/internal/steps" + "go.ate.dev/substrate/internal/resources" ) // namespace is the pool's k8s namespace; it doubles as the atespace holding diff --git a/cmd/ate-setup/internal/demos/jupyter/jupyter.go b/cmd/ate-setup/internal/demos/jupyter/jupyter.go index b18bb4fddc..6dd6057354 100644 --- a/cmd/ate-setup/internal/demos/jupyter/jupyter.go +++ b/cmd/ate-setup/internal/demos/jupyter/jupyter.go @@ -18,9 +18,9 @@ package jupyter import ( - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/cmd/ate-setup/internal/demos" + "go.ate.dev/substrate/cmd/ate-setup/internal/steps" + "go.ate.dev/substrate/internal/resources" ) // namespace is the pool's k8s namespace; it doubles as the atespace holding diff --git a/cmd/ate-setup/internal/demos/multitemplate/multitemplate.go b/cmd/ate-setup/internal/demos/multitemplate/multitemplate.go index 8f36dbb91b..45dd1b9085 100644 --- a/cmd/ate-setup/internal/demos/multitemplate/multitemplate.go +++ b/cmd/ate-setup/internal/demos/multitemplate/multitemplate.go @@ -17,9 +17,9 @@ package multitemplate import ( - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/cmd/ate-setup/internal/demos" + "go.ate.dev/substrate/cmd/ate-setup/internal/steps" + "go.ate.dev/substrate/internal/resources" ) func init() { diff --git a/cmd/ate-setup/internal/demos/parking/parking.go b/cmd/ate-setup/internal/demos/parking/parking.go index e002f99340..80d24d0474 100644 --- a/cmd/ate-setup/internal/demos/parking/parking.go +++ b/cmd/ate-setup/internal/demos/parking/parking.go @@ -17,9 +17,9 @@ package parking import ( - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/cmd/ate-setup/internal/demos" + "go.ate.dev/substrate/cmd/ate-setup/internal/steps" + "go.ate.dev/substrate/internal/resources" ) // namespace is the pool's k8s namespace; it doubles as the atespace holding diff --git a/cmd/ate-setup/internal/demos/render.go b/cmd/ate-setup/internal/demos/render.go index d5061fd43e..941d1d3dc1 100644 --- a/cmd/ate-setup/internal/demos/render.go +++ b/cmd/ate-setup/internal/demos/render.go @@ -15,8 +15,8 @@ package demos import ( - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/render" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" + "go.ate.dev/substrate/cmd/ate-setup/internal/render" + "go.ate.dev/substrate/cmd/ate-setup/internal/steps" ) // bucketNamePlaceholder is substituted into every demo template with the diff --git a/cmd/ate-setup/internal/demos/render_test.go b/cmd/ate-setup/internal/demos/render_test.go index 144ea11e77..ebd48583bb 100644 --- a/cmd/ate-setup/internal/demos/render_test.go +++ b/cmd/ate-setup/internal/demos/render_test.go @@ -19,9 +19,9 @@ package demos_test import ( "testing" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos" - _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/all" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/demotest" + "go.ate.dev/substrate/cmd/ate-setup/internal/demos" + _ "go.ate.dev/substrate/cmd/ate-setup/internal/demos/all" + "go.ate.dev/substrate/cmd/ate-setup/internal/demos/demotest" ) // TestDemoTemplatesRender guards against drift between the demo templates and diff --git a/cmd/ate-setup/internal/demos/sandbox/sandbox.go b/cmd/ate-setup/internal/demos/sandbox/sandbox.go index ca412d489c..2b8ceae726 100644 --- a/cmd/ate-setup/internal/demos/sandbox/sandbox.go +++ b/cmd/ate-setup/internal/demos/sandbox/sandbox.go @@ -17,9 +17,9 @@ package sandbox import ( - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/cmd/ate-setup/internal/demos" + "go.ate.dev/substrate/cmd/ate-setup/internal/steps" + "go.ate.dev/substrate/internal/resources" ) // namespace is the pool's k8s namespace; it doubles as the atespace holding diff --git a/cmd/ate-setup/internal/demos/substrate.go b/cmd/ate-setup/internal/demos/substrate.go index 2c129d0e64..e5812d847b 100644 --- a/cmd/ate-setup/internal/demos/substrate.go +++ b/cmd/ate-setup/internal/demos/substrate.go @@ -18,11 +18,11 @@ import ( "context" "time" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/cmd/ate-setup/internal/steps" + "go.ate.dev/substrate/internal/resources" ) // MicroVMGoldenTimeout is the golden-snapshot budget the micro-VM demos set as diff --git a/cmd/ate-setup/internal/images/images.go b/cmd/ate-setup/internal/images/images.go index 0cfa2e4354..6cc862dd7c 100644 --- a/cmd/ate-setup/internal/images/images.go +++ b/cmd/ate-setup/internal/images/images.go @@ -28,7 +28,7 @@ import ( // ModulePath is this repository's module path. A ko:// reference is it joined // with a component's package. -const ModulePath = "github.com/agent-substrate/substrate" +const ModulePath = "go.ate.dev/substrate" // Components is every package the installer can deploy a published image for, // as an import path within the module. diff --git a/cmd/ate-setup/internal/images/images_test.go b/cmd/ate-setup/internal/images/images_test.go index 523e01950f..293da7e64d 100644 --- a/cmd/ate-setup/internal/images/images_test.go +++ b/cmd/ate-setup/internal/images/images_test.go @@ -25,8 +25,8 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/images" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/images" ) // installTrees are the directories the installer reads manifests from. diff --git a/cmd/ate-setup/internal/images/prebuilt_test.go b/cmd/ate-setup/internal/images/prebuilt_test.go index cbf6acc74c..1264caa09a 100644 --- a/cmd/ate-setup/internal/images/prebuilt_test.go +++ b/cmd/ate-setup/internal/images/prebuilt_test.go @@ -25,9 +25,9 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/images" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/images" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" ) // testSource is the repo/tag every rewrite test resolves against. @@ -68,29 +68,29 @@ func TestPrebuiltResolveBytes(t *testing.T) { }{ { name: "bare scalar", - in: " image: ko://github.com/agent-substrate/substrate/cmd/ateapi\n", + in: " image: ko://go.ate.dev/substrate/cmd/ateapi\n", want: " image: example.com/substrate/ateapi:v1.2.3" + digestSuffix + "\n", }, { name: "double quoted", - in: `image: "ko://github.com/agent-substrate/substrate/cmd/atelet"`, + in: `image: "ko://go.ate.dev/substrate/cmd/atelet"`, want: `image: "example.com/substrate/atelet:v1.2.3` + digestSuffix + `"`, }, { name: "single quoted", - in: `image: 'ko://github.com/agent-substrate/substrate/cmd/atenet'`, + in: `image: 'ko://go.ate.dev/substrate/cmd/atenet'`, want: `image: 'example.com/substrate/atenet:v1.2.3` + digestSuffix + `'`, }, { name: "flow sequence", - in: `images: [ko://github.com/agent-substrate/substrate/demos/counter, other]`, + in: `images: [ko://go.ate.dev/substrate/demos/counter, other]`, want: `images: [example.com/substrate/counter:v1.2.3` + digestSuffix + `, other]`, }, { // The reference is a plain CRD field here, not a pod spec, which is // why the rewrite cannot be driven off a Kubernetes schema. name: "CRD field", - in: "spec:\n workerImage: ko://github.com/agent-substrate/substrate/cmd/ateworker-gvisor\n", + in: "spec:\n workerImage: ko://go.ate.dev/substrate/cmd/ateworker-gvisor\n", want: "spec:\n workerImage: example.com/substrate/ateworker-gvisor:v1.2.3" + digestSuffix + "\n", }, { @@ -99,17 +99,17 @@ func TestPrebuiltResolveBytes(t *testing.T) { // expected output is also what catches a redundant lookup. name: "tag already carries a digest", src: images.Source{Repo: "example.com/substrate", Tag: "v1.2.3" + digestSuffix}, - in: "image: ko://github.com/agent-substrate/substrate/cmd/ateapi\n", + in: "image: ko://go.ate.dev/substrate/cmd/ateapi\n", want: "image: example.com/substrate/ateapi:v1.2.3" + digestSuffix + "\n", }, { name: "nested package maps to its image name", - in: "image: ko://github.com/agent-substrate/substrate/demos/multi-template/fspersist\n", + in: "image: ko://go.ate.dev/substrate/demos/multi-template/fspersist\n", want: "image: example.com/substrate/fspersist:v1.2.3" + digestSuffix + "\n", }, { name: "two references on one line", - in: "a: ko://github.com/agent-substrate/substrate/cmd/ateapi, b: ko://github.com/agent-substrate/substrate/demos/egress\n", + in: "a: ko://go.ate.dev/substrate/cmd/ateapi, b: ko://go.ate.dev/substrate/demos/egress\n", want: "a: example.com/substrate/ateapi:v1.2.3" + digestSuffix + ", b: example.com/substrate/egress:v1.2.3" + digestSuffix + "\n", }, { @@ -133,14 +133,14 @@ func TestPrebuiltResolveBytes(t *testing.T) { // message and nothing else: the truncation cannot collide with a // listed package, because every one of those ends before the "$". name: "unexpanded placeholder", - in: "image: ko://github.com/agent-substrate/substrate/cmd/ateworker-${SANDBOX_CLASS}\n", + in: "image: ko://go.ate.dev/substrate/cmd/ateworker-${SANDBOX_CLASS}\n", error: "cmd/ateworker-${SANDBOX_CLASS has no published image", }, { // A reference extending a known one must not inherit its image. A // literal replacement would leave "-sidecar" dangling on the tag. name: "reference extending a known one", - in: "image: ko://github.com/agent-substrate/substrate/cmd/atelet-sidecar\n", + in: "image: ko://go.ate.dev/substrate/cmd/atelet-sidecar\n", error: "cmd/atelet-sidecar has no published image", }, { @@ -168,20 +168,20 @@ func TestPrebuiltResolveBytes(t *testing.T) { }, { name: "unknown component", - in: "image: ko://github.com/agent-substrate/substrate/cmd/notacomponent\n", + in: "image: ko://go.ate.dev/substrate/cmd/notacomponent\n", error: "cmd/notacomponent has no published image", }, { // The e2e fixtures are not part of an install, so a manifest // reaching the installer with one is a mistake worth reporting. name: "e2e fixture is not installable", - in: "image: ko://github.com/agent-substrate/substrate/internal/e2e/fixtures/probe\n", + in: "image: ko://go.ate.dev/substrate/internal/e2e/fixtures/probe\n", error: "e2e/fixtures/probe has no published image", }, { // Every unmappable reference is named, not just the first. name: "all failures are reported", - in: "a: ko://github.com/agent-substrate/substrate/cmd/one\nb: ko://github.com/agent-substrate/substrate/cmd/two\n", + in: "a: ko://go.ate.dev/substrate/cmd/one\nb: ko://go.ate.dev/substrate/cmd/two\n", error: "cmd/two has no published image", }, } @@ -217,9 +217,9 @@ func TestPrebuiltResolveBytes(t *testing.T) { // Every unmappable reference is reported at once, so a manifest with several // problems takes one install attempt to diagnose rather than several. func TestPrebuiltReportsEveryFailure(t *testing.T) { - in := "a: ko://github.com/agent-substrate/substrate/cmd/ateworker-${SANDBOX_CLASS}\n" + + in := "a: ko://go.ate.dev/substrate/cmd/ateworker-${SANDBOX_CLASS}\n" + "b: ko://github.com/example/other/cmd/thing\n" + - "c: ko://github.com/agent-substrate/substrate/cmd/nope\n" + "c: ko://go.ate.dev/substrate/cmd/nope\n" _, err := images.NewPrebuilt(testSource, newStubRegistry().digest). ResolveBytes(context.Background(), []byte(in)) @@ -235,7 +235,7 @@ func TestPrebuiltReportsEveryFailure(t *testing.T) { // A repeated bad reference is one problem, not one per occurrence. func TestPrebuiltDeduplicatesFailures(t *testing.T) { - ref := "image: ko://github.com/agent-substrate/substrate/cmd/nope\n" + ref := "image: ko://go.ate.dev/substrate/cmd/nope\n" _, err := images.NewPrebuilt(testSource, newStubRegistry().digest). ResolveBytes(context.Background(), []byte(ref+ref+ref)) if err == nil { @@ -253,9 +253,9 @@ func TestPrebuiltLooksUpEachImageOnce(t *testing.T) { registry := newStubRegistry() resolver := images.NewPrebuilt(testSource, registry.digest) - in := "a: ko://github.com/agent-substrate/substrate/cmd/ateapi\n" + - "b: ko://github.com/agent-substrate/substrate/cmd/atelet\n" + - "c: ko://github.com/agent-substrate/substrate/cmd/ateapi\n" + in := "a: ko://go.ate.dev/substrate/cmd/ateapi\n" + + "b: ko://go.ate.dev/substrate/cmd/atelet\n" + + "c: ko://go.ate.dev/substrate/cmd/ateapi\n" for range 2 { if _, err := resolver.ResolveBytes(context.Background(), []byte(in)); err != nil { t.Fatalf("ResolveBytes() error = %v", err) @@ -278,8 +278,8 @@ func TestPrebuiltReportsDigestFailures(t *testing.T) { registry := newStubRegistry() registry.err = errors.New("UNAUTHORIZED") - in := "a: ko://github.com/agent-substrate/substrate/cmd/ateapi\n" + - "b: ko://github.com/agent-substrate/substrate/cmd/atelet\n" + in := "a: ko://go.ate.dev/substrate/cmd/ateapi\n" + + "b: ko://go.ate.dev/substrate/cmd/atelet\n" got, err := images.NewPrebuilt(testSource, registry.digest). ResolveBytes(context.Background(), []byte(in)) if err == nil { diff --git a/cmd/ate-setup/internal/ko/ko.go b/cmd/ate-setup/internal/ko/ko.go index 9c64428929..5b6fea21e4 100644 --- a/cmd/ate-setup/internal/ko/ko.go +++ b/cmd/ate-setup/internal/ko/ko.go @@ -33,11 +33,11 @@ import ( "strings" "time" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" ) // versionPkg is the package whose Version variable receives the build stamp. -const versionPkg = "github.com/agent-substrate/substrate/internal/version" +const versionPkg = "go.ate.dev/substrate/internal/version" // baseImportPaths publishes each image as the last element of its import path, // so cmd/atelet becomes $KO_DOCKER_REPO/atelet. diff --git a/cmd/ate-setup/internal/ko/ko_test.go b/cmd/ate-setup/internal/ko/ko_test.go index 911eed5bb3..dfe5d250af 100644 --- a/cmd/ate-setup/internal/ko/ko_test.go +++ b/cmd/ate-setup/internal/ko/ko_test.go @@ -38,13 +38,13 @@ func TestArgsAlwaysRequestBaseImportPaths(t *testing.T) { }{ {"resolve", r.args("resolve", "-f", "manifests/ate-install")}, {"resolve from stdin", r.args("resolve", "-f", "-")}, - {"build", r.args("build", "github.com/agent-substrate/substrate/cmd/ateworker-gvisor")}, + {"build", r.args("build", "go.ate.dev/substrate/cmd/ateworker-gvisor")}, } { t.Run(tc.name, func(t *testing.T) { if !slices.Contains(tc.args, "--base-import-paths") { t.Errorf("args = %v, want --base-import-paths", tc.args) } - if !slices.Contains(tc.args, "--ldflags=-X=github.com/agent-substrate/substrate/internal/version.Version=v0.0.0-test") { + if !slices.Contains(tc.args, "--ldflags=-X=go.ate.dev/substrate/internal/version.Version=v0.0.0-test") { t.Errorf("args = %v, want the version stamp", tc.args) } }) diff --git a/cmd/ate-setup/internal/kube/apply.go b/cmd/ate-setup/internal/kube/apply.go index f0d7f5f502..9d92dbb08f 100644 --- a/cmd/ate-setup/internal/kube/apply.go +++ b/cmd/ate-setup/internal/kube/apply.go @@ -30,7 +30,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/retry" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" ) // crdKind is the kind whose creation invalidates discovery for later objects. diff --git a/cmd/ate-setup/internal/kube/capool.go b/cmd/ate-setup/internal/kube/capool.go index 15bc9f60af..17b22d0833 100644 --- a/cmd/ate-setup/internal/kube/capool.go +++ b/cmd/ate-setup/internal/kube/capool.go @@ -20,7 +20,7 @@ import ( "encoding/pem" "fmt" - "github.com/agent-substrate/substrate/internal/localca" + "go.ate.dev/substrate/internal/localca" ) // CAPoolRootPEM extracts a CA pool Secret's root certificates and returns them diff --git a/cmd/ate-setup/internal/kube/client.go b/cmd/ate-setup/internal/kube/client.go index 5d89ea2253..a165cbdddc 100644 --- a/cmd/ate-setup/internal/kube/client.go +++ b/cmd/ate-setup/internal/kube/client.go @@ -29,7 +29,7 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/restmapper" - "github.com/agent-substrate/substrate/internal/ateclient" + "go.ate.dev/substrate/internal/ateclient" ) // FieldManager identifies ate-setup's writes in managedFields. diff --git a/cmd/ate-setup/internal/kube/wait.go b/cmd/ate-setup/internal/kube/wait.go index 7ba1fc6cc6..d14b0b54cd 100644 --- a/cmd/ate-setup/internal/kube/wait.go +++ b/cmd/ate-setup/internal/kube/wait.go @@ -19,8 +19,8 @@ import ( "fmt" "time" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" - "github.com/agent-substrate/substrate/internal/clustertrustbundle" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/internal/clustertrustbundle" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/cmd/ate-setup/internal/steps/actors.go b/cmd/ate-setup/internal/steps/actors.go index a8d5658c9a..fccfabb556 100644 --- a/cmd/ate-setup/internal/steps/actors.go +++ b/cmd/ate-setup/internal/steps/actors.go @@ -17,8 +17,8 @@ package steps import ( "context" - "github.com/agent-substrate/substrate/internal/ateclient" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/ateclient" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // TemplateRef identifies a demo's ActorTemplate. Actors are deleted by matching diff --git a/cmd/ate-setup/internal/steps/apienv.go b/cmd/ate-setup/internal/steps/apienv.go index da988eb528..f59b560d1b 100644 --- a/cmd/ate-setup/internal/steps/apienv.go +++ b/cmd/ate-setup/internal/steps/apienv.go @@ -22,11 +22,13 @@ import ( "maps" "os" "slices" + "strings" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/internal/installdefaults" ) // envHashAnnotation carries a digest of the apiserver's environment on the pod @@ -147,6 +149,27 @@ func (e *Env) UpdateAPIServerEnvVars(ctx context.Context) error { return e.annotateAPIServerEnvHash(ctx) } +// sharedBootstrapOwnerWarning returns a warning when enforced authorization +// makes the shared client ServiceAccount a bootstrap owner, and "" otherwise. +// It is the default so demos, benchmarks and e2e work, but everyone who can +// mint its token, including every kubectl-ate user without --token-file, is +// then a global owner. +func sharedBootstrapOwnerWarning(enabled bool, bootstrapOwners, namespace string) string { + if !enabled { + return "" + } + shared := installdefaults.ClientPrincipal(namespace) + for owner := range strings.SplitSeq(bootstrapOwners, ",") { + if strings.TrimSpace(owner) == shared { + return fmt.Sprintf("%s is a bootstrap owner, so anyone who can mint a token for it is a global owner of this install. "+ + "It is the default so the demos, benchmarks and e2e tests work; for a shared or production install, "+ + "set --ateapi-authz-bootstrap-owners (ATE_API_AUTHZ_BOOTSTRAP_OWNERS) to your operators' own principals. "+ + "See docs/authorization.md.", shared) + } + } + return "" +} + // buildAPIServerEnvVars is the Secret payload. ate-api-server takes both // connection strings and the schema from it, and exits on an empty schema; an // unrecognized key here reaches the container as a stray environment variable. @@ -292,9 +315,69 @@ func (e *Env) applyWithAPIServerEnvHash(ctx context.Context, manifest []byte) er if err := stampAPIServerEnvHash(objs, hash); err != nil { return err } + if err := setAPIServerAuthzArgs(objs, e.Cfg.APIExperimentalEnableAuthz, e.Cfg.APIAuthzBootstrapOwners); err != nil { + return err + } + if msg := sharedBootstrapOwnerWarning(e.Cfg.APIExperimentalEnableAuthz, e.Cfg.APIAuthzBootstrapOwners, e.Namespace()); msg != "" { + log.Warnf("%s", msg) + } return e.Kube.Apply(ctx, objs) } +// apiServerAuthzFlags are the ate-api-server flags setAPIServerAuthzArgs owns. +var apiServerAuthzFlags = []string{"--experimental-enable-authz", "--authz-bootstrap-owners"} + +// setAPIServerAuthzArgs sets ate-api-server's authorization flags on its +// container in objs, replacing any the manifest already carries. A changed +// flag changes the pod template, so it rolls the Deployment. +// +// The bootstrap owners are passed even with enforcement off: access policy +// RPCs are checked either way, so without them nobody could set up policies +// before turning enforcement on. +func setAPIServerAuthzArgs(objs []*unstructured.Unstructured, enabled bool, bootstrapOwners string) error { + var dep *unstructured.Unstructured + for _, obj := range objs { + if obj.GetKind() == "Deployment" && obj.GetName() == "ate-api-server" { + dep = obj + break + } + } + if dep == nil { + return fmt.Errorf("the manifest has no deployment/ate-api-server to set authorization flags on") + } + containers, found, err := unstructured.NestedSlice(dep.Object, "spec", "template", "spec", "containers") + if err != nil || !found { + return fmt.Errorf("%s has no containers", kube.Describe(dep)) + } + for i, c := range containers { + container, ok := c.(map[string]any) + if !ok || container["name"] != "ate-api-server" { + continue + } + args, _, err := unstructured.NestedStringSlice(container, "args") + if err != nil { + return fmt.Errorf("%s: %w", kube.Describe(dep), err) + } + args = slices.DeleteFunc(args, func(arg string) bool { + return slices.ContainsFunc(apiServerAuthzFlags, func(flag string) bool { + return arg == flag || strings.HasPrefix(arg, flag+"=") + }) + }) + if enabled { + args = append(args, "--experimental-enable-authz=true") + } + if strings.Trim(bootstrapOwners, " ,") != "" { + args = append(args, "--authz-bootstrap-owners="+bootstrapOwners) + } + if err := unstructured.SetNestedStringSlice(container, args, "args"); err != nil { + return fmt.Errorf("%s: %w", kube.Describe(dep), err) + } + containers[i] = container + return unstructured.SetNestedSlice(dep.Object, containers, "spec", "template", "spec", "containers") + } + return fmt.Errorf("%s has no ate-api-server container", kube.Describe(dep)) +} + // stampAPIServerEnvHash sets the digest on the ate-api-server Deployment's pod // template and leaves every other object alone. A manifest without that // Deployment is an error: applying it unstamped would stop a changed diff --git a/cmd/ate-setup/internal/steps/apienv_test.go b/cmd/ate-setup/internal/steps/apienv_test.go index 903aeeef17..2371e48dc5 100644 --- a/cmd/ate-setup/internal/steps/apienv_test.go +++ b/cmd/ate-setup/internal/steps/apienv_test.go @@ -18,14 +18,15 @@ import ( "strings" "testing" + "github.com/google/go-cmp/cmp" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" - "github.com/agent-substrate/substrate/pkg/postgressetup" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/pkg/postgressetup" ) // The digest exists to turn an envFrom change into a rollout, so what matters @@ -610,3 +611,142 @@ spec: t.Errorf("stampAPIServerEnvHash() without ate-api-server error = nil, want an error") } } + +// Authorization reaches ate-api-server as flags on its container, so a change +// rolls the Deployment, and the envvars ConfigMap carries none of it. +func TestSetAPIServerAuthzArgs(t *testing.T) { + const manifest = `apiVersion: apps/v1 +kind: Deployment +metadata: + name: ate-api-server + namespace: ate-system +spec: + template: + spec: + containers: + - name: objectstore-plugin + args: ["--keep"] + - name: ate-api-server + args: + - --config-file=/run/apiconfig.ate.dev/apiconfig.textproto + - --authz-bootstrap-owners=stale + - --experimental-enable-authz +` + for _, tc := range []struct { + name string + enabled bool + owners string + want []string + }{ + { + name: "enforced", + enabled: true, + owners: "alice,bob", + want: []string{ + "--config-file=/run/apiconfig.ate.dev/apiconfig.textproto", + "--experimental-enable-authz=true", + "--authz-bootstrap-owners=alice,bob", + }, + }, + { + // Owners stay, so access policies can be set up before enforcement is on. + name: "off keeps the owners", + owners: "alice", + want: []string{ + "--config-file=/run/apiconfig.ate.dev/apiconfig.textproto", + "--authz-bootstrap-owners=alice", + }, + }, + { + name: "off with no owners", + want: []string{"--config-file=/run/apiconfig.ate.dev/apiconfig.textproto"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + objs, err := kube.DecodeManifestBytes([]byte(manifest)) + if err != nil { + t.Fatalf("DecodeManifestBytes() error = %v", err) + } + if err := setAPIServerAuthzArgs(objs, tc.enabled, tc.owners); err != nil { + t.Fatalf("setAPIServerAuthzArgs() error = %v", err) + } + containers, _, _ := unstructured.NestedSlice(objs[0].Object, "spec", "template", "spec", "containers") + argsOf := func(i int) []string { + args, _, _ := unstructured.NestedStringSlice(containers[i].(map[string]any), "args") + return args + } + if diff := cmp.Diff(tc.want, argsOf(1)); diff != "" { + t.Errorf("ate-api-server args (-want +got):\n%s", diff) + } + if diff := cmp.Diff([]string{"--keep"}, argsOf(0)); diff != "" { + t.Errorf("other container's args changed (-want +got):\n%s", diff) + } + }) + } + + objs, err := kube.DecodeManifestBytes([]byte("apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: ate-controller\n")) + if err != nil { + t.Fatalf("DecodeManifestBytes() error = %v", err) + } + if err := setAPIServerAuthzArgs(objs, true, "alice"); err == nil { + t.Error("setAPIServerAuthzArgs() without deployment/ate-api-server error = nil, want an error") + } +} + +func TestCreateAPIServerEnvVarsOmitsAuthz(t *testing.T) { + e := &Env{Cfg: &config.Config{ + PostgresReadWriteRole: config.DefaultPostgresReadWriteRole, + PostgresOwnerRole: config.DefaultPostgresOwnerRole, + APIExperimentalEnableAuthz: true, + APIAuthzBootstrapOwners: "system:serviceaccount:ate-system:ate-client", + }, Kube: fakeKube(t, + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: NamespaceAteSystem}}, + apiServerEnvVarsConfigMap(map[string]string{}), + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: NamespaceAteSystem, Name: SecretAPIEnvVars}}, + )} + if err := e.CreateAPIServerEnvVars(t.Context()); err != nil { + t.Fatalf("CreateAPIServerEnvVars() error = %v", err) + } + cm, err := e.Kube.GetConfigMap(t.Context(), NamespaceAteSystem, ConfigMapAPIEnvVars) + if err != nil || cm == nil { + t.Fatalf("GetConfigMap() = %v, %v", cm, err) + } + for _, key := range []string{"ATE_API_EXPERIMENTAL_ENABLE_AUTHZ", "ATE_API_AUTHZ_BOOTSTRAP_OWNERS"} { + if v, ok := cm.Data[key]; ok { + t.Errorf("ConfigMap carries %s=%q; authorization is passed as flags", key, v) + } + } +} + +// The shared client ServiceAccount as a bootstrap owner makes everyone who can +// mint its token a global owner, so the install says so whenever it is listed. +func TestSharedBootstrapOwnerWarning(t *testing.T) { + const shared = "system:serviceaccount:ate-system:ate-client" + for _, tc := range []struct { + name string + enabled bool + owners string + warn bool + }{ + {name: "default owner", enabled: true, owners: shared, warn: true}, + {name: "listed among others", enabled: true, owners: "alice, " + shared + " ,bob", warn: true}, + {name: "own principals only", enabled: true, owners: "alice,bob", warn: false}, + {name: "another namespace's client", enabled: true, owners: "system:serviceaccount:other:ate-client", warn: false}, + {name: "prefix is not a match", enabled: true, owners: shared + "-x", warn: false}, + {name: "authorization off", enabled: false, owners: shared, warn: false}, + } { + t.Run(tc.name, func(t *testing.T) { + got := sharedBootstrapOwnerWarning(tc.enabled, tc.owners, "ate-system") + if (got != "") != tc.warn { + t.Fatalf("sharedBootstrapOwnerWarning(%v, %q) = %q, want warning: %v", tc.enabled, tc.owners, got, tc.warn) + } + if tc.warn { + for _, want := range []string{shared, "--ateapi-authz-bootstrap-owners", "ATE_API_AUTHZ_BOOTSTRAP_OWNERS"} { + if !strings.Contains(got, want) { + t.Errorf("warning %q does not name %q", got, want) + } + } + } + }) + } +} diff --git a/cmd/ate-setup/internal/steps/benchmarks.go b/cmd/ate-setup/internal/steps/benchmarks.go index abc8d6e920..68c51d020c 100644 --- a/cmd/ate-setup/internal/steps/benchmarks.go +++ b/cmd/ate-setup/internal/steps/benchmarks.go @@ -21,8 +21,8 @@ import ( "os/exec" "strconv" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" ) // The benchmark and micro-VM stacks are still driven by shell. They orchestrate diff --git a/cmd/ate-setup/internal/steps/benchmarks_test.go b/cmd/ate-setup/internal/steps/benchmarks_test.go index ce85a93748..221e94c7b3 100644 --- a/cmd/ate-setup/internal/steps/benchmarks_test.go +++ b/cmd/ate-setup/internal/steps/benchmarks_test.go @@ -18,7 +18,7 @@ import ( "slices" "testing" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" ) // deploy_locust.sh takes these as flags only, so anything not forwarded here is diff --git a/cmd/ate-setup/internal/steps/cloudsql.go b/cmd/ate-setup/internal/steps/cloudsql.go index d2b0c9892d..94308571a3 100644 --- a/cmd/ate-setup/internal/steps/cloudsql.go +++ b/cmd/ate-setup/internal/steps/cloudsql.go @@ -20,8 +20,8 @@ import ( "os" "strings" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" ) // Keys the Cloud SQL Auth Proxy sidecar reads out of the ate-api-server-envvars diff --git a/cmd/ate-setup/internal/steps/cloudsql_test.go b/cmd/ate-setup/internal/steps/cloudsql_test.go index 4c1f6e2414..69a901340e 100644 --- a/cmd/ate-setup/internal/steps/cloudsql_test.go +++ b/cmd/ate-setup/internal/steps/cloudsql_test.go @@ -28,8 +28,8 @@ import ( "k8s.io/client-go/kubernetes/fake" "sigs.k8s.io/yaml" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" ) // fakeKube builds a kube.Client backed by the fake clientset, seeded with diff --git a/cmd/ate-setup/internal/steps/create.go b/cmd/ate-setup/internal/steps/create.go index 6cc21de88b..7c1c9127c2 100644 --- a/cmd/ate-setup/internal/steps/create.go +++ b/cmd/ate-setup/internal/steps/create.go @@ -15,17 +15,20 @@ package steps import ( + "bytes" "context" "fmt" "strings" "time" + "google.golang.org/protobuf/encoding/prototext" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" - "github.com/agent-substrate/substrate/internal/localca" - "github.com/agent-substrate/substrate/internal/localjwtauthority" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/internal/localca" + "go.ate.dev/substrate/internal/localjwtauthority" + "go.ate.dev/substrate/pkg/proto/ateconfigpb" ) // Secret and ConfigMap names the control plane reads. @@ -41,6 +44,7 @@ const ( SecretPostgresServerCA = "postgres-server-ca" ConfigMapAPIEnvVars = "ate-api-server-envvars" ConfigMapAPIAuthn = "ate-api-authentication" + ConfigMapAPIConfig = "ate-api-config" // poolKeyID is the identifier given to the first CA in a new pool, // matching the --ca-id the shell scripts passed. poolKeyID = "1" @@ -51,6 +55,7 @@ const ( const caValidity = 365 * 24 * time.Hour // CreateJWTAuthorityPoolSecret generates the actor-identity JWT signing pool. +// An existing pool is kept, and its "jwks" key is brought in step with it. // This is the make-jwt-pool call create_jwt_authority_pool_secret shelled out // to kubectl-ate for. func (e *Env) CreateJWTAuthorityPoolSecret(ctx context.Context) error { @@ -123,6 +128,45 @@ func (e *Env) CreateActorIDCACertsSecret(ctx context.Context) error { }) } +// defaultAPILogLevel is the ate-api-server log level a new install gets when +// none is asked for. +const defaultAPILogLevel = "info" + +// CreateAPIConfig writes the ate-api-server config file. +func (e *Env) CreateAPIConfig(ctx context.Context) error { + log.Step("create_api_config") + if err := e.Kube.EnsureNamespace(ctx, e.Namespace()); err != nil { + return err + } + + logLevel := e.Cfg.APILogLevel + if logLevel == "" { + logLevel = defaultAPILogLevel + } + apiConfig, err := buildAPIConfig(logLevel) + if err != nil { + return err + } + log.Infof("%s apiconfig.textproto:", ConfigMapAPIConfig) + for _, line := range strings.Split(strings.TrimRight(apiConfig, "\n"), "\n") { + log.Infof(" | %s", line) + } + return e.Kube.ApplyConfigMap(ctx, e.Namespace(), ConfigMapAPIConfig, map[string]string{ + "apiconfig.textproto": apiConfig, + }) +} + +// buildAPIConfig renders apiconfig.textproto. +func buildAPIConfig(logLevel string) (string, error) { + b, err := prototext.MarshalOptions{Multiline: true}.Marshal(&ateconfigpb.APIConfig{ + Logging: &ateconfigpb.APILogging{Level: logLevel}, + }) + if err != nil { + return "", fmt.Errorf("while marshaling the ate-api-server config: %w", err) + } + return string(b), nil +} + // CreateAPIAuthenticationConfig writes the default ate-api-server // authentication config, pointing it at the cluster's service account issuer. func (e *Env) CreateAPIAuthenticationConfig(ctx context.Context) error { @@ -254,13 +298,13 @@ func newCAPoolSecretData(id string, keyType localca.KeyType) (map[string][]byte, // with the CA pools, an existing pool is preserved: rotating the signing key // would invalidate every token already minted from it. func (e *Env) createJWTPool(ctx context.Context, namespace, name string) error { - exists, err := e.Kube.SecretExists(ctx, namespace, name) + existing, err := e.Kube.GetSecret(ctx, namespace, name) if err != nil { return err } - if exists { + if existing != nil { log.Infof(" JWT authority pool %s/%s already exists; keeping it", namespace, name) - return nil + return e.publishJWTPoolKeys(ctx, existing) } data, err := newJWTPoolSecretData(e.Cfg.ActorJWTAlgorithm) @@ -270,14 +314,40 @@ func (e *Env) createJWTPool(ctx context.Context, namespace, name string) error { return e.createPoolSecret(ctx, namespace, name, corev1.SecretTypeOpaque, data) } +// publishJWTPoolKeys makes the "jwks" key of a pool Secret match its pool: +// missing on a Secret from before the key existed, or stale after a tool that +// did not know it rewrote the pool. A stale set would keep a removed key +// trusted, so it is recomputed from the pool every run. A matching Secret is +// left alone. +func (e *Env) publishJWTPoolKeys(ctx context.Context, secret *corev1.Secret) error { + jwks, err := localjwtauthority.PublicJWKS(secret.Data["pool"]) + if err != nil { + return fmt.Errorf("while publishing the public keys of %s/%s: %w", secret.Namespace, secret.Name, err) + } + if bytes.Equal(secret.Data["jwks"], jwks) { + return nil + } + secret.Data["jwks"] = jwks + if _, err := e.Kube.Typed.CoreV1().Secrets(secret.Namespace).Update(ctx, secret, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("while writing jwks to %s/%s: %w", secret.Namespace, secret.Name, err) + } + log.Infof(" published the public keys of %s/%s as jwks", secret.Namespace, secret.Name) + return nil +} + // newJWTPoolSecretData generates a pool with one active authority for -// algorithm, keyed by its thumbprint. +// algorithm, keyed by its thumbprint. "jwks" holds the pool's public keys for +// verifiers that must not see the pool. func newJWTPoolSecretData(algorithm string) (map[string][]byte, error) { poolBytes, _, err := localjwtauthority.GeneratePool(algorithm, "") if err != nil { return nil, fmt.Errorf("while generating the JWT pool: %w", err) } - return map[string][]byte{"pool": poolBytes}, nil + jwks, err := localjwtauthority.PublicJWKS(poolBytes) + if err != nil { + return nil, fmt.Errorf("while publishing the JWT pool's public keys: %w", err) + } + return map[string][]byte{"pool": poolBytes, "jwks": jwks}, nil } // createPoolSecret writes pool state. diff --git a/cmd/ate-setup/internal/steps/create_test.go b/cmd/ate-setup/internal/steps/create_test.go index 67fe9d280b..c818bfcd58 100644 --- a/cmd/ate-setup/internal/steps/create_test.go +++ b/cmd/ate-setup/internal/steps/create_test.go @@ -16,6 +16,7 @@ package steps import ( "bytes" + "context" "crypto/tls" "crypto/x509" "maps" @@ -25,9 +26,13 @@ import ( "github.com/google/go-cmp/cmp" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" - "github.com/agent-substrate/substrate/internal/localca" - "github.com/agent-substrate/substrate/internal/localjwtauthority" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/internal/localca" + "go.ate.dev/substrate/internal/localjwtauthority" ) // ate-api-server requires both connection strings and its schema in the @@ -171,7 +176,7 @@ func TestNewJWTPoolSecretData(t *testing.T) { if err != nil { t.Fatalf("newJWTPoolSecretData() error = %v", err) } - if diff := cmp.Diff([]string{"pool"}, slices.Sorted(maps.Keys(data))); diff != "" { + if diff := cmp.Diff([]string{"jwks", "pool"}, slices.Sorted(maps.Keys(data))); diff != "" { t.Errorf("secret keys differ (-want +got):\n%s", diff) } @@ -179,6 +184,9 @@ func TestNewJWTPoolSecretData(t *testing.T) { if err != nil { t.Fatalf("Unmarshal() error = %v", err) } + if jwks, err := pool.JWKS(); err != nil || !bytes.Equal(jwks, data["jwks"]) { + t.Errorf("jwks = %s, want the pool's public keys (%v)", data["jwks"], err) + } if len(pool.Authorities) != 1 { t.Fatalf("pool has %d authorities, want 1", len(pool.Authorities)) } @@ -231,3 +239,49 @@ func TestEnsurePodCertificateCAs(t *testing.T) { } } } + +func TestCreateJWTPoolPublishesKeysOfExistingPool(t *testing.T) { + poolBytes, _, err := localjwtauthority.GeneratePool("ES256", "") + if err != nil { + t.Fatal(err) + } + wantJWKS, err := localjwtauthority.PublicJWKS(poolBytes) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + data map[string][]byte + wantUpdates int + }{ + {"pool from before jwks", map[string][]byte{"pool": poolBytes}, 1}, + {"pool with a stale jwks", map[string][]byte{"pool": poolBytes, "jwks": []byte(`{"keys":[]}`)}, 1}, + {"pool with jwks", map[string][]byte{"pool": poolBytes, "jwks": wantJWKS}, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + kc := fake.NewSimpleClientset(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ate-system", Name: SecretActorIDJWTPool}, + Data: tc.data, + }) + updates := 0 + kc.PrependReactor("update", "secrets", func(k8stesting.Action) (bool, runtime.Object, error) { + updates++ + return false, nil, nil + }) + e := &Env{Kube: &kube.Client{Typed: kc}} + if err := e.createJWTPool(context.Background(), "ate-system", SecretActorIDJWTPool); err != nil { + t.Fatalf("createJWTPool: %v", err) + } + got, err := kc.CoreV1().Secrets("ate-system").Get(context.Background(), SecretActorIDJWTPool, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got.Data["pool"], poolBytes) || !bytes.Equal(got.Data["jwks"], wantJWKS) { + t.Errorf("secret data = %v keys, want the same pool and its jwks", slices.Sorted(maps.Keys(got.Data))) + } + if updates != tc.wantUpdates { + t.Errorf("updates = %d, want %d", updates, tc.wantUpdates) + } + }) + } +} diff --git a/cmd/ate-setup/internal/steps/credprovider.go b/cmd/ate-setup/internal/steps/credprovider.go index b201c74f27..1ec8f2ab18 100644 --- a/cmd/ate-setup/internal/steps/credprovider.go +++ b/cmd/ate-setup/internal/steps/credprovider.go @@ -17,7 +17,7 @@ package steps import ( "context" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" ) // Manifests for the bundled Kubernetes Secrets credential provider. They sit diff --git a/cmd/ate-setup/internal/steps/credprovider_test.go b/cmd/ate-setup/internal/steps/credprovider_test.go index 176d615616..1539f8ac3f 100644 --- a/cmd/ate-setup/internal/steps/credprovider_test.go +++ b/cmd/ate-setup/internal/steps/credprovider_test.go @@ -19,8 +19,8 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" ) // The deploy steps refer to the provider's objects by name. This checks those diff --git a/cmd/ate-setup/internal/steps/csi.go b/cmd/ate-setup/internal/steps/csi.go index 7455a49e12..1fccb87793 100644 --- a/cmd/ate-setup/internal/steps/csi.go +++ b/cmd/ate-setup/internal/steps/csi.go @@ -29,10 +29,10 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/yaml" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kustomize" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" - "github.com/agent-substrate/substrate/internal/nodepath" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/cmd/ate-setup/internal/kustomize" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/internal/nodepath" ) // workerHostDir is the host directory atelet mounts sandbox images under. Both diff --git a/cmd/ate-setup/internal/steps/csi_test.go b/cmd/ate-setup/internal/steps/csi_test.go index aded1b9905..e558bfb9a2 100644 --- a/cmd/ate-setup/internal/steps/csi_test.go +++ b/cmd/ate-setup/internal/steps/csi_test.go @@ -23,8 +23,8 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" ) // setupCSIWithoutACluster runs SetupCSI against an Env holding no cluster diff --git a/cmd/ate-setup/internal/steps/delete.go b/cmd/ate-setup/internal/steps/delete.go index 6ba85d1516..b8cb7a0adf 100644 --- a/cmd/ate-setup/internal/steps/delete.go +++ b/cmd/ate-setup/internal/steps/delete.go @@ -26,9 +26,9 @@ import ( "k8s.io/client-go/dynamic" "k8s.io/client-go/util/retry" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" - "github.com/agent-substrate/substrate/internal/clustertrustbundle" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/internal/clustertrustbundle" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" ) // DeleteAteSystem removes the control plane. diff --git a/cmd/ate-setup/internal/steps/delete_test.go b/cmd/ate-setup/internal/steps/delete_test.go index 0ff7eb6ee8..a2f97f47de 100644 --- a/cmd/ate-setup/internal/steps/delete_test.go +++ b/cmd/ate-setup/internal/steps/delete_test.go @@ -36,10 +36,10 @@ import ( ktesting "k8s.io/client-go/testing" "k8s.io/client-go/util/retry" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" - "github.com/agent-substrate/substrate/internal/clustertrustbundle" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/internal/clustertrustbundle" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" ) // fakeSandboxConfigs is an in-memory dynamic.ResourceInterface implementing diff --git a/cmd/ate-setup/internal/steps/deploy.go b/cmd/ate-setup/internal/steps/deploy.go index dd015eb2bb..02ab4737a4 100644 --- a/cmd/ate-setup/internal/steps/deploy.go +++ b/cmd/ate-setup/internal/steps/deploy.go @@ -27,8 +27,8 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" ) // crdGVK is the kind of a CustomResourceDefinition. diff --git a/cmd/ate-setup/internal/steps/deploy_test.go b/cmd/ate-setup/internal/steps/deploy_test.go index 4edffc0012..3777fceb9b 100644 --- a/cmd/ate-setup/internal/steps/deploy_test.go +++ b/cmd/ate-setup/internal/steps/deploy_test.go @@ -25,8 +25,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" ) // podcertObjects loads the podcertificate-controller manifest as diff --git a/cmd/ate-setup/internal/steps/egress_dns_test.go b/cmd/ate-setup/internal/steps/egress_dns_test.go index 996d9ca424..26bc77137c 100644 --- a/cmd/ate-setup/internal/steps/egress_dns_test.go +++ b/cmd/ate-setup/internal/steps/egress_dns_test.go @@ -29,8 +29,8 @@ import ( k8syaml "k8s.io/apimachinery/pkg/util/yaml" "sigs.k8s.io/yaml" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kustomize" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/kustomize" ) // TestEgressDNSLookupFamily requires ALL on every dynamic forward proxy DNS diff --git a/cmd/ate-setup/internal/steps/env.go b/cmd/ate-setup/internal/steps/env.go index dc51749a5c..63b81920ae 100644 --- a/cmd/ate-setup/internal/steps/env.go +++ b/cmd/ate-setup/internal/steps/env.go @@ -26,12 +26,12 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/images" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/ko" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kustomize" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/images" + "go.ate.dev/substrate/cmd/ate-setup/internal/ko" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/cmd/ate-setup/internal/kustomize" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" ) // Timeouts carried over from the --timeout values in the shell installer. @@ -80,6 +80,11 @@ type Env struct { // work on a machine with no container registry credentials configured, and // the build-from-source resolver fails at construction without them. resolver imageResolver + + // dockerfileImages caches resolved Dockerfile image references by image + // name so a multi-manifest deploy (e.g., router + egress) builds or pins + // each Dockerfile image once. + dockerfileImages map[string]string } // Namespace is the namespace the control plane is installed into. It is @@ -163,22 +168,35 @@ func (e *Env) koRunner() (*ko.Runner, error) { // That only works if the release published it there, which // `ate-setup publish release-images` does. func (e *Env) dockerfileImage(ctx context.Context, name, contextDir string) (string, error) { + if ref, ok := e.dockerfileImages[name]; ok { + return ref, nil + } + var ref string + var err error if !e.Cfg.Images.IsPrebuilt() { - return images.BuildDockerfileImage(ctx, e.Cfg.Root, e.Cfg.KODockerRepo, name, e.Cfg.Path(contextDir), e.Cfg.KODefaultPlatforms, e.Cfg.DockerBuildFlags) + ref, err = images.BuildDockerfileImage(ctx, e.Cfg.Root, e.Cfg.KODockerRepo, name, e.Cfg.Path(contextDir), e.Cfg.KODefaultPlatforms, e.Cfg.DockerBuildFlags) + } else { + resolver, rErr := e.imageResolver() + if rErr != nil { + return "", rErr + } + prebuilt, ok := resolver.(*images.Prebuilt) + if !ok { + return "", fmt.Errorf("image resolver is %T, not a pre-built resolver", resolver) + } + ref, err = prebuilt.Pin(ctx, name) + if err != nil { + return "", fmt.Errorf("%s is built from a Dockerfile, not with ko, so a release has to publish it "+ + "alongside the other images (ate-setup publish release-images): %w", name, err) + } } - resolver, err := e.imageResolver() if err != nil { return "", err } - prebuilt, ok := resolver.(*images.Prebuilt) - if !ok { - return "", fmt.Errorf("image resolver is %T, not a pre-built resolver", resolver) - } - ref, err := prebuilt.Pin(ctx, name) - if err != nil { - return "", fmt.Errorf("%s is built from a Dockerfile, not with ko, so a release has to publish it "+ - "alongside the other images (ate-setup publish release-images): %w", name, err) + if e.dockerfileImages == nil { + e.dockerfileImages = make(map[string]string) } + e.dockerfileImages[name] = ref return ref, nil } diff --git a/cmd/ate-setup/internal/steps/namespace_test.go b/cmd/ate-setup/internal/steps/namespace_test.go index a769c6dcdc..42a46146cc 100644 --- a/cmd/ate-setup/internal/steps/namespace_test.go +++ b/cmd/ate-setup/internal/steps/namespace_test.go @@ -18,7 +18,7 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" ) func TestEnvNamespace(t *testing.T) { diff --git a/cmd/ate-setup/internal/steps/overlay.go b/cmd/ate-setup/internal/steps/overlay.go index 4d38363c01..46edf55c70 100644 --- a/cmd/ate-setup/internal/steps/overlay.go +++ b/cmd/ate-setup/internal/steps/overlay.go @@ -24,9 +24,9 @@ import ( "strings" "time" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kustomize" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/kustomize" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" ) const ( @@ -147,6 +147,13 @@ func (e *Env) renderSystemManifests(ctx context.Context) ([]byte, error) { if err != nil { return nil, err } + if e.Cfg.Router != config.RouterAgentgateway { + imageReference, err := e.dockerfileImage(ctx, envoyDataplaneImage, envoyDataplaneDockefile) + if err != nil { + return nil, err + } + manifest = e.patchEnvoyDataplaneImage(manifest, imageReference) + } return e.ResolveManifestBytes(ctx, manifest) } @@ -156,7 +163,16 @@ func (e *Env) renderAtenetRouterManifest(ctx context.Context) ([]byte, error) { if e.Cfg.Router == config.RouterAgentgateway { return e.renderResolve(ctx, e.Cfg.Path(installDir+"/agentgateway-router")) } - return e.renderResolve(ctx, e.Cfg.Manifest("atenet-router.yaml")) + manifest, err := e.render(e.Cfg.Manifest("atenet-router.yaml")) + if err != nil { + return nil, err + } + imageReference, err := e.dockerfileImage(ctx, envoyDataplaneImage, envoyDataplaneDockefile) + if err != nil { + return nil, err + } + manifest = e.patchEnvoyDataplaneImage(manifest, imageReference) + return e.ResolveManifestBytes(ctx, manifest) } // atenetEgressManifestPath returns the envoy egress gateway manifest. diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index d3ec4ad261..98bb29d613 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -37,9 +37,9 @@ import ( kyaml "sigs.k8s.io/kustomize/kyaml/yaml" "sigs.k8s.io/yaml" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/images" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/images" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" ) // Splicing the provider selection into the real egress manifest replaces the @@ -1134,9 +1134,9 @@ func TestApplyOtelEndpointOverride(t *testing.T) { }) } -// A pre-built install renders the envoy egress manifest without building -// anything: envoy-dataplane is pinned from the release like every ko image, so -// neither docker nor KO_DOCKER_REPO is needed. +// A pre-built install renders the envoy router, egress, and system manifests +// without building anything: envoy-dataplane is pinned from the release like +// every ko image, so neither docker nor KO_DOCKER_REPO is needed. func TestRenderAtenetEgressManifestPrebuilt(t *testing.T) { const digest = "sha256:2222222222222222222222222222222222222222222222222222222222222222" src := images.Source{Repo: "example.com/substrate", Tag: "v1.2.3"} @@ -1149,18 +1149,38 @@ func TestRenderAtenetEgressManifestPrebuilt(t *testing.T) { }), } - out, err := e.renderAtenetEgressManifest(t.Context(), config.CredentialProvider{}) - if err != nil { - t.Fatalf("renderAtenetEgressManifest() error = %v", err) - } want := "example.com/substrate/envoy-dataplane:v1.2.3@" + digest - if !strings.Contains(string(out), "image: "+want) { - t.Errorf("rendered manifest does not install %s:\n%s", want, out) - } - for _, leftover := range []string{"${ENVOY_DATAPLANE_IMAGE}", "ko://"} { - if strings.Contains(string(out), leftover) { - t.Errorf("rendered manifest still contains %q", leftover) - } + for _, tc := range []struct { + name string + render func() ([]byte, error) + }{ + { + name: "egress", + render: func() ([]byte, error) { return e.renderAtenetEgressManifest(t.Context(), config.CredentialProvider{}) }, + }, + { + name: "router", + render: func() ([]byte, error) { return e.renderAtenetRouterManifest(t.Context()) }, + }, + { + name: "system", + render: func() ([]byte, error) { return e.renderSystemManifests(t.Context()) }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + out, err := tc.render() + if err != nil { + t.Fatalf("render() error = %v", err) + } + if !strings.Contains(string(out), "image: "+want) { + t.Errorf("rendered manifest does not install %s:\n%s", want, out) + } + for _, leftover := range []string{"${ENVOY_DATAPLANE_IMAGE}", "ko://"} { + if strings.Contains(string(out), leftover) { + t.Errorf("rendered manifest still contains %q", leftover) + } + } + }) } if !slices.Contains(looked, "example.com/substrate/envoy-dataplane:v1.2.3") { t.Errorf("registry lookups = %v, want one for envoy-dataplane", looked) diff --git a/cmd/ate-setup/internal/steps/postgres.go b/cmd/ate-setup/internal/steps/postgres.go index 9287a75db2..3e8be19b4a 100644 --- a/cmd/ate-setup/internal/steps/postgres.go +++ b/cmd/ate-setup/internal/steps/postgres.go @@ -26,10 +26,10 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "sigs.k8s.io/yaml" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" - "github.com/agent-substrate/substrate/pkg/postgressetup" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/pkg/postgressetup" ) // bundledPostgresHost is where ate-api-server reaches the bundled PostgreSQL. diff --git a/cmd/ate-setup/internal/steps/postgres_test.go b/cmd/ate-setup/internal/steps/postgres_test.go index 7b326f92fb..d0022be87b 100644 --- a/cmd/ate-setup/internal/steps/postgres_test.go +++ b/cmd/ate-setup/internal/steps/postgres_test.go @@ -27,9 +27,9 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" - "github.com/agent-substrate/substrate/pkg/postgressetup" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/pkg/postgressetup" ) // postgresObjects loads the bundled postgres manifest of the given diff --git a/cmd/ate-setup/internal/steps/prereqs.go b/cmd/ate-setup/internal/steps/prereqs.go index cf2dd0b4e1..1a28b15dcd 100644 --- a/cmd/ate-setup/internal/steps/prereqs.go +++ b/cmd/ate-setup/internal/steps/prereqs.go @@ -18,7 +18,7 @@ import ( "context" "fmt" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" ) // trustBundleNames are the ClusterTrustBundles the podcertificate controller @@ -31,11 +31,14 @@ var trustBundleNames = []string{ } // EnsureAPIServerPrerequisites creates the secrets and config ate-api-server -// needs, skipping anything already present. +// needs, skipping anything already present. The PostgreSQL settings and an +// explicitly requested log level are rewritten every time. func (e *Env) EnsureAPIServerPrerequisites(ctx context.Context) error { log.Step("ensure_apiserver_prerequisites") - if err := e.ensureSecret(ctx, e.Namespace(), SecretActorIDJWTPool, e.CreateJWTAuthorityPoolSecret); err != nil { + // Not through ensureSecret: an existing pool Secret may still need its + // public keys published, which CreateJWTAuthorityPoolSecret handles. + if err := e.CreateJWTAuthorityPoolSecret(ctx); err != nil { return err } if err := e.ensureSecret(ctx, e.Namespace(), SecretActorIDCAPool, e.CreateActorIDCAPoolSecret); err != nil { @@ -51,6 +54,10 @@ func (e *Env) EnsureAPIServerPrerequisites(ctx context.Context) error { return err } + if err := e.ensureAPIConfig(ctx); err != nil { + return err + } + exists, err := e.Kube.ConfigMapExists(ctx, e.Namespace(), ConfigMapAPIAuthn) if err != nil { return err @@ -61,6 +68,23 @@ func (e *Env) EnsureAPIServerPrerequisites(ctx context.Context) error { return nil } +// ensureAPIConfig writes the ate-api-server config file when it is missing or +// a log level was asked for explicitly. Otherwise an existing config, possibly +// edited by hand, is kept. +func (e *Env) ensureAPIConfig(ctx context.Context) error { + if e.Cfg.APILogLevel != "" { + return e.CreateAPIConfig(ctx) + } + exists, err := e.Kube.ConfigMapExists(ctx, e.Namespace(), ConfigMapAPIConfig) + if err != nil { + return err + } + if !exists { + return e.CreateAPIConfig(ctx) + } + return nil +} + // EnsurePodCertificateCAs creates the podcertificate signer pools if any is // missing. func (e *Env) EnsurePodCertificateCAs(ctx context.Context) error { diff --git a/cmd/ate-setup/internal/steps/prereqs_test.go b/cmd/ate-setup/internal/steps/prereqs_test.go new file mode 100644 index 0000000000..8bde7919ad --- /dev/null +++ b/cmd/ate-setup/internal/steps/prereqs_test.go @@ -0,0 +1,86 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package steps + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/protobuf/encoding/prototext" + "google.golang.org/protobuf/testing/protocmp" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/pkg/proto/ateconfigpb" +) + +// A deploy without --ateapi-log-level must not undo a level set by hand or by +// an earlier deploy, but an explicit level always wins. +func TestEnsureAPIConfig(t *testing.T) { + const handEdited = `logging { level: "warn" }` + existing := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: NamespaceAteSystem, Name: ConfigMapAPIConfig}, + Data: map[string]string{"apiconfig.textproto": handEdited}, + } + + for _, tc := range []struct { + name string + existing runtime.Object + logLevel string + wantLevel string + }{ + {name: "new install defaults to info", wantLevel: "info"}, + {name: "new install takes the requested level", logLevel: "debug", wantLevel: "debug"}, + {name: "existing config is kept", existing: existing, wantLevel: "warn"}, + {name: "requested level overwrites existing config", existing: existing, logLevel: "error", wantLevel: "error"}, + } { + t.Run(tc.name, func(t *testing.T) { + // NewClientset, unlike the NewSimpleClientset behind fakeKube, + // supports the server-side apply CreateAPIConfig uses. + objects := []runtime.Object{} + if tc.existing != nil { + objects = append(objects, tc.existing) + } + e := &Env{ + Cfg: &config.Config{APILogLevel: tc.logLevel}, + Kube: &kube.Client{Typed: fake.NewClientset(objects...)}, + } + + if err := e.ensureAPIConfig(t.Context()); err != nil { + t.Fatalf("ensureAPIConfig() error = %v", err) + } + + cm, err := e.Kube.GetConfigMap(t.Context(), NamespaceAteSystem, ConfigMapAPIConfig) + if err != nil { + t.Fatalf("GetConfigMap() error = %v", err) + } + if cm == nil { + t.Fatalf("ConfigMap %s was not created", ConfigMapAPIConfig) + } + got := &ateconfigpb.APIConfig{} + if err := prototext.Unmarshal([]byte(cm.Data["apiconfig.textproto"]), got); err != nil { + t.Fatalf("apiconfig.textproto does not parse: %v", err) + } + want := &ateconfigpb.APIConfig{Logging: &ateconfigpb.APILogging{Level: tc.wantLevel}} + if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { + t.Errorf("apiconfig.textproto mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/cmd/ate-setup/internal/steps/preview.go b/cmd/ate-setup/internal/steps/preview.go index 0520ec8a5a..090391019a 100644 --- a/cmd/ate-setup/internal/steps/preview.go +++ b/cmd/ate-setup/internal/steps/preview.go @@ -25,7 +25,7 @@ import ( // substrateImagePrefix marks a container as running a substrate binary. Every // such binary accepts --preview. -const substrateImagePrefix = "ko://github.com/agent-substrate/substrate/cmd/" +const substrateImagePrefix = "ko://go.ate.dev/substrate/cmd/" const previewFlag = "--preview" diff --git a/cmd/ate-setup/internal/steps/preview_test.go b/cmd/ate-setup/internal/steps/preview_test.go index 18de8c6ef8..102836e4df 100644 --- a/cmd/ate-setup/internal/steps/preview_test.go +++ b/cmd/ate-setup/internal/steps/preview_test.go @@ -28,8 +28,8 @@ import ( "sigs.k8s.io/kustomize/kyaml/kio" "sigs.k8s.io/kustomize/kyaml/yaml" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" ) // containerArgs maps "//" to the container's args. @@ -78,7 +78,7 @@ kind: ConfigMap metadata: name: cfg data: - image: ko://github.com/agent-substrate/substrate/cmd/ateapi + image: ko://go.ate.dev/substrate/cmd/ateapi --- apiVersion: apps/v1 kind: Deployment @@ -89,18 +89,18 @@ spec: spec: initContainers: - name: sidecar - image: ko://github.com/agent-substrate/substrate/cmd/atenet + image: ko://go.ate.dev/substrate/cmd/atenet restartPolicy: Always args: - "sdsmint" containers: - name: api - image: ko://github.com/agent-substrate/substrate/cmd/ateapi + image: ko://go.ate.dev/substrate/cmd/ateapi args: - --port=8080 # keep me - --preview=Stale - name: bare - image: ko://github.com/agent-substrate/substrate/cmd/atelet + image: ko://go.ate.dev/substrate/cmd/atelet - name: envoy image: ${ENVOY_DATAPLANE_IMAGE} args: @@ -113,7 +113,7 @@ metadata: spec: containers: - name: ctl - image: ko://github.com/agent-substrate/substrate/cmd/atecontroller + image: ko://go.ate.dev/substrate/cmd/atecontroller ` func TestInjectPreviewArg(t *testing.T) { @@ -147,7 +147,7 @@ func TestInjectPreviewArg(t *testing.T) { } // Everything else survives: comments, placeholders, and the // ConfigMap that merely mentions a substrate image. - for _, s := range []string{"# keep me", "${ENVOY_DATAPLANE_IMAGE}", "kind: ConfigMap", "image: ko://github.com/agent-substrate/substrate/cmd/ateapi\n---"} { + for _, s := range []string{"# keep me", "${ENVOY_DATAPLANE_IMAGE}", "kind: ConfigMap", "image: ko://go.ate.dev/substrate/cmd/ateapi\n---"} { if !strings.Contains(string(out), s) { t.Errorf("output lost %q:\n%s", s, out) } diff --git a/cmd/ate-setup/internal/steps/publish.go b/cmd/ate-setup/internal/steps/publish.go index f3004188c9..52c5d264bc 100644 --- a/cmd/ate-setup/internal/steps/publish.go +++ b/cmd/ate-setup/internal/steps/publish.go @@ -19,8 +19,8 @@ import ( "fmt" "io" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/images" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/cmd/ate-setup/internal/images" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" ) // workerImages are the worker images a WorkerPool points at through diff --git a/cmd/ate-setup/internal/steps/publish_test.go b/cmd/ate-setup/internal/steps/publish_test.go index 99f4add104..f47a47549a 100644 --- a/cmd/ate-setup/internal/steps/publish_test.go +++ b/cmd/ate-setup/internal/steps/publish_test.go @@ -19,8 +19,8 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/images" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/images" ) // TestKoRunnerPrebuilt covers the one resolver that cannot build. A pre-built diff --git a/cmd/ate-setup/internal/steps/substrate.go b/cmd/ate-setup/internal/steps/substrate.go index a9d5999f50..94b70b04aa 100644 --- a/cmd/ate-setup/internal/steps/substrate.go +++ b/cmd/ate-setup/internal/steps/substrate.go @@ -25,11 +25,11 @@ import ( "google.golang.org/protobuf/encoding/protojson" "sigs.k8s.io/yaml" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" - "github.com/agent-substrate/substrate/internal/ateclient" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/internal/ateclient" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // AteClient connects to the ate-api-server, port-forwarding if needed. It diff --git a/cmd/ate-setup/internal/steps/substrate_test.go b/cmd/ate-setup/internal/steps/substrate_test.go index c61ba66fe8..13bc86c561 100644 --- a/cmd/ate-setup/internal/steps/substrate_test.go +++ b/cmd/ate-setup/internal/steps/substrate_test.go @@ -24,10 +24,10 @@ import ( "google.golang.org/grpc" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/internal/ateclient" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/internal/ateclient" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // TestActorTemplateFromManifestDemos parses the protojson ActorTemplate @@ -43,6 +43,7 @@ func TestActorTemplateFromManifestDemos(t *testing.T) { // The values benchmarking/workloads/deploy.sh substitutes, beyond the // ${BUCKET_NAME} every installer provides. benchmarkValues := map[string]string{ + "${ACTOR_CPU}": "1", "${ACTOR_MEMORY}": "256Mi", "${OTLP_ENDPOINT}": "http://otel-collector.otel-system:4317", "${SANDBOX_CLASS}": "gvisor", diff --git a/cmd/ate-setup/internal/steps/systemnetworkpolicy_test.go b/cmd/ate-setup/internal/steps/systemnetworkpolicy_test.go index 26feb30709..31a2ba629b 100644 --- a/cmd/ate-setup/internal/steps/systemnetworkpolicy_test.go +++ b/cmd/ate-setup/internal/steps/systemnetworkpolicy_test.go @@ -27,9 +27,9 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kustomize" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/cmd/ate-setup/internal/kustomize" ) // systemPod is one ate-system workload's pod template, as the NetworkPolicies diff --git a/cmd/ate-setup/internal/steps/version.go b/cmd/ate-setup/internal/steps/version.go index 5fb94f21d9..3fb02ae390 100644 --- a/cmd/ate-setup/internal/steps/version.go +++ b/cmd/ate-setup/internal/steps/version.go @@ -18,7 +18,7 @@ import ( "os" "strings" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/ko" + "go.ate.dev/substrate/cmd/ate-setup/internal/ko" ) // SubstrateVersion returns the build version, the same one ko stamps into the diff --git a/cmd/ate-setup/internal/steps/version_test.go b/cmd/ate-setup/internal/steps/version_test.go index c1674e7eab..055b75cecb 100644 --- a/cmd/ate-setup/internal/steps/version_test.go +++ b/cmd/ate-setup/internal/steps/version_test.go @@ -17,8 +17,8 @@ package steps import ( "testing" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/images" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/images" ) // TestSubstrateVersionPrebuilt covers the version a prebuilt install reports. diff --git a/cmd/ate-setup/main.go b/cmd/ate-setup/main.go index 134d837aeb..d115f75c5c 100644 --- a/cmd/ate-setup/main.go +++ b/cmd/ate-setup/main.go @@ -20,7 +20,7 @@ package main import ( - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/cmd" + "go.ate.dev/substrate/cmd/ate-setup/internal/cmd" ) func main() { diff --git a/cmd/ate-setup/preflight_test.go b/cmd/ate-setup/preflight_test.go index a81d142761..f54785ee71 100644 --- a/cmd/ate-setup/preflight_test.go +++ b/cmd/ate-setup/preflight_test.go @@ -28,7 +28,7 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/cmd" + "go.ate.dev/substrate/cmd/ate-setup/internal/cmd" ) // Each invocation gets its own command tree and flags, including invocations diff --git a/cmd/ate-setup/shim_test.go b/cmd/ate-setup/shim_test.go index 159a3e210a..d3430bbe54 100644 --- a/cmd/ate-setup/shim_test.go +++ b/cmd/ate-setup/shim_test.go @@ -30,11 +30,11 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/cmd" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos" + "go.ate.dev/substrate/cmd/ate-setup/internal/cmd" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/demos" // Registers every bundled demo, so demos.All() is the real list. - _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/all" + _ "go.ate.dev/substrate/cmd/ate-setup/internal/demos/all" ) // stubGo is a `go` that records the arguments of every `go run ./cmd/ate-setup` @@ -225,6 +225,10 @@ func TestShimTranslatesFlags(t *testing.T) { name: "--cluster-size takes an attached value", args: []string{"--cluster-size=size10", "--deploy-ate-apiserver"}, want: []string{"--cluster-size=size10 deploy apiserver"}, + }, { + name: "--ateapi-log-level is forwarded in either value form", + args: []string{"--ateapi-log-level", "debug", "--deploy-ate-apiserver", "--ateapi-log-level=warn"}, + want: []string{"--ateapi-log-level=debug --ateapi-log-level=warn deploy apiserver"}, }, { name: "--credential-provider is forwarded with a separate value", args: []string{"--deploy-atenet", "--credential-provider", `{"name":"custom","address":"cred.ate-system.svc:50051"}`}, @@ -251,6 +255,7 @@ func TestShimTranslatesFlags(t *testing.T) { "--create-podcertificate-controller-cas", "--create-api-server-env-vars", "--create-api-authentication-config", + "--create-api-config", }, want: []string{ "create jwt-authority-pool", @@ -260,6 +265,7 @@ func TestShimTranslatesFlags(t *testing.T) { "create podcertificate-controller-cas", "create api-server-env-vars", "create api-authentication-config", + "create api-config", }, }, { name: "delete flags", diff --git a/cmd/ateapi/helm_test.go b/cmd/ateapi/helm_test.go index 6232627409..106750c52f 100644 --- a/cmd/ateapi/helm_test.go +++ b/cmd/ateapi/helm_test.go @@ -16,7 +16,7 @@ package main import ( "fmt" - "github.com/agent-substrate/substrate/pkg/postgressetup" + "go.ate.dev/substrate/pkg/postgressetup" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" "os" diff --git a/cmd/ateapi/internal/apiauthn/apiauthn.go b/cmd/ateapi/internal/apiauthn/apiauthn.go index 56f37f68da..e26a0c9a92 100644 --- a/cmd/ateapi/internal/apiauthn/apiauthn.go +++ b/cmd/ateapi/internal/apiauthn/apiauthn.go @@ -27,7 +27,7 @@ import ( "log/slog" "strings" - "github.com/agent-substrate/substrate/internal/principal" + "go.ate.dev/substrate/internal/principal" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" diff --git a/cmd/ateapi/internal/apiauthn/apiauthn_test.go b/cmd/ateapi/internal/apiauthn/apiauthn_test.go index fed6cdf14f..6708c620cf 100644 --- a/cmd/ateapi/internal/apiauthn/apiauthn_test.go +++ b/cmd/ateapi/internal/apiauthn/apiauthn_test.go @@ -22,7 +22,7 @@ import ( "net/url" "testing" - "github.com/agent-substrate/substrate/internal/principal" + "go.ate.dev/substrate/internal/principal" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" diff --git a/cmd/ateapi/internal/apiconfig/loader.go b/cmd/ateapi/internal/apiconfig/loader.go new file mode 100644 index 0000000000..904a033a7d --- /dev/null +++ b/cmd/ateapi/internal/apiconfig/loader.go @@ -0,0 +1,56 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apiconfig + +import ( + "fmt" + "os" + "path/filepath" + + "go.ate.dev/substrate/pkg/proto/ateconfigpb" + "google.golang.org/protobuf/encoding/prototext" + "google.golang.org/protobuf/proto" +) + +func LoadConfig(configFile string) (*ateconfigpb.APIConfig, error) { + configBytes, err := os.ReadFile(configFile) + if err != nil { + return nil, fmt.Errorf("while reading config file: %w", err) + } + + config := &ateconfigpb.APIConfig{} + switch ext := filepath.Ext(configFile); ext { + case ".pb": + if err := proto.Unmarshal(configBytes, config); err != nil { + return nil, fmt.Errorf("while unmarshaling protobuf: %w", err) + } + case ".textproto", ".textpb": + if err := prototext.Unmarshal(configBytes, config); err != nil { + return nil, fmt.Errorf("while unmarshaling prototext: %w", err) + } + default: + return nil, fmt.Errorf("unrecognized config extension %q", ext) + } + + // Validate log level + switch config.GetLogging().GetLevel() { + case "debug", "info", "warn", "error": + // Do nothing + default: + return nil, fmt.Errorf("logging.level: invalid value %q", config.GetLogging().GetLevel()) + } + + return config, nil +} diff --git a/cmd/ateapi/internal/apiconfig/loader_test.go b/cmd/ateapi/internal/apiconfig/loader_test.go new file mode 100644 index 0000000000..c55b9b951b --- /dev/null +++ b/cmd/ateapi/internal/apiconfig/loader_test.go @@ -0,0 +1,50 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apiconfig + +import ( + "os" + "path/filepath" + "testing" +) + +func TestLoadConfigValidatesLogLevel(t *testing.T) { + testCases := []struct { + name string + content string + wantErr bool + }{ + {name: "debug", content: `logging { level: "debug" }`}, + {name: "info", content: `logging { level: "info" }`}, + {name: "warn", content: `logging { level: "warn" }`}, + {name: "error", content: `logging { level: "error" }`}, + {name: "invalid level", content: `logging { level: "verbose" }`, wantErr: true}, + {name: "missing level", content: ``, wantErr: true}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + configFile := filepath.Join(t.TempDir(), "config.textproto") + if err := os.WriteFile(configFile, []byte(tc.content), 0o600); err != nil { + t.Fatalf("Failed to write config file: %v", err) + } + + _, err := LoadConfig(configFile) + if gotErr := err != nil; gotErr != tc.wantErr { + t.Errorf("LoadConfig() error = %v, wantErr %v", err, tc.wantErr) + } + }) + } +} diff --git a/cmd/ateapi/internal/apivalidation/access_policy.go b/cmd/ateapi/internal/apivalidation/access_policy.go index 726a0e1d8a..430e06f961 100644 --- a/cmd/ateapi/internal/apivalidation/access_policy.go +++ b/cmd/ateapi/internal/apivalidation/access_policy.go @@ -18,8 +18,8 @@ import ( "context" "slices" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/authz" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/authz" + "go.ate.dev/substrate/pkg/proto/ateapipb" "k8s.io/apimachinery/pkg/api/operation" "k8s.io/apimachinery/pkg/util/validation/field" ) diff --git a/cmd/ateapi/internal/apivalidation/access_policy_test.go b/cmd/ateapi/internal/apivalidation/access_policy_test.go index b88d0add8c..0190a2aa3f 100644 --- a/cmd/ateapi/internal/apivalidation/access_policy_test.go +++ b/cmd/ateapi/internal/apivalidation/access_policy_test.go @@ -19,9 +19,9 @@ import ( "fmt" "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/authz" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/authz" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "k8s.io/apimachinery/pkg/util/validation/field" ) diff --git a/cmd/ateapi/internal/apivalidation/actor.go b/cmd/ateapi/internal/apivalidation/actor.go index a77942fcd3..92c9f0c7b4 100644 --- a/cmd/ateapi/internal/apivalidation/actor.go +++ b/cmd/ateapi/internal/apivalidation/actor.go @@ -17,8 +17,8 @@ package apivalidation import ( "context" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "k8s.io/apimachinery/pkg/api/operation" "k8s.io/apimachinery/pkg/api/validate" "k8s.io/apimachinery/pkg/util/validation/field" @@ -138,6 +138,11 @@ func ValidateCustom_WorkerAssignment_WorkerPodIps(_ context.Context, _ operation return validateDualStackIPs(fldPath, value) } +// This is needed because DV doesn't have a format for SPIFFE IDs. +func ValidateCustom_WorkerAssignment_WorkerSpiffeId(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { + return validateSPIFFEID(fldPath, *value) +} + func ValidateCustom_ExternalVolume_VolumeType(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { return resources.ValidateVolumeType(fldPath, *value) } diff --git a/cmd/ateapi/internal/apivalidation/actor_template.go b/cmd/ateapi/internal/apivalidation/actor_template.go index f906c2e8b5..7a8cced518 100644 --- a/cmd/ateapi/internal/apivalidation/actor_template.go +++ b/cmd/ateapi/internal/apivalidation/actor_template.go @@ -18,9 +18,9 @@ import ( "context" "fmt" - "github.com/agent-substrate/substrate/internal/resources" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/resources" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/pkg/proto/ateapipb" "k8s.io/apimachinery/pkg/api/operation" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/sets" diff --git a/cmd/ateapi/internal/apivalidation/actor_template_test.go b/cmd/ateapi/internal/apivalidation/actor_template_test.go index ebbba44356..a0e1c883b6 100644 --- a/cmd/ateapi/internal/apivalidation/actor_template_test.go +++ b/cmd/ateapi/internal/apivalidation/actor_template_test.go @@ -20,9 +20,9 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "k8s.io/apimachinery/pkg/util/validation/field" ) diff --git a/cmd/ateapi/internal/apivalidation/actor_test.go b/cmd/ateapi/internal/apivalidation/actor_test.go index dea6ab5882..f8540e86b5 100644 --- a/cmd/ateapi/internal/apivalidation/actor_test.go +++ b/cmd/ateapi/internal/apivalidation/actor_test.go @@ -20,7 +20,7 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "k8s.io/apimachinery/pkg/util/validation/field" ) @@ -354,6 +354,21 @@ func TestValidateActorUpdate(t *testing.T) { want: field.ErrorList{ field.TooMany(field.NewPath("status", "worker_assignment", "worker_pod_ips"), 3, 2).WithOrigin("maxItems"), }, + }, { + name: "set valid actor.status.worker_assignment.worker_spiffe_id", + oldVal: validInput(withStatus()), + newVal: validOutput(withStatus(withWorkerAssignment(func(wa *ateapipb.WorkerAssignment) { + wa.WorkerSpiffeId = "spiffe://cluster.local/ns/ns/sa/ateworker" + }))), + }, { + name: "invalid actor.status.worker_assignment.worker_spiffe_id", + oldVal: validInput(withStatus()), + newVal: validOutput(withStatus(withWorkerAssignment(func(wa *ateapipb.WorkerAssignment) { + wa.WorkerSpiffeId = "ns/ns/sa/ateworker" + }))), + want: field.ErrorList{ + field.Invalid(field.NewPath("status", "worker_assignment", "worker_spiffe_id"), nil, ""), + }, }, { name: "clear actor.status.worker_assignment", oldVal: validInput(withStatus(withWorkerAssignment())), diff --git a/cmd/ateapi/internal/apivalidation/atespace.go b/cmd/ateapi/internal/apivalidation/atespace.go index 4bc4c3060b..1e4126e9e6 100644 --- a/cmd/ateapi/internal/apivalidation/atespace.go +++ b/cmd/ateapi/internal/apivalidation/atespace.go @@ -17,7 +17,7 @@ package apivalidation import ( "context" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "k8s.io/apimachinery/pkg/util/validation/field" ) diff --git a/cmd/ateapi/internal/apivalidation/atespace_test.go b/cmd/ateapi/internal/apivalidation/atespace_test.go index 0f06d560b6..2a338fe436 100644 --- a/cmd/ateapi/internal/apivalidation/atespace_test.go +++ b/cmd/ateapi/internal/apivalidation/atespace_test.go @@ -19,7 +19,7 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "k8s.io/apimachinery/pkg/util/validation/field" ) diff --git a/cmd/ateapi/internal/apivalidation/common.go b/cmd/ateapi/internal/apivalidation/common.go index 73c8dcd959..4e4cafcfc1 100644 --- a/cmd/ateapi/internal/apivalidation/common.go +++ b/cmd/ateapi/internal/apivalidation/common.go @@ -19,9 +19,10 @@ import ( "fmt" "net/netip" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "github.com/spiffe/go-spiffe/v2/spiffeid" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "k8s.io/apimachinery/pkg/api/operation" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" @@ -119,3 +120,12 @@ func validateDualStackIPs(fldPath *field.Path, ips []string) field.ErrorList { } return errs } + +// validateSPIFFEID checks that id is a SPIFFE ID. Presence and length are +// enforced by tags. +func validateSPIFFEID(fldPath *field.Path, id string) field.ErrorList { + if _, err := spiffeid.FromString(id); err != nil { + return field.ErrorList{field.Invalid(fldPath, id, err.Error())} + } + return nil +} diff --git a/cmd/ateapi/internal/apivalidation/common_test.go b/cmd/ateapi/internal/apivalidation/common_test.go index 150340911a..e829d811ba 100644 --- a/cmd/ateapi/internal/apivalidation/common_test.go +++ b/cmd/ateapi/internal/apivalidation/common_test.go @@ -21,7 +21,7 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/internal/preview" + "go.ate.dev/substrate/internal/preview" "k8s.io/apimachinery/pkg/util/validation/field" ) diff --git a/cmd/ateapi/internal/apivalidation/doc.go b/cmd/ateapi/internal/apivalidation/doc.go index 9cc9849150..393bfea63a 100644 --- a/cmd/ateapi/internal/apivalidation/doc.go +++ b/cmd/ateapi/internal/apivalidation/doc.go @@ -16,7 +16,7 @@ // work. // +k8s:validation-gen=TypesWithSuffix=Request -// +k8s:validation-gen-input=github.com/agent-substrate/substrate/pkg/proto/ateapipb +// +k8s:validation-gen-input=go.ate.dev/substrate/pkg/proto/ateapipb // +k8s:validation-gen-scheme-registry=nil // +k8s:validation-gen-deep-equal-func=ateDeepEqual diff --git a/cmd/ateapi/internal/apivalidation/egress_policy.go b/cmd/ateapi/internal/apivalidation/egress_policy.go index c890c45179..be444b47cb 100644 --- a/cmd/ateapi/internal/apivalidation/egress_policy.go +++ b/cmd/ateapi/internal/apivalidation/egress_policy.go @@ -20,8 +20,8 @@ import ( "net/url" "strings" - "github.com/agent-substrate/substrate/internal/egresspolicy" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/egresspolicy" + "go.ate.dev/substrate/pkg/proto/ateapipb" "k8s.io/apimachinery/pkg/api/operation" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" diff --git a/cmd/ateapi/internal/apivalidation/egress_policy_test.go b/cmd/ateapi/internal/apivalidation/egress_policy_test.go index 89829a165b..d6f0fa3701 100644 --- a/cmd/ateapi/internal/apivalidation/egress_policy_test.go +++ b/cmd/ateapi/internal/apivalidation/egress_policy_test.go @@ -20,9 +20,9 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/defaults" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/defaults" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" "k8s.io/apimachinery/pkg/util/validation/field" ) diff --git a/cmd/ateapi/internal/apivalidation/tag.go b/cmd/ateapi/internal/apivalidation/tag.go index add2300c6f..125e8cf54e 100644 --- a/cmd/ateapi/internal/apivalidation/tag.go +++ b/cmd/ateapi/internal/apivalidation/tag.go @@ -17,7 +17,7 @@ package apivalidation import ( "context" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "k8s.io/apimachinery/pkg/api/operation" "k8s.io/apimachinery/pkg/api/validate" "k8s.io/apimachinery/pkg/util/validation/field" diff --git a/cmd/ateapi/internal/apivalidation/tag_test.go b/cmd/ateapi/internal/apivalidation/tag_test.go index 742e48263d..9bc5892863 100644 --- a/cmd/ateapi/internal/apivalidation/tag_test.go +++ b/cmd/ateapi/internal/apivalidation/tag_test.go @@ -19,7 +19,7 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "k8s.io/apimachinery/pkg/util/validation/field" ) diff --git a/cmd/ateapi/internal/apivalidation/validation_test.go b/cmd/ateapi/internal/apivalidation/validation_test.go index 120335bb21..6e1451850e 100644 --- a/cmd/ateapi/internal/apivalidation/validation_test.go +++ b/cmd/ateapi/internal/apivalidation/validation_test.go @@ -21,7 +21,7 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" "k8s.io/apimachinery/pkg/api/operation" diff --git a/cmd/ateapi/internal/apivalidation/worker.go b/cmd/ateapi/internal/apivalidation/worker.go index c56056a361..73ec35e07b 100644 --- a/cmd/ateapi/internal/apivalidation/worker.go +++ b/cmd/ateapi/internal/apivalidation/worker.go @@ -18,8 +18,8 @@ import ( "context" "fmt" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "k8s.io/apimachinery/pkg/api/operation" "k8s.io/apimachinery/pkg/api/validate" "k8s.io/apimachinery/pkg/util/validation/field" @@ -96,7 +96,13 @@ func ValidateMarkWorkerUnavailableRequest(ctx context.Context, req *ateapipb.Mar func ValidateReportActorSnapshotStateRequest(ctx context.Context, req *ateapipb.ReportActorSnapshotStateRequest) field.ErrorList { op := operation.Operation{Type: operation.Create} - return Validate_ReportActorSnapshotStateRequest(ctx, op, nil, req, nil) + errs := Validate_ReportActorSnapshotStateRequest(ctx, op, nil, req, nil) + // A report with neither latest_local nor uploading is a detach, which + // names the snapshot it made durable. + if st := req.GetState(); st != nil && st.GetLatestLocal() == nil && st.GetUploading() == nil && st.GetLatestDurable() == nil { + errs = append(errs, field.Required(field.NewPath("state", "latest_durable"), "a report with neither latest_local nor uploading must set latest_durable")) + } + return errs } // validateWorkerUpdate validates a Worker against the previous stored value. @@ -128,6 +134,11 @@ func ValidateCustom_Worker_Ips(_ context.Context, _ operation.Operation, fldPath return validateDualStackIPs(fldPath, value) } +// This is needed because DV doesn't have a format for SPIFFE IDs. +func ValidateCustom_Worker_WorkerSpiffeId(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { + return validateSPIFFEID(fldPath, *value) +} + // epoch only moves forward: a lower value would make Actors placed during the // current run look older than it, and crash them. func ValidateCustom_Worker_Epoch(_ context.Context, op operation.Operation, fldPath *field.Path, value, oldValue *int64) field.ErrorList { diff --git a/cmd/ateapi/internal/apivalidation/worker_test.go b/cmd/ateapi/internal/apivalidation/worker_test.go index 30c246ebdc..f9f198cdb6 100644 --- a/cmd/ateapi/internal/apivalidation/worker_test.go +++ b/cmd/ateapi/internal/apivalidation/worker_test.go @@ -20,7 +20,7 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" "k8s.io/apimachinery/pkg/util/validation/field" @@ -142,6 +142,23 @@ func TestValidateCreateWorkerRequest(t *testing.T) { w.Ips = []string{"10.1.2.3", "fd00::1", "10.1.2.4"} })), want: field.ErrorList{field.TooMany(field.NewPath("worker", "ips"), 3, 2).WithOrigin("maxItems")}, + }, { + name: "valid worker_spiffe_id", + req: validReq(validWorker(apiWorkerName, func(w *ateapipb.Worker) { + w.WorkerSpiffeId = "spiffe://cluster.local/ns/ate-system/sa/ateworker" + })), + }, { + name: "invalid worker_spiffe_id", + req: validReq(validWorker(apiWorkerName, func(w *ateapipb.Worker) { + w.WorkerSpiffeId = "https://cluster.local/ns/ate-system/sa/ateworker" + })), + want: field.ErrorList{field.Invalid(field.NewPath("worker", "worker_spiffe_id"), nil, "")}, + }, { + name: "worker_spiffe_id too long", + req: validReq(validWorker(apiWorkerName, func(w *ateapipb.Worker) { + w.WorkerSpiffeId = "spiffe://cluster.local/" + strings.Repeat("a", 2048) + })), + want: field.ErrorList{field.TooLong(field.NewPath("worker", "worker_spiffe_id"), nil, 2048).WithOrigin("maxLength")}, }, { name: "sandbox_class too long", req: validReq(validWorker(apiWorkerName, func(w *ateapipb.Worker) { w.SandboxClass = strings.Repeat("x", 64) })), @@ -697,8 +714,14 @@ func TestValidateReportActorSnapshotStateRequest(t *testing.T) { r.State.UploadDueTime = nil }), }, { - name: "valid empty state", + name: "empty state", req: valid(func(r *ateapipb.ReportActorSnapshotStateRequest) { r.State = &ateapipb.ActorSnapshotState{} }), + want: field.ErrorList{field.Required(field.NewPath("state", "latest_durable"), "")}, + }, { + name: "valid upload in flight with no local snapshot", + req: valid(func(r *ateapipb.ReportActorSnapshotStateRequest) { + r.State = &ateapipb.ActorSnapshotState{Uploading: r.State.GetUploading()} + }), }, { name: "missing actor", req: valid(func(r *ateapipb.ReportActorSnapshotStateRequest) { r.Actor = nil }), diff --git a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go index d6e8ce53b7..dd490a1a36 100644 --- a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go +++ b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go @@ -22,7 +22,7 @@ package apivalidation import ( context "context" - ateapipb "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + ateapipb "go.ate.dev/substrate/pkg/proto/ateapipb" timestamppb "google.golang.org/protobuf/types/known/timestamppb" operation "k8s.io/apimachinery/pkg/api/operation" safe "k8s.io/apimachinery/pkg/api/safe" @@ -2454,7 +2454,7 @@ func Validate_CreateWorkerRequest( return errs } -var unionMembershipFor_github_com_agent_substrate_substrate_pkg_proto_ateapipb_CredentialHeader_ = validate.NewUnionMembership(validate.NewUnionMember("credential_uri"), validate.NewUnionMember("actor_jwt")) +var unionMembershipFor_go_ate_dev_substrate_pkg_proto_ateapipb_CredentialHeader_ = validate.NewUnionMembership(validate.NewUnionMember("credential_uri"), validate.NewUnionMember("actor_jwt")) // Validate_CredentialHeader validates an instance of CredentialHeader according // to declarative validation rules in the API schema. @@ -2462,7 +2462,7 @@ func Validate_CredentialHeader( ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *ateapipb.CredentialHeader) (errs field.ErrorList) { - if e := validate.Union(ctx, op, fldPath, obj, oldObj, unionMembershipFor_github_com_agent_substrate_substrate_pkg_proto_ateapipb_CredentialHeader_, + if e := validate.Union(ctx, op, fldPath, obj, oldObj, unionMembershipFor_go_ate_dev_substrate_pkg_proto_ateapipb_CredentialHeader_, func(obj *ateapipb.CredentialHeader) bool { if obj == nil { return false @@ -3553,7 +3553,7 @@ func Validate_EgressPolicy( return errs } -var unionMembershipFor_github_com_agent_substrate_substrate_pkg_proto_ateapipb_EgressRule_ = validate.NewUnionMembership(validate.NewUnionMember("http"), validate.NewUnionMember("https"), validate.NewUnionMember("tls_passthrough")) +var unionMembershipFor_go_ate_dev_substrate_pkg_proto_ateapipb_EgressRule_ = validate.NewUnionMembership(validate.NewUnionMember("http"), validate.NewUnionMember("https"), validate.NewUnionMember("tls_passthrough")) // Validate_EgressRule validates an instance of EgressRule according // to declarative validation rules in the API schema. @@ -3561,7 +3561,7 @@ func Validate_EgressRule( ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *ateapipb.EgressRule) (errs field.ErrorList) { - if e := validate.Union(ctx, op, fldPath, obj, oldObj, unionMembershipFor_github_com_agent_substrate_substrate_pkg_proto_ateapipb_EgressRule_, + if e := validate.Union(ctx, op, fldPath, obj, oldObj, unionMembershipFor_go_ate_dev_substrate_pkg_proto_ateapipb_EgressRule_, func(obj *ateapipb.EgressRule) bool { if obj == nil { return false @@ -6046,6 +6046,56 @@ func Validate_MintWorkerActorCertificateRequest( errs = append(errs, fn(fldPath.Child("certificate_signing_request"), obj.CertificateSigningRequest, oldVal, oldObj != nil)...) } + { // field ateapipb.MintWorkerActorCertificateRequest.Worker + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.ObjectRef, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + func() { // cohort = "atespace" + earlyReturn := false + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", + func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.ForbiddenValue).MarkBeta().MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", + func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.OptionalValue).MarkBeta().MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", + func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.OptionalValue).MarkBeta().MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + }() + // call the type's validation function + errs = append(errs, Validate_ObjectRef(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.MintWorkerActorCertificateRequest) *ateapipb.ObjectRef { + return oldObj.Worker + }) + errs = append(errs, fn(fldPath.Child("worker"), obj.Worker, oldVal, oldObj != nil)...) + } + return errs } @@ -6353,7 +6403,7 @@ func Validate_PauseActorRequest( return errs } -var unionMembershipFor_github_com_agent_substrate_substrate_pkg_proto_ateapipb_Ports_ = validate.NewUnionMembership(validate.NewUnionMember("all"), validate.NewUnionMember("numbers")) +var unionMembershipFor_go_ate_dev_substrate_pkg_proto_ateapipb_Ports_ = validate.NewUnionMembership(validate.NewUnionMember("all"), validate.NewUnionMember("numbers")) // Validate_Ports validates an instance of Ports according // to declarative validation rules in the API schema. @@ -6361,7 +6411,7 @@ func Validate_Ports( ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *ateapipb.Ports) (errs field.ErrorList) { - if e := validate.Union(ctx, op, fldPath, obj, oldObj, unionMembershipFor_github_com_agent_substrate_substrate_pkg_proto_ateapipb_Ports_, + if e := validate.Union(ctx, op, fldPath, obj, oldObj, unionMembershipFor_go_ate_dev_substrate_pkg_proto_ateapipb_Ports_, func(obj *ateapipb.Ports) bool { if obj == nil { return false @@ -7452,7 +7502,7 @@ func Validate_SecurityContext( return errs } -var unionMembershipFor_github_com_agent_substrate_substrate_pkg_proto_ateapipb_Selector_ = validate.NewUnionMembership(validate.NewUnionMember("match_labels")) +var unionMembershipFor_go_ate_dev_substrate_pkg_proto_ateapipb_Selector_ = validate.NewUnionMembership(validate.NewUnionMember("match_labels")) // Validate_Selector validates an instance of Selector according // to declarative validation rules in the API schema. @@ -7460,7 +7510,7 @@ func Validate_Selector( ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *ateapipb.Selector) (errs field.ErrorList) { - if e := validate.Union(ctx, op, fldPath, obj, oldObj, unionMembershipFor_github_com_agent_substrate_substrate_pkg_proto_ateapipb_Selector_, + if e := validate.Union(ctx, op, fldPath, obj, oldObj, unionMembershipFor_go_ate_dev_substrate_pkg_proto_ateapipb_Selector_, func(obj *ateapipb.Selector) bool { if obj == nil { return false @@ -8214,7 +8264,7 @@ func Validate_SuspendActorRequest( return errs } -var unionMembershipFor_github_com_agent_substrate_substrate_pkg_proto_ateapipb_SystemInfoDataSource_ = validate.NewUnionMembership(validate.NewUnionMember("actor_metadata"), validate.NewUnionMember("trust_bundle")) +var unionMembershipFor_go_ate_dev_substrate_pkg_proto_ateapipb_SystemInfoDataSource_ = validate.NewUnionMembership(validate.NewUnionMember("actor_metadata"), validate.NewUnionMember("trust_bundle")) // Validate_SystemInfoDataSource validates an instance of SystemInfoDataSource according // to declarative validation rules in the API schema. @@ -8222,7 +8272,7 @@ func Validate_SystemInfoDataSource( ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *ateapipb.SystemInfoDataSource) (errs field.ErrorList) { - if e := validate.Union(ctx, op, fldPath, obj, oldObj, unionMembershipFor_github_com_agent_substrate_substrate_pkg_proto_ateapipb_SystemInfoDataSource_, + if e := validate.Union(ctx, op, fldPath, obj, oldObj, unionMembershipFor_go_ate_dev_substrate_pkg_proto_ateapipb_SystemInfoDataSource_, func(obj *ateapipb.SystemInfoDataSource) bool { if obj == nil { return false @@ -9263,7 +9313,7 @@ func Validate_VersionedSandboxCompat( return errs } -var unionMembershipFor_github_com_agent_substrate_substrate_pkg_proto_ateapipb_Volume_ = validate.NewUnionMembership(validate.NewUnionMember("durable_dir"), validate.NewUnionMember("external_volume_template"), validate.NewUnionMember("system_info"), validate.NewUnionMember("image")) +var unionMembershipFor_go_ate_dev_substrate_pkg_proto_ateapipb_Volume_ = validate.NewUnionMembership(validate.NewUnionMember("durable_dir"), validate.NewUnionMember("external_volume_template"), validate.NewUnionMember("system_info"), validate.NewUnionMember("image")) // Validate_Volume validates an instance of Volume according // to declarative validation rules in the API schema. @@ -9271,7 +9321,7 @@ func Validate_Volume( ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *ateapipb.Volume) (errs field.ErrorList) { - if e := validate.Union(ctx, op, fldPath, obj, oldObj, unionMembershipFor_github_com_agent_substrate_substrate_pkg_proto_ateapipb_Volume_, + if e := validate.Union(ctx, op, fldPath, obj, oldObj, unionMembershipFor_go_ate_dev_substrate_pkg_proto_ateapipb_Volume_, func(obj *ateapipb.Volume) bool { if obj == nil { return false @@ -9892,6 +9942,45 @@ func Validate_Worker( errs = append(errs, fn(fldPath.Child("ips"), obj.Ips, oldVal, oldObj != nil)...) } + { // field ateapipb.Worker.WorkerSpiffeId + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.Immutable(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // custom validation + if e := ValidateCustom_Worker_WorkerSpiffeId(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 2048); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.Worker) *string { + return &oldObj.WorkerSpiffeId + }) + errs = append(errs, fn(fldPath.Child("worker_spiffe_id"), &obj.WorkerSpiffeId, oldVal, oldObj != nil)...) + } + { // field ateapipb.Worker.SandboxClass fn := func( fldPath *field.Path, @@ -10356,6 +10445,41 @@ func Validate_WorkerAssignment( errs = append(errs, fn(fldPath.Child("sandbox_runtime"), obj.SandboxRuntime, oldVal, oldObj != nil)...) } + { // field ateapipb.WorkerAssignment.WorkerSpiffeId + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // custom validation + if e := ValidateCustom_WorkerAssignment_WorkerSpiffeId(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 2048); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.WorkerAssignment) *string { + return &oldObj.WorkerSpiffeId + }) + errs = append(errs, fn(fldPath.Child("worker_spiffe_id"), &obj.WorkerSpiffeId, oldVal, oldObj != nil)...) + } + return errs } diff --git a/cmd/ateapi/internal/ateletauth/ateletauth.go b/cmd/ateapi/internal/ateletauth/ateletauth.go index 445cb44dd8..39f009c5a6 100644 --- a/cmd/ateapi/internal/ateletauth/ateletauth.go +++ b/cmd/ateapi/internal/ateletauth/ateletauth.go @@ -20,8 +20,8 @@ import ( "context" "log/slog" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/substratex509" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/substratex509" "google.golang.org/grpc/credentials" "google.golang.org/grpc/peer" ) diff --git a/cmd/ateapi/internal/ateletauth/ateletauth_test.go b/cmd/ateapi/internal/ateletauth/ateletauth_test.go index 93a5485511..96b7bfae3f 100644 --- a/cmd/ateapi/internal/ateletauth/ateletauth_test.go +++ b/cmd/ateapi/internal/ateletauth/ateletauth_test.go @@ -17,9 +17,9 @@ package ateletauth_test import ( "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/ateletauth" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/ateletauth/ateletauthtest" - "github.com/agent-substrate/substrate/internal/installdefaults" + "go.ate.dev/substrate/cmd/ateapi/internal/ateletauth" + "go.ate.dev/substrate/cmd/ateapi/internal/ateletauth/ateletauthtest" + "go.ate.dev/substrate/internal/installdefaults" ) // TestAuthenticateHonorsConfiguredIdentity checks that the identity diff --git a/cmd/ateapi/internal/ateletauth/ateletauthtest/ateletauthtest.go b/cmd/ateapi/internal/ateletauth/ateletauthtest/ateletauthtest.go index d4bc203ca2..ec4cd6c313 100644 --- a/cmd/ateapi/internal/ateletauth/ateletauthtest/ateletauthtest.go +++ b/cmd/ateapi/internal/ateletauth/ateletauthtest/ateletauthtest.go @@ -29,8 +29,8 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/internal/substratex509" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/substratex509" "google.golang.org/grpc/credentials" "google.golang.org/grpc/peer" ) diff --git a/cmd/ateapi/internal/authz/authorizer.go b/cmd/ateapi/internal/authz/authorizer.go index 57d8a78056..68983aeef4 100644 --- a/cmd/ateapi/internal/authz/authorizer.go +++ b/cmd/ateapi/internal/authz/authorizer.go @@ -18,10 +18,10 @@ import ( "context" "strings" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/principal" openfgav1 "github.com/openfga/api/proto/openfga/v1" "github.com/openfga/openfga/pkg/server" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/principal" ) // Authorizer is the read-only policy decision point that evaluates runtime diff --git a/cmd/ateapi/internal/authz/authz_test.go b/cmd/ateapi/internal/authz/authz_test.go index 0a8a744049..c391ffb48d 100644 --- a/cmd/ateapi/internal/authz/authz_test.go +++ b/cmd/ateapi/internal/authz/authz_test.go @@ -33,8 +33,9 @@ import ( "github.com/testcontainers/testcontainers-go/modules/postgres" "google.golang.org/grpc/codes" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/principal" + "go.ate.dev/substrate/cmd/ateapi/internal/store/dockerenv" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/principal" ) func configureDockerEnv(ctx context.Context) error { @@ -63,8 +64,14 @@ func configureDockerEnv(ctx context.Context) error { func startPostgres(t *testing.T) *pgxpool.Pool { t.Helper() ctx := context.Background() + // Without Docker the tests skip locally, but must fail in CI, where a + // skip would pass the package without running it. + unavailable := t.Skipf + if dockerenv.Required() { + unavailable = t.Fatalf + } if err := configureDockerEnv(ctx); err != nil { - t.Skipf("skipping test; docker is unavailable: %v", err) + unavailable("docker is unavailable: %v", err) } pgContainer, err := postgres.Run(ctx, @@ -75,7 +82,7 @@ func startPostgres(t *testing.T) *pgxpool.Pool { postgres.BasicWaitStrategies(), ) if err != nil { - t.Skipf("skipping test; failed to start postgres container: %v", err) + unavailable("failed to start postgres container: %v", err) } t.Cleanup(func() { _ = testcontainers.TerminateContainer(pgContainer) diff --git a/cmd/ateapi/internal/authz/authz_util.go b/cmd/ateapi/internal/authz/authz_util.go index 608d660469..e4c86cc91e 100644 --- a/cmd/ateapi/internal/authz/authz_util.go +++ b/cmd/ateapi/internal/authz/authz_util.go @@ -47,6 +47,7 @@ const ( RoleController = "controller" RoleEgressGateway = "egress_gateway" RoleIngressRouter = "ingress_router" + RoleAtelet = "atelet" RelationCanCreateAtespace = "can_create_atespace" RelationCanListAtespaces = "can_list_atespaces" @@ -234,6 +235,7 @@ type SystemPrincipals struct { ControllerSPIFFEID string EgressGatewaySPIFFEID string IngressRouterSPIFFEID string + AteletSPIFFEID string } // New provisions the default OpenFGA store and authorization model via @@ -293,6 +295,7 @@ func parseSystemPrincipals(system SystemPrincipals) (map[string][]string, error) {role: RoleController, spiffeID: system.ControllerSPIFFEID}, {role: RoleEgressGateway, spiffeID: system.EgressGatewaySPIFFEID}, {role: RoleIngressRouter, spiffeID: system.IngressRouterSPIFFEID}, + {role: RoleAtelet, spiffeID: system.AteletSPIFFEID}, } out := make(map[string][]string, len(bindings)) for _, b := range bindings { diff --git a/cmd/ateapi/internal/authz/authz_util_test.go b/cmd/ateapi/internal/authz/authz_util_test.go index 40ad23d1f2..7007d23675 100644 --- a/cmd/ateapi/internal/authz/authz_util_test.go +++ b/cmd/ateapi/internal/authz/authz_util_test.go @@ -19,8 +19,8 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/internal/installdefaults" ) func testSystemPrincipals() SystemPrincipals { @@ -28,6 +28,7 @@ func testSystemPrincipals() SystemPrincipals { ControllerSPIFFEID: installdefaults.ControllerSPIFFEID(installdefaults.SystemNamespace), EgressGatewaySPIFFEID: installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace), IngressRouterSPIFFEID: installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), + AteletSPIFFEID: installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), } } @@ -96,6 +97,7 @@ func TestParseSystemPrincipals(t *testing.T) { ControllerSPIFFEID: "spiffe://cluster.local/ns/ate-system/sa/ate-controller", EgressGatewaySPIFFEID: router, IngressRouterSPIFFEID: router, + AteletSPIFFEID: "spiffe://cluster.local/ns/ate-system/sa/atelet", }) if err != nil { t.Fatalf("parseSystemPrincipals failed: %v", err) @@ -103,6 +105,7 @@ func TestParseSystemPrincipals(t *testing.T) { want := map[string][]string{ "user:spiffe%3A//cluster.local/ns/ate-system/sa/ate-controller": {RoleController}, "user:spiffe%3A//cluster.local/ns/ate-system/sa/atenet-router": {RoleEgressGateway, RoleIngressRouter}, + "user:spiffe%3A//cluster.local/ns/ate-system/sa/atelet": {RoleAtelet}, } if diff := cmp.Diff(want, got); diff != "" { t.Errorf("parseSystemPrincipals (-want +got):\n%s", diff) @@ -119,6 +122,7 @@ func TestNewRejectsInvalidSystemPrincipals(t *testing.T) { {"empty controller SPIFFE ID", func(p *SystemPrincipals) { p.ControllerSPIFFEID = "" }, "invalid controller SPIFFE ID"}, {"empty egress gateway SPIFFE ID", func(p *SystemPrincipals) { p.EgressGatewaySPIFFEID = "" }, "invalid egress_gateway SPIFFE ID"}, {"empty ingress router SPIFFE ID", func(p *SystemPrincipals) { p.IngressRouterSPIFFEID = "" }, "invalid ingress_router SPIFFE ID"}, + {"empty atelet SPIFFE ID", func(p *SystemPrincipals) { p.AteletSPIFFEID = "" }, "invalid atelet SPIFFE ID"}, {"bare ServiceAccount name", func(p *SystemPrincipals) { p.ControllerSPIFFEID = "ate-controller" }, "invalid controller SPIFFE ID"}, {"non-spiffe URI", func(p *SystemPrincipals) { p.EgressGatewaySPIFFEID = "https://cluster.local/ns/ate-system/sa/atenet-egress" diff --git a/cmd/ateapi/internal/authz/interceptor.go b/cmd/ateapi/internal/authz/interceptor.go index 9fd77198c8..ee20395535 100644 --- a/cmd/ateapi/internal/authz/interceptor.go +++ b/cmd/ateapi/internal/authz/interceptor.go @@ -17,8 +17,8 @@ package authz import ( "context" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/principal" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/principal" "google.golang.org/grpc" ) diff --git a/cmd/ateapi/internal/authz/interceptor_test.go b/cmd/ateapi/internal/authz/interceptor_test.go index f7bf2c4fb6..4bab436a23 100644 --- a/cmd/ateapi/internal/authz/interceptor_test.go +++ b/cmd/ateapi/internal/authz/interceptor_test.go @@ -19,9 +19,9 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/principal" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/principal" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "google.golang.org/grpc/codes" ) @@ -688,12 +688,14 @@ func TestUnaryServerInterceptor_SystemPrincipals(t *testing.T) { controllerID = "spiffe://cluster.local/ns/ate-system/sa/ate-controller" egressID = "spiffe://cluster.local/ns/ate-system/sa/atenet-egress" routerID = "spiffe://cluster.local/ns/ate-system/sa/atenet-router" + ateletID = "spiffe://cluster.local/ns/ate-system/sa/atelet" otherID = "spiffe://cluster.local/ns/ate-system/sa/other" ) authorizer, _, err := New(ctx, pool, fgaServer, nil, SystemPrincipals{ ControllerSPIFFEID: controllerID, EgressGatewaySPIFFEID: egressID, IngressRouterSPIFFEID: routerID, + AteletSPIFFEID: ateletID, }) if err != nil { t.Fatalf("New failed: %v", err) @@ -779,11 +781,22 @@ func TestUnaryServerInterceptor_SystemPrincipals(t *testing.T) { {"ingress router cannot mint actor JWT", mtls(routerID), mintJWT, codes.PermissionDenied}, {"controller cannot mint actor certificate", mtls(controllerID), mintCert, codes.PermissionDenied}, + // atelet reads egress policies for the worker that enforces them, and + // nothing else. + {"atelet gets egress policy", mtls(ateletID), getEgress, codes.OK}, + {"atelet cannot update egress policy", mtls(ateletID), updateEgress, codes.PermissionDenied}, + {"atelet cannot get actor", mtls(ateletID), getActor, codes.PermissionDenied}, + {"atelet cannot list actors", mtls(ateletID), listAllActors, codes.PermissionDenied}, + {"atelet cannot resume actor", mtls(ateletID), resume, codes.PermissionDenied}, + {"atelet cannot mint actor JWT", mtls(ateletID), mintJWT, codes.PermissionDenied}, + {"atelet cannot get worker", mtls(ateletID), getWorker, codes.PermissionDenied}, + // A bearer token whose subject is a granted SPIFFE ID gets nothing. {"jwt with controller ID cannot create worker", jwt(controllerID), createWorker, codes.PermissionDenied}, {"jwt with egress ID cannot get actor", jwt(egressID), getActor, codes.PermissionDenied}, {"jwt with egress ID cannot mint actor JWT", jwt(egressID), mintJWT, codes.PermissionDenied}, {"jwt with router ID cannot resume actor", jwt(routerID), resume, codes.PermissionDenied}, + {"jwt with atelet ID cannot get egress policy", jwt(ateletID), getEgress, codes.PermissionDenied}, // Other SPIFFE IDs get nothing. {"other mTLS identity cannot resume actor", mtls(otherID), resume, codes.PermissionDenied}, diff --git a/cmd/ateapi/internal/authz/model.fga b/cmd/ateapi/internal/authz/model.fga index ccd8242108..ea41388938 100644 --- a/cmd/ateapi/internal/authz/model.fga +++ b/cmd/ateapi/internal/authz/model.fga @@ -50,6 +50,7 @@ type global define controller: [user] define egress_gateway: [user] define ingress_router: [user] + define atelet: [user] define can_create_atespace: owner define can_list_atespaces: viewer @@ -81,6 +82,7 @@ type atespace # System component roles, inherited from global only. define egress_gateway: egress_gateway from parent_global define ingress_router: ingress_router from parent_global + define atelet: atelet from parent_global # --- 2. Policy Governance --- define can_create_access_policy: owner @@ -143,7 +145,15 @@ type actor define can_delete: editor from parent_atespace # Egress policy: one per actor, so its permissions live on the actor. - define can_get_egress_policy: viewer from parent_atespace or egress_gateway from parent_atespace + # atelet reads it on behalf of the worker hosting the actor, which + # enforces it. + # + # TODO(#1564): Every node's atelet shares one SPIFFE ID, so any atelet can read + # any actor's egress policy, and atelet does not check that the worker + # asking hosts the actor. Serve the read from WorkerService instead, + # authorizing atelet on the node hosting the actor as MintWorkerActorJwt + # does, have atelet check the requesting worker, and drop atelet here. + define can_get_egress_policy: viewer from parent_atespace or egress_gateway from parent_atespace or atelet from parent_atespace define can_create_egress_policy: editor from parent_atespace define can_update_egress_policy: editor from parent_atespace define can_delete_egress_policy: editor from parent_atespace diff --git a/cmd/ateapi/internal/authz/model_fga_test.go b/cmd/ateapi/internal/authz/model_fga_test.go new file mode 100644 index 0000000000..5524349846 --- /dev/null +++ b/cmd/ateapi/internal/authz/model_fga_test.go @@ -0,0 +1,134 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package authz + +import ( + "context" + "os" + "slices" + "testing" + + openfgav1 "github.com/openfga/api/proto/openfga/v1" + "sigs.k8s.io/yaml" +) + +// modelTestFile is in the format `fga model test` reads, so the same file +// also runs with the OpenFGA CLI. +const modelTestFile = "model_test.fga.yaml" + +// modelTests is the subset of the `fga model test` format the file uses. +// Parsing is strict, so a test using any other part of the format (contextual +// tuples, list_objects, conditions, ...) fails here rather than being +// silently skipped. +type modelTests struct { + Name string `json:"name"` + ModelFile string `json:"model_file"` + Tuples []modelTuple `json:"tuples"` + Tests []struct { + Name string `json:"name"` + Check []struct { + User string `json:"user"` + Object string `json:"object"` + Assertions map[string]bool `json:"assertions"` + } `json:"check"` + } `json:"tests"` +} + +type modelTuple struct { + User string `json:"user"` + Relation string `json:"relation"` + Object string `json:"object"` +} + +// TestModelFGATests runs the checks in model_test.fga.yaml against model.fga +// as ate-api-server loads it. +func TestModelFGATests(t *testing.T) { + data, err := os.ReadFile(modelTestFile) + if err != nil { + t.Fatalf("reading %s: %v", modelTestFile, err) + } + var mt modelTests + if err := yaml.UnmarshalStrict(data, &mt); err != nil { + t.Fatalf("parsing %s: %v", modelTestFile, err) + } + // The checks must run against the model ate-api-server embeds. + if mt.ModelFile != "./model.fga" { + t.Fatalf("%s: model_file = %q, want %q", modelTestFile, mt.ModelFile, "./model.fga") + } + if len(mt.Tests) == 0 { + t.Fatalf("%s has no tests", modelTestFile) + } + + ctx := context.Background() + pool := startPostgres(t) + fgaSrv, storeID, modelID := newTestAuthz(t, ctx, pool) + + // OpenFGA caps the tuples one Write may carry. + const maxTuplesPerWrite = 100 + for chunk := range slices.Chunk(mt.Tuples, maxTuplesPerWrite) { + keys := make([]*openfgav1.TupleKey, 0, len(chunk)) + for _, tk := range chunk { + keys = append(keys, &openfgav1.TupleKey{User: tk.User, Relation: tk.Relation, Object: tk.Object}) + } + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatalf("pool.Begin failed: %v", err) + } + if _, err := fgaSrv.Write(ContextWithTx(ctx, tx), &openfgav1.WriteRequest{ + StoreId: storeID, + AuthorizationModelId: modelID, + Writes: &openfgav1.WriteRequestWrites{TupleKeys: keys}, + }); err != nil { + _ = tx.Rollback(ctx) + t.Fatalf("writing tuples: %v", err) + } + if err := tx.Commit(ctx); err != nil { + t.Fatalf("tx.Commit failed: %v", err) + } + } + + for _, tc := range mt.Tests { + t.Run(tc.Name, func(t *testing.T) { + if len(tc.Check) == 0 { + t.Fatal("test has no checks") + } + for _, c := range tc.Check { + relations := make([]string, 0, len(c.Assertions)) + for r := range c.Assertions { + relations = append(relations, r) + } + slices.Sort(relations) + for _, relation := range relations { + resp, err := fgaSrv.Check(ctx, &openfgav1.CheckRequest{ + StoreId: storeID, + AuthorizationModelId: modelID, + TupleKey: &openfgav1.CheckRequestTupleKey{ + User: c.User, + Relation: relation, + Object: c.Object, + }, + }) + if err != nil { + t.Errorf("Check(user=%s, relation=%s, object=%s): %v", c.User, relation, c.Object, err) + continue + } + if want := c.Assertions[relation]; resp.GetAllowed() != want { + t.Errorf("Check(user=%s, relation=%s, object=%s) = %v, want %v", c.User, relation, c.Object, resp.GetAllowed(), want) + } + } + } + }) + } +} diff --git a/cmd/ateapi/internal/authz/model_test.fga.yaml b/cmd/ateapi/internal/authz/model_test.fga.yaml index 0464b336cd..e2f945c27a 100644 --- a/cmd/ateapi/internal/authz/model_test.fga.yaml +++ b/cmd/ateapi/internal/authz/model_test.fga.yaml @@ -87,6 +87,9 @@ tuples: - user: user:sys-router relation: ingress_router object: global:root + - user: user:sys-atelet + relation: atelet + object: global:root tests: - name: 1. Global Superadmin & Viewer Propagation @@ -420,6 +423,33 @@ tests: viewer: false can_list_actors: false can_get: false + - user: user:sys-atelet # atelet reads egress policies for the worker that enforces them + object: actor:runner-1 + assertions: + can_get_egress_policy: true + can_get: false + can_update: false + can_delete: false + can_create_egress_policy: false + can_update_egress_policy: false + can_delete_egress_policy: false + can_mint_actor_jwt: false + can_mint_actor_certificate: false + can_resume: false + can_suspend: false + - user: user:sys-atelet + object: atespace:space-1 + assertions: + viewer: false + can_get: false + can_list_actors: false + can_create_actor: false + - user: user:sys-atelet + object: global:root + assertions: + can_list_actors: false + can_list_workers: false + can_create_worker: false - user: user:sys-router # atenet-router resumes actors and lists them object: actor:runner-1 assertions: @@ -445,12 +475,14 @@ tests: assertions: egress_gateway: false ingress_router: false + atelet: false - user: user:carol # Global owner does not hold system roles object: global:root assertions: controller: false egress_gateway: false ingress_router: false + atelet: false - name: 8. Tag Operations (Inherited from Atespace) check: diff --git a/cmd/ateapi/internal/authz/policy_manager.go b/cmd/ateapi/internal/authz/policy_manager.go index f6fc0d2d9f..99a9d11f81 100644 --- a/cmd/ateapi/internal/authz/policy_manager.go +++ b/cmd/ateapi/internal/authz/policy_manager.go @@ -19,10 +19,10 @@ import ( "errors" "fmt" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/jackc/pgx/v5" openfgav1 "github.com/openfga/api/proto/openfga/v1" "github.com/openfga/openfga/pkg/server" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/types/known/wrapperspb" ) diff --git a/cmd/ateapi/internal/authz/registry.go b/cmd/ateapi/internal/authz/registry.go index 65231d9e9c..c22d81a810 100644 --- a/cmd/ateapi/internal/authz/registry.go +++ b/cmd/ateapi/internal/authz/registry.go @@ -17,9 +17,9 @@ package authz import ( "slices" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "k8s.io/apimachinery/pkg/util/validation/field" ) @@ -338,10 +338,9 @@ var defaultRPCPermissions = map[string]rpcRule{ // WorkerService is served only to atelet, which each handler // authenticates by its mTLS SPIFFE ID. - ateapipb.WorkerService_RegisterWorker_FullMethodName: inHandler("ateletauth: caller is atelet on the Worker's node"), - ateapipb.WorkerService_RequestActorSuspend_FullMethodName: inHandler("ateletauth: caller is atelet on the node of the Worker hosting the Actor"), - // TODO: also require that the Actor runs on the caller's node. - ateapipb.WorkerService_MintWorkerActorCertificate_FullMethodName: inHandler("ateletauth: caller is atelet"), + ateapipb.WorkerService_RegisterWorker_FullMethodName: inHandler("ateletauth: caller is atelet on the Worker's node"), + ateapipb.WorkerService_RequestActorSuspend_FullMethodName: inHandler("ateletauth: caller is atelet on the node of the Worker hosting the Actor"), + ateapipb.WorkerService_MintWorkerActorCertificate_FullMethodName: inHandler("ateletauth: caller is atelet on the node of the Worker the Actor is assigned to"), ateapipb.WorkerService_MintWorkerActorJwt_FullMethodName: inHandler("ateletauth: caller is atelet on the node hosting the Actor"), // TODO(#2570): not served yet; the handler must authenticate atelet on the // node of the Worker hosting the Actor when it lands. diff --git a/cmd/ateapi/internal/authz/registry_test.go b/cmd/ateapi/internal/authz/registry_test.go index ce3f8ceaf6..b042904319 100644 --- a/cmd/ateapi/internal/authz/registry_test.go +++ b/cmd/ateapi/internal/authz/registry_test.go @@ -19,8 +19,8 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" ) diff --git a/cmd/ateapi/internal/controlapi/access_policy.go b/cmd/ateapi/internal/controlapi/access_policy.go index 1cdd5c0653..f83c5a3940 100644 --- a/cmd/ateapi/internal/controlapi/access_policy.go +++ b/cmd/ateapi/internal/controlapi/access_policy.go @@ -19,12 +19,12 @@ import ( "errors" "fmt" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/apivalidation" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/defaults" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/apivalidation" + "go.ate.dev/substrate/cmd/ateapi/internal/defaults" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" "k8s.io/apimachinery/pkg/util/validation/field" ) diff --git a/cmd/ateapi/internal/controlapi/access_policy_errors_test.go b/cmd/ateapi/internal/controlapi/access_policy_errors_test.go index bf55298c63..fcc0f22603 100644 --- a/cmd/ateapi/internal/controlapi/access_policy_errors_test.go +++ b/cmd/ateapi/internal/controlapi/access_policy_errors_test.go @@ -19,8 +19,8 @@ import ( "fmt" "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/apierror" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/apierror" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) diff --git a/cmd/ateapi/internal/controlapi/access_policy_test.go b/cmd/ateapi/internal/controlapi/access_policy_test.go index 77a2ea2a52..ec3fcc1348 100644 --- a/cmd/ateapi/internal/controlapi/access_policy_test.go +++ b/cmd/ateapi/internal/controlapi/access_policy_test.go @@ -19,12 +19,12 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/authz" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/principal" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/authz" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/principal" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "google.golang.org/grpc/codes" ) diff --git a/cmd/ateapi/internal/controlapi/actor.go b/cmd/ateapi/internal/controlapi/actor.go index b0591d18e0..9b0ad3f727 100644 --- a/cmd/ateapi/internal/controlapi/actor.go +++ b/cmd/ateapi/internal/controlapi/actor.go @@ -25,16 +25,16 @@ import ( "slices" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/apivalidation" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/defaults" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/actoridjwt" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/substratex509" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/apivalidation" + "go.ate.dev/substrate/cmd/ateapi/internal/defaults" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/actoridjwt" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/substratex509" + "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel/attribute" "google.golang.org/grpc/credentials" "google.golang.org/grpc/peer" diff --git a/cmd/ateapi/internal/controlapi/actor_sizing_test.go b/cmd/ateapi/internal/controlapi/actor_sizing_test.go index 573257cfe9..d6cec28580 100644 --- a/cmd/ateapi/internal/controlapi/actor_sizing_test.go +++ b/cmd/ateapi/internal/controlapi/actor_sizing_test.go @@ -17,7 +17,7 @@ package controlapi import ( "testing" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // TestActorResourceLimits covers the actor-side extraction: the CPU/memory limits diff --git a/cmd/ateapi/internal/controlapi/actor_template.go b/cmd/ateapi/internal/controlapi/actor_template.go index c625463e1b..d362038038 100644 --- a/cmd/ateapi/internal/controlapi/actor_template.go +++ b/cmd/ateapi/internal/controlapi/actor_template.go @@ -19,12 +19,12 @@ import ( "errors" "fmt" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/apivalidation" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/defaults" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/apivalidation" + "go.ate.dev/substrate/cmd/ateapi/internal/defaults" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" "k8s.io/apimachinery/pkg/util/validation/field" ) diff --git a/cmd/ateapi/internal/controlapi/actor_template_test.go b/cmd/ateapi/internal/controlapi/actor_template_test.go index b28a4d5747..b6028de4d3 100644 --- a/cmd/ateapi/internal/controlapi/actor_template_test.go +++ b/cmd/ateapi/internal/controlapi/actor_template_test.go @@ -21,15 +21,15 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/objectstoreplugin/objectstoreplugintest" - "github.com/agent-substrate/substrate/internal/resources" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/objectstoreplugin/objectstoreplugintest" + "go.ate.dev/substrate/internal/resources" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + listersv1alpha1 "go.ate.dev/substrate/pkg/client/listers/api/v1alpha1" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/protobuf/testing/protocmp" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/cmd/ateapi/internal/controlapi/actor_test.go b/cmd/ateapi/internal/controlapi/actor_test.go index 479cb03204..3ba9b7079b 100644 --- a/cmd/ateapi/internal/controlapi/actor_test.go +++ b/cmd/ateapi/internal/controlapi/actor_test.go @@ -26,15 +26,16 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/apivalidation" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/localca" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/substratex509" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/cmd/ateapi/internal/apivalidation" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/localca" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/substratex509" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/peer" @@ -102,7 +103,7 @@ func TestValidateActorStatusCrash(t *testing.T) { name: "truncated crash message fits", oldVal: validActor(withActorStatus()), newVal: validActor(withCrash(func(c *ateapipb.ActorCrash) { - c.Message = newActorCrash("pause", strings.Repeat("x", 2*maxCrashMessageBytes)).GetMessage() + c.Message = newActorCrash(ateattr.OperationSuspend, strings.Repeat("x", 2*maxCrashMessageBytes)).GetMessage() })), }, { diff --git a/cmd/ateapi/internal/controlapi/atespace.go b/cmd/ateapi/internal/controlapi/atespace.go index f3f809a99d..2d72b37eef 100644 --- a/cmd/ateapi/internal/controlapi/atespace.go +++ b/cmd/ateapi/internal/controlapi/atespace.go @@ -19,12 +19,12 @@ import ( "errors" "fmt" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/apivalidation" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/defaults" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/apivalidation" + "go.ate.dev/substrate/cmd/ateapi/internal/defaults" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) func (s *RPCService) CreateAtespace(ctx context.Context, req *ateapipb.CreateAtespaceRequest) (*ateapipb.Atespace, error) { diff --git a/cmd/ateapi/internal/controlapi/atespace_test.go b/cmd/ateapi/internal/controlapi/atespace_test.go index 4c8bf5613e..4564485076 100644 --- a/cmd/ateapi/internal/controlapi/atespace_test.go +++ b/cmd/ateapi/internal/controlapi/atespace_test.go @@ -18,14 +18,14 @@ import ( "context" "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/authz" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/principal" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" openfgav1 "github.com/openfga/api/proto/openfga/v1" + "go.ate.dev/substrate/cmd/ateapi/internal/authz" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/principal" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "google.golang.org/grpc/codes" ) diff --git a/cmd/ateapi/internal/controlapi/common_test.go b/cmd/ateapi/internal/controlapi/common_test.go index 55e18b7c19..2d5ec8bd7a 100644 --- a/cmd/ateapi/internal/controlapi/common_test.go +++ b/cmd/ateapi/internal/controlapi/common_test.go @@ -18,8 +18,8 @@ import ( "context" "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/testing/protocmp" "k8s.io/apimachinery/pkg/util/validation/field" ) diff --git a/cmd/ateapi/internal/controlapi/converter.go b/cmd/ateapi/internal/controlapi/converter.go index e30d192a01..174c371cc3 100644 --- a/cmd/ateapi/internal/controlapi/converter.go +++ b/cmd/ateapi/internal/controlapi/converter.go @@ -15,8 +15,8 @@ package controlapi import ( - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // fidelityToAtelet maps the public fidelity ladder onto the identical atelet @@ -34,3 +34,39 @@ func fidelityToAtelet(in ateapipb.SnapshotFidelity) ateletpb.SnapshotFidelity { return ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED } } + +// fidelityFromAtelet maps atelet's fidelity enum back onto the identical +// public one. Anything else comes back UNSPECIFIED. +func fidelityFromAtelet(in ateletpb.SnapshotFidelity) ateapipb.SnapshotFidelity { + switch in { + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: + return ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS: + return ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: + return ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY + default: + return ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED + } +} + +// snapshotReportFromAtelet converts a snapshot atelet reports into the form +// ReportActorSnapshotState takes, which carries the snapshot's fields inline. +// A report that names no snapshot converts to nil. +func snapshotReportFromAtelet(r *ateletpb.SnapshotReport) *ateapipb.SnapshotReport { + if r.GetSnapshot() == nil { + return nil + } + snap := r.GetSnapshot() + out := &ateapipb.SnapshotReport{ + Uuid: snap.GetSnapshotUuid(), + Generation: snap.GetGeneration(), + Fidelity: fidelityFromAtelet(snap.GetFidelity()), + LocalCommitTime: r.GetLocalCommitTime(), + DurableCompleteTime: r.GetDurableCompleteTime(), + } + if obj := snap.GetObject(); obj != nil { + out.Object = &ateapipb.ObjectSnapshot{SnapshotUri: obj.GetSnapshotUri()} + } + return out +} diff --git a/cmd/ateapi/internal/controlapi/converter_test.go b/cmd/ateapi/internal/controlapi/converter_test.go index c9ef4c0e4a..5266c46e38 100644 --- a/cmd/ateapi/internal/controlapi/converter_test.go +++ b/cmd/ateapi/internal/controlapi/converter_test.go @@ -16,9 +16,14 @@ package controlapi import ( "testing" + "time" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "github.com/google/go-cmp/cmp" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/timestamppb" + + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // TestFidelityToAtelet pins the level-by-level mapping onto the atelet enum: @@ -65,3 +70,80 @@ func TestFidelityToAtelet(t *testing.T) { }) } } + +// TestFidelityFromAtelet verifies each atelet fidelity maps back to the same +// public level, and anything else to UNSPECIFIED. +func TestFidelityFromAtelet(t *testing.T) { + tests := []struct { + name string + in ateletpb.SnapshotFidelity + want ateapipb.SnapshotFidelity + }{ + {"memory", ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY}, + {"rootfs", ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS}, + {"volumes", ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES}, + {"unspecified stays unspecified", ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED}, + {"value outside the enum", ateletpb.SnapshotFidelity(99), ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := fidelityFromAtelet(tt.in); got != tt.want { + t.Errorf("fidelityFromAtelet(%v) = %v, want %v", tt.in, got, tt.want) + } + }) + } +} + +func TestSnapshotReportFromAtelet(t *testing.T) { + committed := timestamppb.New(time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)) + uploaded := timestamppb.New(time.Date(2026, 10, 14, 12, 5, 0, 0, time.UTC)) + tests := []struct { + name string + in *ateletpb.SnapshotReport + want *ateapipb.SnapshotReport + }{{ + name: "every field", + in: &ateletpb.SnapshotReport{ + Snapshot: &ateletpb.Snapshot{ + SnapshotUuid: "snap-2", + Generation: 2, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, + Object: &ateletpb.ObjectStorage{SnapshotUri: "gs://snapshots/snap-2"}, + }, + LocalCommitTime: committed, + DurableCompleteTime: uploaded, + }, + want: &ateapipb.SnapshotReport{ + Uuid: "snap-2", + Generation: 2, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: "gs://snapshots/snap-2"}, + LocalCommitTime: committed, + DurableCompleteTime: uploaded, + }, + }, { + name: "a local snapshot has no object", + in: &ateletpb.SnapshotReport{ + Snapshot: &ateletpb.Snapshot{SnapshotUuid: "snap-3", Generation: 3, Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY}, + LocalCommitTime: committed, + }, + want: &ateapipb.SnapshotReport{ + Uuid: "snap-3", + Generation: 3, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + LocalCommitTime: committed, + }, + }, { + name: "a report that names no snapshot converts to nil", + in: &ateletpb.SnapshotReport{LocalCommitTime: committed}, + }, { + name: "nil stays nil", + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if diff := cmp.Diff(tt.want, snapshotReportFromAtelet(tt.in), protocmp.Transform()); diff != "" { + t.Errorf("snapshotReportFromAtelet mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/cmd/ateapi/internal/controlapi/crash.go b/cmd/ateapi/internal/controlapi/crash.go index c40628c995..5b505d0355 100644 --- a/cmd/ateapi/internal/controlapi/crash.go +++ b/cmd/ateapi/internal/controlapi/crash.go @@ -21,12 +21,12 @@ import ( "log/slog" "strings" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/actorevent" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/actorevent" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/timestamppb" diff --git a/cmd/ateapi/internal/controlapi/crash_test.go b/cmd/ateapi/internal/controlapi/crash_test.go index 67a06958c2..8e39921a7e 100644 --- a/cmd/ateapi/internal/controlapi/crash_test.go +++ b/cmd/ateapi/internal/controlapi/crash_test.go @@ -27,14 +27,14 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" - "github.com/agent-substrate/substrate/internal/actorevent" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/internal/actorevent" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel/attribute" otellog "go.opentelemetry.io/otel/log" "go.opentelemetry.io/otel/log/global" diff --git a/cmd/ateapi/internal/controlapi/delete_options.go b/cmd/ateapi/internal/controlapi/delete_options.go index f9f6086850..85a83f1930 100644 --- a/cmd/ateapi/internal/controlapi/delete_options.go +++ b/cmd/ateapi/internal/controlapi/delete_options.go @@ -15,8 +15,8 @@ package controlapi import ( - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) func toDeletePreconditions(opts *ateapipb.DeleteOptions) store.DeletePreconditions { diff --git a/cmd/ateapi/internal/controlapi/dialer.go b/cmd/ateapi/internal/controlapi/dialer.go index db3e72fa01..07160498b1 100644 --- a/cmd/ateapi/internal/controlapi/dialer.go +++ b/cmd/ateapi/internal/controlapi/dialer.go @@ -24,13 +24,13 @@ import ( "strconv" "sync" - "github.com/agent-substrate/substrate/internal/atelet" - "github.com/agent-substrate/substrate/internal/credbundle" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/internal/substratex509" "github.com/spiffe/go-spiffe/v2/bundle/x509bundle" "github.com/spiffe/go-spiffe/v2/spiffeid" "github.com/spiffe/go-spiffe/v2/svid/x509svid" + "go.ate.dev/substrate/internal/atelet" + "go.ate.dev/substrate/internal/credbundle" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/substratex509" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "google.golang.org/grpc/credentials" diff --git a/cmd/ateapi/internal/controlapi/dialer_test.go b/cmd/ateapi/internal/controlapi/dialer_test.go index 424d102ae9..728c5f0030 100644 --- a/cmd/ateapi/internal/controlapi/dialer_test.go +++ b/cmd/ateapi/internal/controlapi/dialer_test.go @@ -28,10 +28,10 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/internal/substratex509" "github.com/spiffe/go-spiffe/v2/bundle/x509bundle" "github.com/spiffe/go-spiffe/v2/spiffeid" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/substratex509" "google.golang.org/grpc" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials" diff --git a/cmd/ateapi/internal/controlapi/egress_policy.go b/cmd/ateapi/internal/controlapi/egress_policy.go index a2404088e7..0072db889f 100644 --- a/cmd/ateapi/internal/controlapi/egress_policy.go +++ b/cmd/ateapi/internal/controlapi/egress_policy.go @@ -19,12 +19,12 @@ import ( "errors" "fmt" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/apivalidation" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/defaults" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/apivalidation" + "go.ate.dev/substrate/cmd/ateapi/internal/defaults" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" "k8s.io/apimachinery/pkg/util/validation/field" ) diff --git a/cmd/ateapi/internal/controlapi/egress_policy_test.go b/cmd/ateapi/internal/controlapi/egress_policy_test.go index 90f7777ade..207aa3adfb 100644 --- a/cmd/ateapi/internal/controlapi/egress_policy_test.go +++ b/cmd/ateapi/internal/controlapi/egress_policy_test.go @@ -17,11 +17,11 @@ package controlapi import ( "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/protobuf/proto" ) diff --git a/cmd/ateapi/internal/controlapi/export_test.go b/cmd/ateapi/internal/controlapi/export_test.go new file mode 100644 index 0000000000..df91876b22 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/export_test.go @@ -0,0 +1,137 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "context" + "testing" + + "go.ate.dev/substrate/cmd/ateapi/internal/scheduling" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" + "google.golang.org/grpc/codes" +) + +// Hooks for the controlapi_test package, whose tests call WorkerService, which +// imports controlapi. + +var ( + NewTestPersistence = newTestPersistence + SeedAssignment = seedAssignment + WireTestAssignment = wireTestAssignment + SeedSuspendTemplate = seedSuspendTemplate + SuspendTestTemplateRef = suspendTestTemplateRef +) + +// TestStorageLocation is the storage location ReleaseWindowWorkflows' +// durable snapshots are under. +const TestStorageLocation = testStorageLocation + +// ConflictingUpdateStore fails every actor update with a version conflict. +type ConflictingUpdateStore = conflictingUpdateStore + +// ReleaseWindowWorkflow is a workflow step that releases an actor's worker +// before it records the actor's new state. +type ReleaseWindowWorkflow struct { + Name string + // State is the actor's state while the worker holds it. + State ateapipb.ActorState + // Snapshot, when set, is the actor's latest snapshot. + Snapshot func(t *testing.T) *ateapipb.Snapshot + // WorkerClass is the worker's sandbox class. Templates ask for gVisor, + // so "microvm" makes the worker ineligible. + WorkerClass string + // Finalize runs the workflow or step that releases the worker and then + // records the actor's new state against st. Pause and suspend reach the + // fake atelet newWireCaptureWorkflow serves for the worker + // WireTestAssignment names, so the actor must be assigned that worker + // and run from the template SeedSuspendTemplate stores. + Finalize func(t *testing.T, ctx context.Context, st store.Interface, actorRef resources.ActorRef, actor *ateapipb.Actor) error + // WantRetry is the code of Finalize retried against the real store. + WantRetry codes.Code + // WantFinalState is the actor's state once the retry commits. + WantFinalState ateapipb.ActorState +} + +// ReleaseWindowWorkflows returns pause, suspend, crash, and a resume onto an +// ineligible worker: the workflows that release a worker before rewriting the +// actor's status while the actor is in a state HostingNode resolves through +// the worker assignment. Revert and delete release theirs in REVERTING and +// DELETING, which HostingNode never resolves to a node. +func ReleaseWindowWorkflows() []ReleaseWindowWorkflow { + gvisorTemplate := &ateapipb.ActorTemplate{SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor"}} + return []ReleaseWindowWorkflow{ + { + Name: "pause", + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, + Snapshot: func(*testing.T) *ateapipb.Snapshot { + // A pause in progress: node-local and not yet captured. + return newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-snap-1", "") + }, + WorkerClass: "gvisor", + // The actor is seeded SUSPENDING, so the pause skips its mark step + // and its first actor update is the finalize commit. + Finalize: func(t *testing.T, ctx context.Context, st store.Interface, actorRef resources.ActorRef, _ *ateapipb.Actor) error { + w, _ := newWireCaptureWorkflow(t, st) + _, err := w.PauseActor(ctx, actorRef) + return err + }, + WantRetry: codes.OK, + WantFinalState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, + }, + { + Name: "suspend", + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, + Snapshot: func(t *testing.T) *ateapipb.Snapshot { + return newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snapshot-1", someActorSnapshotURI(t, testStorageLocation, "team-a", "snapshot-1"), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) + }, + WorkerClass: "gvisor", + // As for the pause, the finalize commit is the first actor update. + Finalize: func(t *testing.T, ctx context.Context, st store.Interface, actorRef resources.ActorRef, _ *ateapipb.Actor) error { + w, _ := newWireCaptureWorkflow(t, st) + _, err := w.SuspendActor(ctx, actorRef) + return err + }, + WantRetry: codes.OK, + WantFinalState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, + }, + { + Name: "crash", + State: ateapipb.ActorState_ACTOR_STATE_RUNNING, + WorkerClass: "gvisor", + Finalize: func(_ *testing.T, ctx context.Context, st store.Interface, actorRef resources.ActorRef, _ *ateapipb.Actor) error { + return crashActor(ctx, st, actorRef, ateattr.OperationResume, "test crash") + }, + WantRetry: codes.OK, + WantFinalState: ateapipb.ActorState_ACTOR_STATE_CRASHED, + }, + { + // A resume that finds its worker ineligible releases it, then + // crashes the actor. The retry finds the worker no longer hosts + // the actor and crashes it. + Name: "resume onto an ineligible worker", + State: ateapipb.ActorState_ACTOR_STATE_RESUMING, + WorkerClass: "microvm", + Finalize: func(_ *testing.T, ctx context.Context, st store.Interface, actorRef resources.ActorRef, actor *ateapipb.Actor) error { + _, err := (&ActorWorkflow{store: st, scheduler: scheduling.New(nil)}).validateAssignedWorker(ctx, actorRef, actor, gvisorTemplate) + return err + }, + WantRetry: codes.Aborted, + WantFinalState: ateapipb.ActorState_ACTOR_STATE_CRASHED, + }, + } +} diff --git a/cmd/ateapi/internal/controlapi/functionaltest/access_policy_test.go b/cmd/ateapi/internal/controlapi/functionaltest/access_policy_test.go index 70d15ed8db..0e73d0a5b4 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/access_policy_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/access_policy_test.go @@ -18,9 +18,9 @@ import ( "context" "testing" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go index 2ff8dbdd28..27df83d3f0 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go @@ -20,14 +20,14 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/controlapi" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/cmd/ateapi/internal/controlapi" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/resources" "google.golang.org/grpc/status" "k8s.io/apimachinery/pkg/util/wait" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" "google.golang.org/protobuf/testing/protocmp" diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index 02f6acd32d..b061f3d5b3 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -28,19 +28,19 @@ import ( "time" "cloud.google.com/go/storage" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/actoridjwt" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/objectstore" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/volume" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/actoridjwt" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/objectstore" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/volume" + "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/sdk/metric/metricdata" "google.golang.org/grpc/codes" @@ -5032,7 +5032,7 @@ func TestCheckpointFailureCrashes(t *testing.T) { _, err := tc.client.PauseActor(context.Background(), &ateapipb.PauseActorRequest{Actor: ref}) return err }, - wantMessage: "pause failed: atelet Checkpoint: injected checkpoint failure", + wantMessage: "suspend failed: atelet Checkpoint: injected checkpoint failure", }, { name: "suspend", @@ -5214,7 +5214,7 @@ func TestLifecycleOpPoolAttributesOnSuccess(t *testing.T) { }, { name: "pause", - op: ateattr.OperationPause, + op: ateattr.OperationSuspend, run: func(t *testing.T, tc *testContext, actor *ateapipb.ObjectRef) { t.Helper() if _, err := tc.client.PauseActor(context.Background(), &ateapipb.PauseActorRequest{Actor: actor}); err != nil { diff --git a/cmd/ateapi/internal/controlapi/functionaltest/atespace_test.go b/cmd/ateapi/internal/controlapi/functionaltest/atespace_test.go index c79db5bcf0..b682b50cd6 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/atespace_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/atespace_test.go @@ -19,8 +19,8 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/protobuf/testing/protocmp" "google.golang.org/protobuf/types/known/timestamppb" diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index c0f912416b..4454282ed8 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -22,24 +22,24 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/controlapi" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" - "github.com/agent-substrate/substrate/internal/ateinterceptors" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/internal/localca" - "github.com/agent-substrate/substrate/internal/localjwtauthority" - "github.com/agent-substrate/substrate/internal/objectstore/objectstoretest" - "github.com/agent-substrate/substrate/internal/objectstoreplugin/objectstoreplugintest" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/volume" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/client/clientset/versioned" - "github.com/agent-substrate/substrate/pkg/client/informers/externalversions" - listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/controlapi" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/cmd/ateapi/internal/workercache" + "go.ate.dev/substrate/internal/ateinterceptors" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/localca" + "go.ate.dev/substrate/internal/localjwtauthority" + "go.ate.dev/substrate/internal/objectstore/objectstoretest" + "go.ate.dev/substrate/internal/objectstoreplugin/objectstoreplugintest" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/volume" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/pkg/client/clientset/versioned" + "go.ate.dev/substrate/pkg/client/informers/externalversions" + listersv1alpha1 "go.ate.dev/substrate/pkg/client/listers/api/v1alpha1" + "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel/attribute" sdkmetric "go.opentelemetry.io/otel/sdk/metric" sdktrace "go.opentelemetry.io/otel/sdk/trace" diff --git a/cmd/ateapi/internal/controlapi/functionaltest/egress_policy_test.go b/cmd/ateapi/internal/controlapi/functionaltest/egress_policy_test.go index 65c5693b9f..470a53b499 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/egress_policy_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/egress_policy_test.go @@ -19,9 +19,9 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/testing/protocmp" diff --git a/cmd/ateapi/internal/controlapi/functionaltest/main_test.go b/cmd/ateapi/internal/controlapi/functionaltest/main_test.go index f169e2523f..9ea0ae0df7 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/main_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/main_test.go @@ -24,11 +24,11 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" - "github.com/agent-substrate/substrate/internal/objectstore/objectstoretest" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/serverboot" - "github.com/agent-substrate/substrate/internal/testenv" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/internal/objectstore/objectstoretest" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/serverboot" + "go.ate.dev/substrate/internal/testenv" "google.golang.org/grpc" "google.golang.org/protobuf/proto" corev1 "k8s.io/api/core/v1" diff --git a/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go b/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go index 54a268a0d8..4972f12fe7 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go @@ -19,11 +19,11 @@ import ( "fmt" "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/protobuf/testing/protocmp" ) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/worker_contention_test.go b/cmd/ateapi/internal/controlapi/functionaltest/worker_contention_test.go index 8325eb9c20..63863ffd06 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/worker_contention_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/worker_contention_test.go @@ -23,9 +23,9 @@ import ( "k8s.io/apimachinery/pkg/util/wait" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // TestResumeActor_ConcurrentOntoOneWorker is the regression test for a refusal diff --git a/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go b/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go index 810f5bb59f..01cc0e7306 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go @@ -20,9 +20,9 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/protobuf/testing/protocmp" "k8s.io/apimachinery/pkg/util/wait" diff --git a/cmd/ateapi/internal/controlapi/hosting.go b/cmd/ateapi/internal/controlapi/hosting.go index 1fa19663c9..852fb8c149 100644 --- a/cmd/ateapi/internal/controlapi/hosting.go +++ b/cmd/ateapi/internal/controlapi/hosting.go @@ -19,8 +19,8 @@ import ( "errors" "fmt" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // HostingNode is the node that holds the actor, and "" when no node does. The diff --git a/cmd/ateapi/internal/controlapi/hosting_test.go b/cmd/ateapi/internal/controlapi/hosting_test.go index 8e6e0b130e..0480a8c2b1 100644 --- a/cmd/ateapi/internal/controlapi/hosting_test.go +++ b/cmd/ateapi/internal/controlapi/hosting_test.go @@ -20,112 +20,33 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/scheduling" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - "google.golang.org/grpc/codes" + "go.ate.dev/substrate/cmd/ateapi/internal/scheduling" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/cmd/ateapi/internal/workercache" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" ) -// TestHostingNodeAfterWorkerRelease drives each workflow that releases an -// actor's worker before it rewrites the actor's status into the window -// between the two, by failing the second write. The status still names the -// worker there, and keeps doing so until the workflow is retried. The worker's -// node must not count as hosting the actor in that window, nor once the worker -// hosts another actor. +// TestHostingNodeAfterWorkerRelease drives each of ReleaseWindowWorkflows into +// the window between releasing the actor's worker and rewriting its status, by +// failing the second write. The status still names the worker there, and keeps +// doing so until the workflow is retried. The worker's node must not count as +// hosting the actor in that window, nor once the worker hosts another actor. func TestHostingNodeAfterWorkerRelease(t *testing.T) { // The worker is the one wireTestAssignment names, so a pause or suspend // reaches newWireCaptureWorkflow's fake atelet on its node. wa := wireTestAssignment() node := wa.GetNodeName() - gvisorTemplate := &ateapipb.ActorTemplate{SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor"}} - tests := []struct { - name string - state ateapipb.ActorState - // snapshot, when set, is the actor's latest snapshot. - snapshot func(t *testing.T) *ateapipb.Snapshot - // workerClass is the worker's sandbox class. A template asks for - // gVisor, so "microvm" makes the worker ineligible. - workerClass string - // finalize runs the workflow or step that releases the worker and - // then records the actor's new state against st. - finalize func(t *testing.T, ctx context.Context, st store.Interface, actorRef resources.ActorRef, actor *ateapipb.Actor) error - // wantRetry is the code of the retried finalize against the real - // store. - wantRetry codes.Code - // wantFinalState and wantFinalNode are the actor's state and - // HostingNode once the retry commits. - wantFinalState ateapipb.ActorState - wantFinalNode string - }{ - { - name: "pause", - state: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, - snapshot: func(*testing.T) *ateapipb.Snapshot { - // A pause in progress: node-local and not yet captured. - return newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-snap-1", "") - }, - workerClass: "gvisor", - // The actor is seeded SUSPENDING, so the pause skips its mark step - // and its first actor update is the finalize commit. - finalize: func(t *testing.T, ctx context.Context, st store.Interface, actorRef resources.ActorRef, _ *ateapipb.Actor) error { - w, _ := newWireCaptureWorkflow(t, st) - _, err := w.PauseActor(ctx, actorRef) - return err - }, - wantRetry: codes.OK, - wantFinalState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - // Paused: the node holding the local snapshot. - wantFinalNode: node, - }, - { - name: "suspend", - state: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, - snapshot: func(t *testing.T) *ateapipb.Snapshot { - return newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snapshot-1", someActorSnapshotURI(t, testStorageLocation, "team-a", "snapshot-1"), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) - }, - workerClass: "gvisor", - // As for the pause, the finalize commit is the first actor update. - finalize: func(t *testing.T, ctx context.Context, st store.Interface, actorRef resources.ActorRef, _ *ateapipb.Actor) error { - w, _ := newWireCaptureWorkflow(t, st) - _, err := w.SuspendActor(ctx, actorRef) - return err - }, - wantRetry: codes.OK, - wantFinalState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - }, - { - name: "crash", - state: ateapipb.ActorState_ACTOR_STATE_RUNNING, - workerClass: "gvisor", - finalize: func(_ *testing.T, ctx context.Context, st store.Interface, actorRef resources.ActorRef, _ *ateapipb.Actor) error { - return crashActor(ctx, st, actorRef, ateattr.OperationResume, "test crash") - }, - wantRetry: codes.OK, - wantFinalState: ateapipb.ActorState_ACTOR_STATE_CRASHED, - }, - { - // A resume that finds its worker ineligible releases it, then - // crashes the actor. The retry finds the worker no longer hosts - // the actor and crashes it. - name: "resume onto an ineligible worker", - state: ateapipb.ActorState_ACTOR_STATE_RESUMING, - workerClass: "microvm", - finalize: func(_ *testing.T, ctx context.Context, st store.Interface, actorRef resources.ActorRef, actor *ateapipb.Actor) error { - _, err := (&ActorWorkflow{store: st, scheduler: scheduling.New(nil)}).validateAssignedWorker(ctx, actorRef, actor, gvisorTemplate) - return err - }, - wantRetry: codes.Aborted, - wantFinalState: ateapipb.ActorState_ACTOR_STATE_CRASHED, - }, + // wantFinalNode is HostingNode once the retry commits, by workflow name. + wantFinalNode := map[string]string{ + // Paused: the node holding the local snapshot. + "pause": node, } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { + for _, tc := range ReleaseWindowWorkflows() { + t.Run(tc.Name, func(t *testing.T) { ctx := context.Background() persistence := newTestPersistence(t) seedSuspendTemplate(t, ctx, persistence, &ateapipb.SnapshotConfig{StorageLocation: testStorageLocation}) @@ -138,19 +59,19 @@ func TestHostingNodeAfterWorkerRelease(t *testing.T) { WorkerPod: wa.GetWorkerPod(), WorkerPodUid: wa.GetWorkerPodUid(), NodeName: node, - SandboxClass: tc.workerClass, + SandboxClass: tc.WorkerClass, Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}}, }); err != nil { t.Fatalf("CreateWorker: %v", err) } status := &ateapipb.ActorStatus{ - State: tc.state, + State: tc.State, AssignedNode: node, WorkerAssignment: wireTestAssignment(), LastAssignedGeneration: 1, } - if tc.snapshot != nil { - status.Snapshots = []*ateapipb.Snapshot{tc.snapshot(t)} + if tc.Snapshot != nil { + status.Snapshots = []*ateapipb.Snapshot{tc.Snapshot(t)} } created := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, @@ -184,15 +105,15 @@ func TestHostingNodeAfterWorkerRelease(t *testing.T) { } // The release commits; recording the new state fails. - if err := tc.finalize(t, ctx, conflictingUpdateStore{Interface: persistence}, actorRef, actor); err == nil { + if err := tc.Finalize(t, ctx, conflictingUpdateStore{Interface: persistence}, actorRef, actor); err == nil { t.Fatal("finalize succeeded through a store that fails every actor update") } if _, err := persistence.GetWorkerAssignment(ctx, workerName, uid); !errors.Is(err, store.ErrNotFound) { t.Fatalf("worker assignment after release: err %v, want ErrNotFound", err) } got, actor = hostingNode("released, status not finalized") - if actor.GetStatus().GetState() != tc.state || actor.GetStatus().GetWorkerAssignment() == nil { - t.Fatalf("actor after the failed update: state %v, worker assignment %v; want %v naming the worker", actor.GetStatus().GetState(), actor.GetStatus().GetWorkerAssignment(), tc.state) + if actor.GetStatus().GetState() != tc.State || actor.GetStatus().GetWorkerAssignment() == nil { + t.Fatalf("actor after the failed update: state %v, worker assignment %v; want %v naming the worker", actor.GetStatus().GetState(), actor.GetStatus().GetWorkerAssignment(), tc.State) } if got != "" { t.Errorf("HostingNode after the worker was released = %q, want \"\"", got) @@ -208,15 +129,15 @@ func TestHostingNodeAfterWorkerRelease(t *testing.T) { } // A retry records the new state. - if err := tc.finalize(t, ctx, persistence, actorRef, actor); apierror.Code(err) != tc.wantRetry { - t.Fatalf("finalize retry: code %v, want %v (err: %v)", apierror.Code(err), tc.wantRetry, err) + if err := tc.Finalize(t, ctx, persistence, actorRef, actor); apierror.Code(err) != tc.WantRetry { + t.Fatalf("finalize retry: code %v, want %v (err: %v)", apierror.Code(err), tc.WantRetry, err) } got, actor = hostingNode("finalized") - if actor.GetStatus().GetState() != tc.wantFinalState { - t.Errorf("actor state once finalized = %v, want %v", actor.GetStatus().GetState(), tc.wantFinalState) + if actor.GetStatus().GetState() != tc.WantFinalState { + t.Errorf("actor state once finalized = %v, want %v", actor.GetStatus().GetState(), tc.WantFinalState) } - if got != tc.wantFinalNode { - t.Errorf("HostingNode once finalized = %q, want %q", got, tc.wantFinalNode) + if want := wantFinalNode[tc.Name]; got != want { + t.Errorf("HostingNode once finalized = %q, want %q", got, want) } }) } diff --git a/cmd/ateapi/internal/controlapi/main_test.go b/cmd/ateapi/internal/controlapi/main_test.go index b9afb01cd8..d60fd4bc2e 100644 --- a/cmd/ateapi/internal/controlapi/main_test.go +++ b/cmd/ateapi/internal/controlapi/main_test.go @@ -17,7 +17,7 @@ package controlapi import ( "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" ) func TestMain(m *testing.M) { diff --git a/cmd/ateapi/internal/controlapi/metrics.go b/cmd/ateapi/internal/controlapi/metrics.go index 709b272a56..67d898e185 100644 --- a/cmd/ateapi/internal/controlapi/metrics.go +++ b/cmd/ateapi/internal/controlapi/metrics.go @@ -19,10 +19,10 @@ import ( "fmt" "time" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" "k8s.io/apimachinery/pkg/labels" @@ -161,7 +161,7 @@ func NewInstruments(meter metric.Meter) (*Instruments, error) { lifecycleOpDuration, err := meter.Float64Histogram( lifecycleOpDurationMetric, metric.WithUnit("s"), - metric.WithDescription("Duration of an actor lifecycle operation (create, resume, suspend, pause, delete) handled by ateapi."), + metric.WithDescription("Duration of an actor lifecycle operation (create, resume, suspend, delete, revert) handled by ateapi."), metric.WithExplicitBucketBoundaries(0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.25, 0.5, 1, 2.5, 5, 10, 30), ) if err != nil { diff --git a/cmd/ateapi/internal/controlapi/metrics_test.go b/cmd/ateapi/internal/controlapi/metrics_test.go index 6c67bfb96c..eb8bfe7654 100644 --- a/cmd/ateapi/internal/controlapi/metrics_test.go +++ b/cmd/ateapi/internal/controlapi/metrics_test.go @@ -21,10 +21,10 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel/attribute" sdkmetric "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/metricdata" @@ -321,7 +321,7 @@ func TestRecordLifecycleOp_OutcomeClassification(t *testing.T) { {name: "wrapped apierror", op: ateattr.OperationResume, err: fmt.Errorf("workflow failed at step Load: %w", apierror.NotFound("missing")), wantErrorType: "NotFound"}, {name: "upstream status is what the caller gets", op: ateattr.OperationResume, err: fmt.Errorf("actor crashed: %w", status.Error(codes.Unavailable, "atelet down")), wantErrorType: "Internal"}, {name: "context deadline", op: ateattr.OperationSuspend, err: fmt.Errorf("while checkpointing: %w", context.DeadlineExceeded), wantErrorType: "DeadlineExceeded"}, - {name: "plain error", op: ateattr.OperationPause, err: errors.New("store is down"), wantErrorType: "Internal"}, + {name: "plain error", op: ateattr.OperationSuspend, err: errors.New("store is down"), wantErrorType: "Internal"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/cmd/ateapi/internal/controlapi/mint_after_release_test.go b/cmd/ateapi/internal/controlapi/mint_after_release_test.go new file mode 100644 index 0000000000..369f0bb616 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/mint_after_release_test.go @@ -0,0 +1,201 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi_test + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "errors" + "testing" + "time" + + "go.ate.dev/substrate/cmd/ateapi/internal/ateletauth/ateletauthtest" + "go.ate.dev/substrate/cmd/ateapi/internal/controlapi" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/cmd/ateapi/internal/workerservice" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/localca" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" + "google.golang.org/grpc/codes" +) + +// TestMintWorkerActorCertificateAfterWorkerRelease drives each of +// ReleaseWindowWorkflows into the window between releasing the actor's worker +// and rewriting its status, by failing the second write. The status still +// names the worker there, and keeps doing so until the workflow is retried. +// The worker must not get a worker-for-actor certificate for the actor in that +// window, nor once it hosts another actor or the status is finalized, and +// another worker on the node never gets one. +func TestMintWorkerActorCertificateAfterWorkerRelease(t *testing.T) { + // The worker is the one WireTestAssignment names, so a pause or suspend + // reaches the workflow's fake atelet on its node. + wa := controlapi.WireTestAssignment() + node := wa.GetNodeName() + for _, wf := range controlapi.ReleaseWindowWorkflows() { + t.Run(wf.Name, func(t *testing.T) { + ctx := context.Background() + persistence := controlapi.NewTestPersistence(t) + controlapi.SeedSuspendTemplate(t, ctx, persistence, &ateapipb.SnapshotConfig{StorageLocation: controlapi.TestStorageLocation}) + actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"} + // wa's worker hosts the actor; the neighbor is another worker on + // its node. + workerName := wa.GetWorker().GetName() + neighborName := "neighbor-1" + for _, w := range []struct{ name, pod, podUID string }{ + {workerName, wa.GetWorkerPod(), wa.GetWorkerPodUid()}, + {neighborName, "neighbor-pod", "neighbor-pod-uid"}, + } { + if _, err := persistence.CreateWorker(ctx, &ateapipb.Worker{ + Metadata: &ateapipb.ResourceMetadata{Name: w.name}, + WorkerNamespace: wa.GetWorkerNamespace(), + WorkerPool: wa.GetWorkerPool(), + WorkerPod: w.pod, + WorkerPodUid: w.podUID, + NodeName: node, + SandboxClass: wf.WorkerClass, + Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}}, + }); err != nil { + t.Fatalf("CreateWorker(%s): %v", w.name, err) + } + } + status := &ateapipb.ActorStatus{ + State: wf.State, + AssignedNode: node, + WorkerAssignment: controlapi.WireTestAssignment(), + LastAssignedGeneration: 1, + } + if wf.Snapshot != nil { + status.Snapshots = []*ateapipb.Snapshot{wf.Snapshot(t)} + } + created := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, + ActorTemplate: controlapi.SuspendTestTemplateRef(), + Status: status, + }) + uid := created.GetMetadata().GetUid() + controlapi.SeedAssignment(t, persistence, workerName, &ateapipb.ActorAssignment{ + Actor: &ateapipb.ObjectRef{Atespace: actorRef.Atespace, Name: actorRef.Name}, + ActorUid: uid, + }) + + minter := newTestMinter(t, persistence) + csr := newTestCSR(t) + // mintAs asks for the actor's certificate as worker, through the + // atelet on node. + mintAs := func(worker string) codes.Code { + t.Helper() + _, err := minter.MintWorkerActorCertificate(ateletauthtest.ContextWith(ateletauthtest.CertOn(t, node)), &ateapipb.MintWorkerActorCertificateRequest{ + Actor: &ateapipb.ObjectRef{Atespace: actorRef.Atespace, Name: actorRef.Name}, + ActorUid: uid, + CertificateSigningRequest: csr, + Worker: &ateapipb.ObjectRef{Name: worker}, + }) + return apierror.Code(err) + } + // mint asks for the actor's certificate as the worker that the + // actor is assigned to and returns the actor as the mint saw it + // and the mint's code. + mint := func(stage string) (*ateapipb.Actor, codes.Code) { + t.Helper() + code := mintAs(workerName) + actor, gerr := persistence.GetActor(ctx, actorRef) + if gerr != nil { + t.Fatalf("GetActor: %v", gerr) + } + t.Logf("%s: actor_state=%s status_worker_set=%t cert_code=%s", + stage, actor.GetStatus().GetState(), actor.GetStatus().GetWorkerAssignment() != nil, code) + return actor, code + } + + actor, got := mint("hosted") + if got != codes.OK { + t.Fatalf("certificate mint while the worker holds the actor: code %v, want OK", got) + } + if got := mintAs(neighborName); got != codes.NotFound { + t.Errorf("certificate mint by another worker on the node: code %v, want NotFound", got) + } + + // The release commits; recording the new state fails. + if err := wf.Finalize(t, ctx, controlapi.ConflictingUpdateStore{Interface: persistence}, actorRef, actor); err == nil { + t.Fatal("finalize succeeded through a store that fails every actor update") + } + if _, err := persistence.GetWorkerAssignment(ctx, workerName, uid); !errors.Is(err, store.ErrNotFound) { + t.Fatalf("worker assignment after release: err %v, want ErrNotFound", err) + } + actor, got = mint("released, status not finalized") + if actor.GetStatus().GetState() != wf.State || actor.GetStatus().GetWorkerAssignment() == nil { + t.Fatalf("actor after the failed update: state %v, worker assignment %v; want %v naming the worker", actor.GetStatus().GetState(), actor.GetStatus().GetWorkerAssignment(), wf.State) + } + if got != codes.NotFound { + t.Errorf("certificate mint after the worker was released: code %v, want NotFound", got) + } + + // The worker takes another actor. + controlapi.SeedAssignment(t, persistence, workerName, &ateapipb.ActorAssignment{ + Actor: &ateapipb.ObjectRef{Atespace: actorRef.Atespace, Name: "actor-2"}, + ActorUid: "5c1d2e3f-4a5b-4c6d-8e7f-9a0b1c2d3e4f", + }) + if _, got := mint("worker rebooked for another actor"); got != codes.NotFound { + t.Errorf("certificate mint after the worker was rebooked: code %v, want NotFound", got) + } + + // A retry records the new state. + if err := wf.Finalize(t, ctx, persistence, actorRef, actor); apierror.Code(err) != wf.WantRetry { + t.Fatalf("finalize retry: code %v, want %v (err: %v)", apierror.Code(err), wf.WantRetry, err) + } + actor, got = mint("finalized") + if actor.GetStatus().GetState() != wf.WantFinalState { + t.Errorf("actor state once finalized = %v, want %v", actor.GetStatus().GetState(), wf.WantFinalState) + } + // No worker runs the actor once finalized, paused or not. + if got != codes.NotFound { + t.Errorf("certificate mint once finalized: code %v, want NotFound", got) + } + }) + } +} + +// newTestMinter returns a WorkerService that mints worker-for-actor +// certificates against st for the atelets ateletauthtest certifies. +func newTestMinter(t *testing.T, st store.Interface) *workerservice.Server { + t.Helper() + ca, err := localca.GenerateCA("1", localca.KeyTypeECDSAP256, 24*time.Hour) + if err != nil { + t.Fatalf("GenerateCA: %v", err) + } + caPool := &localca.ConcretePool{CAs: []*localca.CA{ca}, ActiveForSigning: "1"} + ateletID := installdefaults.SPIFFEID(installdefaults.SystemNamespace, installdefaults.AteletServiceAccount) + return workerservice.New(st, nil, ateletID, caPool, "", nil) +} + +// newTestCSR returns a certificate signing request for a fresh actor key. +func newTestCSR(t *testing.T) []byte { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("GenerateKey: %v", err) + } + csr, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{}, key) + if err != nil { + t.Fatalf("CreateCertificateRequest: %v", err) + } + return csr +} diff --git a/cmd/ateapi/internal/controlapi/pagination.go b/cmd/ateapi/internal/controlapi/pagination.go index f525f56cb9..61e24e74fc 100644 --- a/cmd/ateapi/internal/controlapi/pagination.go +++ b/cmd/ateapi/internal/controlapi/pagination.go @@ -17,9 +17,9 @@ package controlapi import ( "errors" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) const maxPageSize = 1000 diff --git a/cmd/ateapi/internal/controlapi/pagination_test.go b/cmd/ateapi/internal/controlapi/pagination_test.go index c8414f454b..81325afe79 100644 --- a/cmd/ateapi/internal/controlapi/pagination_test.go +++ b/cmd/ateapi/internal/controlapi/pagination_test.go @@ -17,8 +17,8 @@ package controlapi import ( "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) func TestToStoreListOptions(t *testing.T) { diff --git a/cmd/ateapi/internal/controlapi/sandbox_assets.go b/cmd/ateapi/internal/controlapi/sandbox_assets.go index ab3b838f54..47593c8462 100644 --- a/cmd/ateapi/internal/controlapi/sandbox_assets.go +++ b/cmd/ateapi/internal/controlapi/sandbox_assets.go @@ -17,11 +17,11 @@ package controlapi import ( "fmt" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/proto/ateletpb" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + listersv1alpha1 "go.ate.dev/substrate/pkg/client/listers/api/v1alpha1" + "go.ate.dev/substrate/pkg/proto/ateapipb" k8serrors "k8s.io/apimachinery/pkg/api/errors" ) diff --git a/cmd/ateapi/internal/controlapi/sandbox_assets_test.go b/cmd/ateapi/internal/controlapi/sandbox_assets_test.go index 3cd70e2329..4bc227eb55 100644 --- a/cmd/ateapi/internal/controlapi/sandbox_assets_test.go +++ b/cmd/ateapi/internal/controlapi/sandbox_assets_test.go @@ -18,9 +18,9 @@ import ( "strings" "testing" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + listersv1alpha1 "go.ate.dev/substrate/pkg/client/listers/api/v1alpha1" + "go.ate.dev/substrate/pkg/proto/ateapipb" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" "k8s.io/utils/ptr" diff --git a/cmd/ateapi/internal/controlapi/service.go b/cmd/ateapi/internal/controlapi/service.go index dcb5321e21..c109ddf5ed 100644 --- a/cmd/ateapi/internal/controlapi/service.go +++ b/cmd/ateapi/internal/controlapi/service.go @@ -19,16 +19,16 @@ import ( "sync" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" - "github.com/agent-substrate/substrate/internal/localca" - "github.com/agent-substrate/substrate/internal/localjwtauthority" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/volume" - "github.com/agent-substrate/substrate/internal/volume/csi" - listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/workercache" + "go.ate.dev/substrate/internal/localca" + "go.ate.dev/substrate/internal/localjwtauthority" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/volume" + "go.ate.dev/substrate/internal/volume/csi" + listersv1alpha1 "go.ate.dev/substrate/pkg/client/listers/api/v1alpha1" + "go.ate.dev/substrate/pkg/proto/ateapipb" + objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" storagev1listers "k8s.io/client-go/listers/storage/v1" ) diff --git a/cmd/ateapi/internal/controlapi/snapshot_helpers.go b/cmd/ateapi/internal/controlapi/snapshot_helpers.go index 2350e95bd0..d952125079 100644 --- a/cmd/ateapi/internal/controlapi/snapshot_helpers.go +++ b/cmd/ateapi/internal/controlapi/snapshot_helpers.go @@ -15,7 +15,7 @@ package controlapi import ( - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // findSnapshotByGeneration returns the Snapshot in snapshots with the given diff --git a/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go b/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go index c6eed381cc..647d44b5fc 100644 --- a/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go +++ b/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go @@ -17,8 +17,8 @@ package controlapi import ( "testing" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/testing/protocmp" ) diff --git a/cmd/ateapi/internal/controlapi/snapshot_report.go b/cmd/ateapi/internal/controlapi/snapshot_report.go new file mode 100644 index 0000000000..01a2b5fa30 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/snapshot_report.go @@ -0,0 +1,231 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "context" + "errors" + "log/slog" + + "google.golang.org/protobuf/proto" + + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" +) + +// ApplyActorSnapshotState records the snapshot state that the atelet on node +// reports for the Actor. The caller has authenticated that atelet. +func (s *RPCService) ApplyActorSnapshotState(ctx context.Context, actorRef resources.ActorRef, actorUID, node string, state *ateapipb.ActorSnapshotState) error { + setSpanActorRefAttributes(ctx, actorRef) + return s.actorWorkflow.ApplyReportedSnapshotState(ctx, actorRef, actorUID, node, state) +} + +// ApplyReportedSnapshotState applies a report from the atelet on node to the +// Actor. It holds the Actor's lease and applies reports only to a SUSPENDED or +// RUNNING Actor, so a report never interleaves with a workflow. A report that +// changes nothing writes nothing. +func (w *ActorWorkflow) ApplyReportedSnapshotState(ctx context.Context, actorRef resources.ActorRef, actorUID, node string, state *ateapipb.ActorSnapshotState) error { + ctx = preview.ContextWithOptIn(ctx) + // Check before taking the lease, so an atelet can't hold the lease of an + // Actor that isn't on its node. Check again under the lease. + if _, err := w.loadReportedActor(ctx, actorRef, actorUID, node); err != nil { + return err + } + leaseCtx, lease, err := w.acquireActorLease(ctx, actorRef) + if err != nil { + return err + } + defer lease.Close() + before, err := w.loadReportedActor(leaseCtx, actorRef, actorUID, node) + if err != nil { + return err + } + + // A workflow owns a transitioning Actor's snapshots, even after its lease + // lapses. A crashed Actor's snapshots wait for a revert. + switch st := before.GetStatus().GetState(); st { + case ateapipb.ActorState_ACTOR_STATE_SUSPENDED, ateapipb.ActorState_ACTOR_STATE_RUNNING: + case ateapipb.ActorState_ACTOR_STATE_CRASHED: + return apierror.FailedPrecondition("actor %s is %s; revert it first", actorRef, st) + case ateapipb.ActorState_ACTOR_STATE_SUSPENDING, ateapipb.ActorState_ACTOR_STATE_RESUMING, + ateapipb.ActorState_ACTOR_STATE_DELETING, ateapipb.ActorState_ACTOR_STATE_REVERTING: + return apierror.Aborted("actor %s is %s; retry once that finishes", actorRef, st) + default: + return apierror.Internal("actor %s has unexpected state %s", actorRef, st) + } + + applied := proto.Clone(before).(*ateapipb.Actor) + if err := applyActorSnapshotState(leaseCtx, applied.GetStatus(), node, state); err != nil { + return err + } + if proto.Equal(applied.GetStatus(), before.GetStatus()) { + return nil + } + if _, err := w.store.UpdateActor(leaseCtx, actorRef, store.PreconditionFrom(before), func(toUpdate *ateapipb.Actor) error { + toUpdate.Status = applied.GetStatus() + return nil + }); err != nil { + if errors.Is(err, store.ErrVersionConflict) { + return apierror.Aborted("concurrent update conflict, please retry") + } + return err + } + return nil +} + +// loadReportedActor returns the Actor if it is the incarnation actorUID names +// and is on node. Otherwise it returns NotFound, so an atelet learns nothing +// about Actors on other nodes. +func (w *ActorWorkflow) loadReportedActor(ctx context.Context, actorRef resources.ActorRef, actorUID, node string) (*ateapipb.Actor, error) { + actor, err := w.store.GetActor(ctx, actorRef) + if err != nil { + if errors.Is(err, store.ErrNotFound) { + return nil, apierror.NotFound("Actor %s not found", actorRef) + } + return nil, err + } + if got := actor.GetMetadata().GetUid(); got != actorUID { + slog.WarnContext(ctx, "Refusing a snapshot state report naming a stale actor incarnation", + slog.Any("actor", actorRef), + slog.String("requested_uid", actorUID), + slog.String("actual_uid", got), + slog.String("caller_node", node)) + return nil, apierror.NotFound("Actor %s not found", actorRef) + } + if !actorOnNode(actor, node) { + slog.WarnContext(ctx, "Refusing a snapshot state report for an actor not on the caller's node", + slog.Any("actor", actorRef), + slog.String("caller_node", node), + slog.String("assigned_node", actor.GetStatus().GetAssignedNode())) + return nil, apierror.NotFound("Actor %s not found", actorRef) + } + return actor, nil +} + +// actorOnNode reports whether the Actor is placed on node or has a local +// snapshot there. Unlike HostingNode, it includes a node that only holds the +// Actor's snapshots, since that node's atelet reports them. +func actorOnNode(actor *ateapipb.Actor, node string) bool { + if actor.GetStatus().GetAssignedNode() == node { + return true + } + for _, snap := range actor.GetStatus().GetSnapshots() { + if snap.GetLocality() == node { + return true + } + } + return false +} + +// Messages logged for a reported durable snapshot that changes nothing. +const ( + msgDurableOlder = "Ignoring a durable snapshot older than the latest completed one" + msgDurableNotRecorded = "Ignoring a durable snapshot at a generation the actor no longer records" +) + +// applyActorSnapshotState applies the two facts a report carries: an upload +// finished, and the atelet on node detached the Actor. A report with neither +// latest_local nor uploading is a detach. +func applyActorSnapshotState(ctx context.Context, status *ateapipb.ActorStatus, node string, state *ateapipb.ActorSnapshotState) error { + if err := applyLatestDurable(ctx, status, state.GetLatestDurable()); err != nil { + return err + } + if state.GetLatestLocal() == nil && state.GetUploading() == nil { + return applyDetach(status, node, state.GetLatestDurable().GetGeneration()) + } + return nil +} + +// applyCheckpointState records an upload that atelet's state, as returned by +// Checkpoint or PromoteSnapshot, says finished, with the same rules as a +// report. atelet echoes the Snapshot it was asked for, so the fidelity recorded +// is the one requested, not read from the manifest. Unlike a report, a state +// with no local snapshot is not a detach: the workflow that made the call +// records where its snapshot lives. +func applyCheckpointState(ctx context.Context, status *ateapipb.ActorStatus, state *ateletpb.ActorSnapshotState) error { + return applyLatestDurable(ctx, status, snapshotReportFromAtelet(state.GetLatestDurable())) +} + +// applyLatestDurable marks the reported durable snapshot COMPLETED and records +// the fidelity from its manifest. Its uuid and upload URI must match the +// recorded ones. A generation at or below the latest COMPLETED one changes +// nothing, and so does one status no longer has, such as one a revert +// discarded. A late report is expected, so an older generation and a +// discarded one are logged at Info. A repeat of the latest isn't logged. +func applyLatestDurable(ctx context.Context, status *ateapipb.ActorStatus, reported *ateapipb.SnapshotReport) error { + if reported == nil { + return nil + } + if latest := findLatestDurableSnapshot(status, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); latest.GetGeneration() >= reported.GetGeneration() { + if latest.GetGeneration() > reported.GetGeneration() { + slog.InfoContext(ctx, msgDurableOlder, + slog.Int("generation", int(reported.GetGeneration())), + slog.String("uuid", reported.GetUuid()), + slog.Int("latest_completed_generation", int(latest.GetGeneration()))) + } + return nil + } + snap := findSnapshotByGeneration(status.GetSnapshots(), reported.GetGeneration()) + if snap == nil { + if reported.GetGeneration() <= status.GetLastAssignedGeneration() { + slog.InfoContext(ctx, msgDurableNotRecorded, + slog.Int("generation", int(reported.GetGeneration())), + slog.String("uuid", reported.GetUuid())) + return nil + } + return apierror.FailedPrecondition("snapshot generation %d (%s) is not recorded for this actor", reported.GetGeneration(), reported.GetUuid()) + } + if snap.GetUuid() != reported.GetUuid() { + return apierror.FailedPrecondition("snapshot generation %d is recorded as %s, not %s", reported.GetGeneration(), snap.GetUuid(), reported.GetUuid()) + } + want := snap.GetDurableSnapshot().GetObject().GetSnapshotUri() + if want == "" { + return apierror.FailedPrecondition("snapshot generation %d has no upload location recorded", reported.GetGeneration()) + } + if got := reported.GetObject().GetSnapshotUri(); got != want { + return apierror.InvalidArgument("snapshot generation %d was uploaded to %q, want %q", reported.GetGeneration(), got, want) + } + snap.Fidelity = reported.GetFidelity() + snap.DurableSnapshot.Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + return nil +} + +// applyDetach records that the atelet on node deleted its copies of the +// Actor's snapshots up to generation upTo, the latest snapshot it reported +// durable. A newer snapshot on node postdates the detach, so it stays. A +// SUSPENDED Actor is released from node once none of its snapshots are there. +// It refuses if the latest deleted snapshot isn't durable, so the Actor can't +// resume from an older one. +func applyDetach(status *ateapipb.ActorStatus, node string, upTo int32) error { + deleted := func(s *ateapipb.Snapshot) bool { return s.GetLocality() == node && s.GetGeneration() <= upTo } + if latest := latestSnapshot(status, deleted); latest != nil && + latest.GetDurableSnapshot().GetStatus() != ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + return apierror.FailedPrecondition("snapshot generation %d on node %s is not durable", latest.GetGeneration(), node) + } + for _, snap := range status.GetSnapshots() { + if deleted(snap) { + snap.Locality = "" + } + } + onNode := latestSnapshot(status, func(s *ateapipb.Snapshot) bool { return s.GetLocality() == node }) + if status.GetState() == ateapipb.ActorState_ACTOR_STATE_SUSPENDED && status.GetAssignedNode() == node && onNode == nil { + status.AssignedNode = "" + } + return nil +} diff --git a/cmd/ateapi/internal/controlapi/snapshot_report_test.go b/cmd/ateapi/internal/controlapi/snapshot_report_test.go new file mode 100644 index 0000000000..5e0a0f261e --- /dev/null +++ b/cmd/ateapi/internal/controlapi/snapshot_report_test.go @@ -0,0 +1,639 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "context" + "fmt" + "log/slog" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/codes" + "google.golang.org/protobuf/testing/protocmp" + + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" +) + +const ( + reportAtespace = "team-a" + reportActorUID = "5f2c1a90-7b34-4e6d-8a11-0c3e9d5b7f42" + reportNode = "node-a" + reportOther = "node-b" +) + +func snapName(gen int32) string { return fmt.Sprintf("snap-%d", gen) } + +// ownedURI returns the upload URI of actorUID's snapshot of generation gen. +func ownedURI(t *testing.T, actorUID string, gen int32) string { + t.Helper() + uri, err := resources.NewActorSnapshotURI("gs://snapshots", reportAtespace, actorUID, snapName(gen)) + if err != nil { + t.Fatal(err) + } + return uri.String() +} + +// localOnly returns a Snapshot held on locality with no durable copy. +func localOnly(gen int32, locality string) *ateapipb.Snapshot { + return &ateapipb.Snapshot{Uuid: snapName(gen), Generation: gen, Locality: locality, Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY} +} + +// withDurable returns a Snapshot with a durable copy in the given storage +// status. +func withDurable(t *testing.T, gen int32, locality string, storage ateapipb.SnapshotStorageStatus) *ateapipb.Snapshot { + t.Helper() + snap := localOnly(gen, locality) + snap.DurableSnapshot = &ateapipb.SnapshotStorage{ + Status: storage, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: ownedURI(t, reportActorUID, gen)}, + } + return snap +} + +func reported(t *testing.T, gen int32) *ateapipb.SnapshotReport { + t.Helper() + return reportedAt(t, gen, ownedURI(t, reportActorUID, gen)) +} + +func reportedAt(t *testing.T, gen int32, uri string) *ateapipb.SnapshotReport { + t.Helper() + return &ateapipb.SnapshotReport{ + Uuid: snapName(gen), + Generation: gen, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: uri}, + } +} + +func TestApplyActorSnapshotState(t *testing.T) { + const ( + pending = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + completed = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + suspended = ateapipb.ActorState_ACTOR_STATE_SUSPENDED + running = ateapipb.ActorState_ACTOR_STATE_RUNNING + ) + tests := []struct { + name string + status *ateapipb.ActorStatus + state *ateapipb.ActorSnapshotState + want *ateapipb.ActorStatus + wantCode codes.Code + }{{ + name: "upload marks the pending snapshot COMPLETED and keeps it local", + status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, reportNode, pending), + }}, + state: &ateapipb.ActorSnapshotState{LatestLocal: reported(t, 2), LatestDurable: reported(t, 2)}, + want: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, reportNode, completed), + }}, + }, { + name: "upload records the fidelity the manifest reports", + status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, reportNode, pending), + }}, + state: &ateapipb.ActorSnapshotState{LatestLocal: reported(t, 2), LatestDurable: func() *ateapipb.SnapshotReport { + r := reported(t, 2) + r.Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS + return r + }()}, + want: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ + func() *ateapipb.Snapshot { + snap := withDurable(t, 2, reportNode, completed) + snap.Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS + return snap + }(), + }}, + }, { + name: "a snapshot already COMPLETED changes nothing", + status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, reportNode, completed), + }}, + state: &ateapipb.ActorSnapshotState{LatestLocal: reported(t, 2), LatestDurable: reported(t, 2)}, + want: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, reportNode, completed), + }}, + }, { + name: "a generation below the latest COMPLETED one changes nothing", + status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 1, reportNode, pending), + withDurable(t, 2, reportNode, completed), + }}, + state: &ateapipb.ActorSnapshotState{LatestLocal: reported(t, 2), LatestDurable: func() *ateapipb.SnapshotReport { + r := reported(t, 1) + r.Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS + return r + }()}, + want: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 1, reportNode, pending), + withDurable(t, 2, reportNode, completed), + }}, + }, { + name: "a generation status no longer has changes nothing", + status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, LastAssignedGeneration: 3, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 3, reportNode, pending), + }}, + state: &ateapipb.ActorSnapshotState{LatestLocal: reported(t, 3), LatestDurable: reported(t, 2)}, + want: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, LastAssignedGeneration: 3, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 3, reportNode, pending), + }}, + }, { + name: "a generation newer than any assigned is refused", + status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, reportNode, pending), + }}, + state: &ateapipb.ActorSnapshotState{LatestLocal: reported(t, 2), LatestDurable: reported(t, 5)}, + wantCode: codes.FailedPrecondition, + }, { + name: "a uuid other than the recorded one is refused", + status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, reportNode, pending), + }}, + state: &ateapipb.ActorSnapshotState{LatestLocal: reported(t, 2), LatestDurable: func() *ateapipb.SnapshotReport { + r := reported(t, 2) + r.Uuid = "other" + return r + }()}, + wantCode: codes.FailedPrecondition, + }, { + name: "a snapshot with no recorded upload URI is refused", + status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ + localOnly(2, reportNode), + }}, + state: &ateapipb.ActorSnapshotState{LatestLocal: reported(t, 2), LatestDurable: reported(t, 2)}, + wantCode: codes.FailedPrecondition, + }, { + name: "an upload URI other than the recorded one is refused", + status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, reportNode, pending), + }}, + state: &ateapipb.ActorSnapshotState{LatestLocal: reported(t, 2), LatestDurable: reportedAt(t, 2, ownedURI(t, reportActorUID, 9))}, + wantCode: codes.InvalidArgument, + }, { + name: "latest_local and uploading are not checked", + status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, reportNode, pending), + }}, + state: &ateapipb.ActorSnapshotState{LatestLocal: reported(t, 7), Uploading: reported(t, 8)}, + want: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, reportNode, pending), + }}, + }, { + name: "an upload in flight with no local snapshot is not a detach", + status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, reportNode, pending), + }}, + state: &ateapipb.ActorSnapshotState{Uploading: reported(t, 2)}, + want: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, reportNode, pending), + }}, + }, { + name: "detach releases a suspended actor from the node", + status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 1, "", completed), + withDurable(t, 2, reportNode, completed), + }}, + state: &ateapipb.ActorSnapshotState{LatestDurable: reported(t, 2)}, + want: &ateapipb.ActorStatus{State: suspended, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 1, "", completed), + withDurable(t, 2, "", completed), + }}, + }, { + name: "a detach can complete the upload it reports", + status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, reportNode, pending), + }}, + state: &ateapipb.ActorSnapshotState{LatestDurable: reported(t, 2)}, + want: &ateapipb.ActorStatus{State: suspended, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, "", completed), + }}, + }, { + name: "a late detach leaves a newer snapshot on the node and keeps the actor placed", + status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, reportNode, completed), + withDurable(t, 3, reportNode, completed), + }}, + state: &ateapipb.ActorSnapshotState{LatestDurable: reported(t, 2)}, + want: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, "", completed), + withDurable(t, 3, reportNode, completed), + }}, + }, { + name: "a late detach leaves a newer local-only snapshot on the node", + status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 1, "", completed), + localOnly(2, reportNode), + }}, + state: &ateapipb.ActorSnapshotState{LatestDurable: reported(t, 1)}, + want: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 1, "", completed), + localOnly(2, reportNode), + }}, + }, { + name: "a detach of a snapshot that isn't durable is refused", + status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ + localOnly(1, reportNode), + }}, + state: &ateapipb.ActorSnapshotState{LatestDurable: reported(t, 2)}, + wantCode: codes.FailedPrecondition, + }, { + name: "a detach from a node that holds no snapshot only releases the actor", + status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, "", completed), + }}, + state: &ateapipb.ActorSnapshotState{LatestDurable: reported(t, 2)}, + want: &ateapipb.ActorStatus{State: suspended, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, "", completed), + }}, + }, { + name: "a running actor's detach keeps it placed on the node", + status: &ateapipb.ActorStatus{State: running, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, reportNode, completed), + }}, + state: &ateapipb.ActorSnapshotState{LatestDurable: reported(t, 2)}, + want: &ateapipb.ActorStatus{State: running, AssignedNode: reportNode, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, "", completed), + }}, + }, { + name: "a detach from one node leaves another node alone", + status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportOther, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, reportNode, completed), + localOnly(3, reportOther), + }}, + state: &ateapipb.ActorSnapshotState{LatestDurable: reported(t, 2)}, + want: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportOther, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, "", completed), + localOnly(3, reportOther), + }}, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := applyActorSnapshotState(context.Background(), tt.status, reportNode, tt.state) + if tt.wantCode != codes.OK { + if got := apierror.Code(err); got != tt.wantCode { + t.Fatalf("applyActorSnapshotState() error = %v, want code %v", err, tt.wantCode) + } + return + } + if err != nil { + t.Fatalf("applyActorSnapshotState() error = %v", err) + } + if diff := cmp.Diff(tt.want, tt.status, protocmp.Transform()); diff != "" { + t.Errorf("status mismatch (-want +got):\n%s", diff) + } + }) + } +} + +// A reported durable snapshot older than the latest COMPLETED one, or at a +// generation status no longer has, is logged at Info, once, with the +// generation it named. A repeat of the latest, and one that is applied, aren't +// logged. +func TestApplyLatestDurableLogsStaleReports(t *testing.T) { + const ( + pending = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + completed = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + ) + tests := []struct { + name string + status *ateapipb.ActorStatus + gen int32 + wantMsg string + wantAttrs map[string]string + }{{ + name: "below the latest COMPLETED generation", + status: &ateapipb.ActorStatus{LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 1, reportNode, pending), + withDurable(t, 2, reportNode, completed), + }}, + gen: 1, + wantMsg: msgDurableOlder, + wantAttrs: map[string]string{"generation": "1", "uuid": snapName(1), "latest_completed_generation": "2"}, + }, { + name: "a generation status no longer has", + status: &ateapipb.ActorStatus{LastAssignedGeneration: 3, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 3, reportNode, pending), + }}, + gen: 2, + wantMsg: msgDurableNotRecorded, + wantAttrs: map[string]string{"generation": "2", "uuid": snapName(2)}, + }, { + name: "the latest COMPLETED generation itself", + status: &ateapipb.ActorStatus{LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, reportNode, completed), + }}, + gen: 2, + }, { + name: "applied", + status: &ateapipb.ActorStatus{LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, reportNode, pending), + }}, + gen: 2, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + type logged struct { + msg string + attrs map[string]string + } + var got []logged + prev := slog.Default() + slog.SetDefault(slog.New(slogHandlerFunc(func(r slog.Record) { + if r.Message != msgDurableOlder && r.Message != msgDurableNotRecorded { + return + } + l := logged{msg: r.Message, attrs: map[string]string{}} + r.Attrs(func(a slog.Attr) bool { + l.attrs[a.Key] = a.Value.String() + return true + }) + got = append(got, l) + }))) + t.Cleanup(func() { slog.SetDefault(prev) }) + + if err := applyLatestDurable(context.Background(), tt.status, reported(t, tt.gen)); err != nil { + t.Fatalf("applyLatestDurable() error = %v", err) + } + var want []logged + if tt.wantMsg != "" { + want = []logged{{msg: tt.wantMsg, attrs: tt.wantAttrs}} + } + if diff := cmp.Diff(want, got, cmp.AllowUnexported(logged{})); diff != "" { + t.Errorf("logged records mismatch (-want +got):\n%s", diff) + } + }) + } +} + +// A report is applied to the stored actor under its lease. A refused report, +// and a report that changes nothing, write nothing. +func TestApplyReportedSnapshotState(t *testing.T) { + ctx := context.Background() + persistence := newTestPersistence(t) + w := newMarkOnlyWorkflow(persistence) + tmpl := &ateapipb.ActorTemplate{SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: testStorageLocation}} + + actorRef := resources.ActorRef{Atespace: reportAtespace, Name: "actor-1"} + actor := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, + Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}, + }) + uid := actor.GetMetadata().GetUid() + pending := mustActorSnapshotURI(t, tmpl, actor, "snap-2") + actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { + s.AssignedNode = reportNode + s.LastAssignedGeneration = 2 + snap := newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", pending.Name(), pending.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) + snap.Locality = reportNode + s.Snapshots = []*ateapipb.Snapshot{snap} + }) + uploaded := reportedAt(t, 2, pending.String()) + state := &ateapipb.ActorSnapshotState{LatestLocal: uploaded, LatestDurable: uploaded} + + assertUnwritten := func(t *testing.T) { + t.Helper() + got, err := persistence.GetActor(ctx, actorRef) + if err != nil { + t.Fatal(err) + } + if got.GetMetadata().GetVersion() != actor.GetMetadata().GetVersion() { + t.Errorf("a refused report wrote the actor: version %d, want %d", got.GetMetadata().GetVersion(), actor.GetMetadata().GetVersion()) + } + } + + t.Run("refused while another operation holds the actor", func(t *testing.T) { + lease, err := persistence.AcquireLease(ctx, actorLeaseKey(actorRef)) + if err != nil { + t.Fatalf("AcquireLease() failed: %v", err) + } + defer lease.Close() + if err := w.ApplyReportedSnapshotState(ctx, actorRef, uid, reportNode, state); apierror.Code(err) != codes.Aborted { + t.Fatalf("ApplyReportedSnapshotState() = %v, want Aborted", err) + } + assertUnwritten(t) + }) + for _, tc := range []struct { + name, uid, node string + ref resources.ActorRef + }{ + {"refused for another node", uid, reportOther, actorRef}, + {"refused for a stale incarnation", "1a7e4c83-6d20-4f95-b3c8-9e0a2f6d4b17", reportNode, actorRef}, + {"refused for a missing actor", uid, reportNode, resources.ActorRef{Atespace: reportAtespace, Name: "no-such-actor"}}, + } { + t.Run(tc.name, func(t *testing.T) { + if err := w.ApplyReportedSnapshotState(ctx, tc.ref, tc.uid, tc.node, state); apierror.Code(err) != codes.NotFound { + t.Fatalf("ApplyReportedSnapshotState() = %v, want NotFound", err) + } + assertUnwritten(t) + }) + } + // A transitioning actor gets Aborted, and a crashed one FailedPrecondition. + for _, tc := range []struct { + state ateapipb.ActorState + want codes.Code + }{ + {ateapipb.ActorState_ACTOR_STATE_SUSPENDING, codes.Aborted}, + {ateapipb.ActorState_ACTOR_STATE_RESUMING, codes.Aborted}, + {ateapipb.ActorState_ACTOR_STATE_DELETING, codes.Aborted}, + {ateapipb.ActorState_ACTOR_STATE_REVERTING, codes.Aborted}, + {ateapipb.ActorState_ACTOR_STATE_CRASHED, codes.FailedPrecondition}, + } { + t.Run("refused for a "+tc.state.String()+" actor", func(t *testing.T) { + actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { s.State = tc.state }) + defer func() { + actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { + s.State = ateapipb.ActorState_ACTOR_STATE_SUSPENDED + }) + }() + if err := w.ApplyReportedSnapshotState(ctx, actorRef, uid, reportNode, state); apierror.Code(err) != tc.want { + t.Fatalf("ApplyReportedSnapshotState() = %v, want %v", err, tc.want) + } + assertUnwritten(t) + }) + } + t.Run("refused for an upload to the wrong URI", func(t *testing.T) { + wrong := &ateapipb.ActorSnapshotState{LatestLocal: uploaded, LatestDurable: reportedAt(t, 2, ownedURI(t, uid, 9))} + if err := w.ApplyReportedSnapshotState(ctx, actorRef, uid, reportNode, wrong); apierror.Code(err) != codes.InvalidArgument { + t.Fatalf("ApplyReportedSnapshotState() = %v, want InvalidArgument", err) + } + assertUnwritten(t) + }) + + if err := w.ApplyReportedSnapshotState(ctx, actorRef, uid, reportNode, state); err != nil { + t.Fatalf("ApplyReportedSnapshotState() failed: %v", err) + } + got, err := persistence.GetActor(ctx, actorRef) + if err != nil { + t.Fatal(err) + } + if st := got.GetStatus().GetSnapshots()[0].GetDurableSnapshot().GetStatus(); st != ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + t.Errorf("durable snapshot status = %v, want COMPLETED", st) + } + + // A report that changes nothing writes nothing. + actor = got + if err := w.ApplyReportedSnapshotState(ctx, actorRef, uid, reportNode, state); err != nil { + t.Fatalf("repeated ApplyReportedSnapshotState() failed: %v", err) + } + assertUnwritten(t) +} + +func TestActorOnNode(t *testing.T) { + actor := func(assigned string, localities ...string) *ateapipb.Actor { + status := &ateapipb.ActorStatus{AssignedNode: assigned} + for i, l := range localities { + status.Snapshots = append(status.Snapshots, localOnly(int32(i+1), l)) + } + return &ateapipb.Actor{Status: status} + } + for _, tc := range []struct { + name string + actor *ateapipb.Actor + want bool + }{ + {"placed on the node", actor(reportNode), true}, + {"holds a snapshot on the node", actor(reportOther, "", reportNode), true}, + {"neither", actor(reportOther, reportOther), false}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := actorOnNode(tc.actor, reportNode); got != tc.want { + t.Errorf("actorOnNode() = %v, want %v", got, tc.want) + } + }) + } +} + +// ateletReported is reported(t, gen) as atelet returns it. +func ateletReported(t *testing.T, gen int32) *ateletpb.SnapshotReport { + t.Helper() + return &ateletpb.SnapshotReport{Snapshot: &ateletpb.Snapshot{ + SnapshotUuid: snapName(gen), + Generation: gen, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateletpb.ObjectStorage{SnapshotUri: ownedURI(t, reportActorUID, gen)}, + }} +} + +func TestApplyCheckpointState(t *testing.T) { + const ( + pending = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + completed = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + suspended = ateapipb.ActorState_ACTOR_STATE_SUSPENDED + suspending = ateapipb.ActorState_ACTOR_STATE_SUSPENDING + ) + tests := []struct { + name string + status *ateapipb.ActorStatus + state *ateletpb.ActorSnapshotState + want *ateapipb.ActorStatus + wantCode codes.Code + }{{ + name: "an immediate checkpoint marks its snapshot COMPLETED", + status: &ateapipb.ActorStatus{State: suspending, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, "", pending), + }}, + state: &ateletpb.ActorSnapshotState{LatestDurable: ateletReported(t, 2)}, + want: &ateapipb.ActorStatus{State: suspending, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, "", completed), + }}, + }, { + name: "an immediate checkpoint records the fidelity it reports", + status: &ateapipb.ActorStatus{State: suspending, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, "", pending), + }}, + state: &ateletpb.ActorSnapshotState{LatestDurable: func() *ateletpb.SnapshotReport { + r := ateletReported(t, 2) + r.Snapshot.Fidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS + return r + }()}, + want: &ateapipb.ActorStatus{State: suspending, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ + func() *ateapipb.Snapshot { + snap := withDurable(t, 2, "", completed) + snap.Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS + return snap + }(), + }}, + }, { + name: "a deferred checkpoint changes nothing", + status: &ateapipb.ActorStatus{State: suspending, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ + localOnly(2, ""), + }}, + state: &ateletpb.ActorSnapshotState{LatestLocal: ateletReported(t, 2)}, + want: &ateapipb.ActorStatus{State: suspending, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ + localOnly(2, ""), + }}, + }, { + name: "a deferred checkpoint records an older upload that finished", + status: &ateapipb.ActorStatus{State: suspending, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 1, reportNode, pending), + localOnly(2, ""), + }}, + state: &ateletpb.ActorSnapshotState{LatestLocal: ateletReported(t, 2), LatestDurable: ateletReported(t, 1)}, + want: &ateapipb.ActorStatus{State: suspending, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 1, reportNode, completed), + localOnly(2, ""), + }}, + }, { + name: "no local snapshot is not a detach", + status: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, reportNode, completed), + }}, + state: &ateletpb.ActorSnapshotState{LatestDurable: ateletReported(t, 2)}, + want: &ateapipb.ActorStatus{State: suspended, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, reportNode, completed), + }}, + }, { + name: "an upload URI other than the recorded one is refused", + status: &ateapipb.ActorStatus{State: suspending, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, "", pending), + }}, + state: &ateletpb.ActorSnapshotState{LatestDurable: func() *ateletpb.SnapshotReport { + r := ateletReported(t, 2) + r.Snapshot.Object.SnapshotUri = ownedURI(t, reportActorUID, 9) + return r + }()}, + wantCode: codes.InvalidArgument, + }, { + name: "an unset state changes nothing", + status: &ateapipb.ActorStatus{State: suspending, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, "", pending), + }}, + want: &ateapipb.ActorStatus{State: suspending, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ + withDurable(t, 2, "", pending), + }}, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := applyCheckpointState(context.Background(), tt.status, tt.state) + if tt.wantCode != codes.OK { + if got := apierror.Code(err); got != tt.wantCode { + t.Fatalf("applyCheckpointState() error = %v, want code %v", err, tt.wantCode) + } + return + } + if err != nil { + t.Fatalf("applyCheckpointState() error = %v", err) + } + if diff := cmp.Diff(tt.want, tt.status, protocmp.Transform()); diff != "" { + t.Errorf("status mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/cmd/ateapi/internal/controlapi/span_identity.go b/cmd/ateapi/internal/controlapi/span_identity.go index a344daedd4..417be0f2be 100644 --- a/cmd/ateapi/internal/controlapi/span_identity.go +++ b/cmd/ateapi/internal/controlapi/span_identity.go @@ -19,9 +19,9 @@ import ( "go.opentelemetry.io/otel/trace" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // setSpanActorAttributes annotates the RPC's server span (from ctx) with the diff --git a/cmd/ateapi/internal/controlapi/span_identity_test.go b/cmd/ateapi/internal/controlapi/span_identity_test.go index 5ece95cc38..9bcf83d420 100644 --- a/cmd/ateapi/internal/controlapi/span_identity_test.go +++ b/cmd/ateapi/internal/controlapi/span_identity_test.go @@ -22,9 +22,9 @@ import ( sdktrace "go.opentelemetry.io/otel/sdk/trace" "go.opentelemetry.io/otel/sdk/trace/tracetest" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // recordRootSpanAttrs runs fn under a fresh recording root span from a local diff --git a/cmd/ateapi/internal/controlapi/tag.go b/cmd/ateapi/internal/controlapi/tag.go index c356290af1..cd6187076e 100644 --- a/cmd/ateapi/internal/controlapi/tag.go +++ b/cmd/ateapi/internal/controlapi/tag.go @@ -19,12 +19,12 @@ import ( "errors" "fmt" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/apivalidation" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/defaults" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/apivalidation" + "go.ate.dev/substrate/cmd/ateapi/internal/defaults" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" "k8s.io/apimachinery/pkg/util/validation/field" ) diff --git a/cmd/ateapi/internal/controlapi/tag_test.go b/cmd/ateapi/internal/controlapi/tag_test.go index 7af624d22d..347dc3f65f 100644 --- a/cmd/ateapi/internal/controlapi/tag_test.go +++ b/cmd/ateapi/internal/controlapi/tag_test.go @@ -22,11 +22,11 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/protobuf/testing/protocmp" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // TestListTags checks the atespace scoping and the paging of the diff --git a/cmd/ateapi/internal/controlapi/template_reconciler.go b/cmd/ateapi/internal/controlapi/template_reconciler.go index 2a00e3439b..cbb583de08 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler.go @@ -22,10 +22,10 @@ import ( "time" "unicode/utf8" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/protobuf/types/known/timestamppb" "k8s.io/apimachinery/pkg/util/wait" diff --git a/cmd/ateapi/internal/controlapi/template_reconciler_test.go b/cmd/ateapi/internal/controlapi/template_reconciler_test.go index f4dfd27dd9..274aae5a45 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler_test.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler_test.go @@ -25,11 +25,11 @@ import ( "time" "unicode/utf8" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/apivalidation" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/apivalidation" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" diff --git a/cmd/ateapi/internal/controlapi/validate.go b/cmd/ateapi/internal/controlapi/validate.go index 1c03d1708d..c83b3fdac3 100644 --- a/cmd/ateapi/internal/controlapi/validate.go +++ b/cmd/ateapi/internal/controlapi/validate.go @@ -15,8 +15,8 @@ package controlapi import ( - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/pkg/proto/ateapipb" "k8s.io/apimachinery/pkg/util/validation/field" ) diff --git a/cmd/ateapi/internal/controlapi/volumes.go b/cmd/ateapi/internal/controlapi/volumes.go index 021a79ccf9..fa9e25194e 100644 --- a/cmd/ateapi/internal/controlapi/volumes.go +++ b/cmd/ateapi/internal/controlapi/volumes.go @@ -20,9 +20,9 @@ import ( "fmt" "log/slog" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/volume" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/volume" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" k8serrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/cmd/ateapi/internal/controlapi/volumes_test.go b/cmd/ateapi/internal/controlapi/volumes_test.go index 545384d569..129cee7193 100644 --- a/cmd/ateapi/internal/controlapi/volumes_test.go +++ b/cmd/ateapi/internal/controlapi/volumes_test.go @@ -26,9 +26,9 @@ import ( "google.golang.org/grpc/status" "google.golang.org/protobuf/testing/protocmp" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/volume" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/volume" + "go.ate.dev/substrate/pkg/proto/ateapipb" storagev1 "k8s.io/api/storage/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/cmd/ateapi/internal/controlapi/worker.go b/cmd/ateapi/internal/controlapi/worker.go index 8fb2d47c83..c7778cd82f 100644 --- a/cmd/ateapi/internal/controlapi/worker.go +++ b/cmd/ateapi/internal/controlapi/worker.go @@ -19,12 +19,12 @@ import ( "errors" "fmt" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/apivalidation" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/defaults" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/apivalidation" + "go.ate.dev/substrate/cmd/ateapi/internal/defaults" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" "k8s.io/apimachinery/pkg/util/validation/field" ) diff --git a/cmd/ateapi/internal/controlapi/worker_assignment_reconciler.go b/cmd/ateapi/internal/controlapi/worker_assignment_reconciler.go index 45c3c85583..3b5bca3ac9 100644 --- a/cmd/ateapi/internal/controlapi/worker_assignment_reconciler.go +++ b/cmd/ateapi/internal/controlapi/worker_assignment_reconciler.go @@ -21,8 +21,8 @@ import ( "log/slog" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/pkg/proto/ateapipb" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/workqueue" ) diff --git a/cmd/ateapi/internal/controlapi/worker_assignment_reconciler_test.go b/cmd/ateapi/internal/controlapi/worker_assignment_reconciler_test.go index 9e47a76230..6962f9a882 100644 --- a/cmd/ateapi/internal/controlapi/worker_assignment_reconciler_test.go +++ b/cmd/ateapi/internal/controlapi/worker_assignment_reconciler_test.go @@ -20,9 +20,9 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/workercache" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // noWorkers is a workerWatcher that never delivers a Worker. diff --git a/cmd/ateapi/internal/controlapi/worker_assignments.go b/cmd/ateapi/internal/controlapi/worker_assignments.go index cb44bc0202..a4654c6cf6 100644 --- a/cmd/ateapi/internal/controlapi/worker_assignments.go +++ b/cmd/ateapi/internal/controlapi/worker_assignments.go @@ -19,8 +19,8 @@ import ( "errors" "fmt" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) type workerAssignmentReader interface { diff --git a/cmd/ateapi/internal/controlapi/worker_test.go b/cmd/ateapi/internal/controlapi/worker_test.go index ab422b2853..8ec1a6dcd3 100644 --- a/cmd/ateapi/internal/controlapi/worker_test.go +++ b/cmd/ateapi/internal/controlapi/worker_test.go @@ -20,11 +20,11 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/testing/protocmp" diff --git a/cmd/ateapi/internal/controlapi/workflow.go b/cmd/ateapi/internal/controlapi/workflow.go index b0ac809e0f..488135cbb9 100644 --- a/cmd/ateapi/internal/controlapi/workflow.go +++ b/cmd/ateapi/internal/controlapi/workflow.go @@ -21,17 +21,17 @@ import ( "log/slog" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/scheduling" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" - "github.com/agent-substrate/substrate/internal/actorevent" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/pluginsocket" - "github.com/agent-substrate/substrate/internal/resources" - listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "go.ate.dev/substrate/cmd/ateapi/internal/scheduling" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/workercache" + "go.ate.dev/substrate/internal/actorevent" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/pluginsocket" + "go.ate.dev/substrate/internal/resources" + listersv1alpha1 "go.ate.dev/substrate/pkg/client/listers/api/v1alpha1" + "go.ate.dev/substrate/pkg/proto/ateapipb" + objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" diff --git a/cmd/ateapi/internal/controlapi/workflow_delete.go b/cmd/ateapi/internal/controlapi/workflow_delete.go index d0402c9e91..de65c19a54 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete.go @@ -20,13 +20,13 @@ import ( "fmt" "log/slog" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) diff --git a/cmd/ateapi/internal/controlapi/workflow_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_delete_test.go index 67536ae114..0b2b3fe84a 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete_test.go @@ -21,13 +21,13 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/objectstore/objectstoretest" - "github.com/agent-substrate/substrate/internal/objectstoreplugin/objectstoreplugintest" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/objectstore/objectstoretest" + "go.ate.dev/substrate/internal/objectstoreplugin/objectstoreplugintest" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) diff --git a/cmd/ateapi/internal/controlapi/workflow_lease_test.go b/cmd/ateapi/internal/controlapi/workflow_lease_test.go index b17f0583d1..13dfcba8c6 100644 --- a/cmd/ateapi/internal/controlapi/workflow_lease_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_lease_test.go @@ -20,8 +20,8 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/resources" ) type leaseStore struct{ store.Interface } diff --git a/cmd/ateapi/internal/controlapi/workflow_objectstore_plugin_test.go b/cmd/ateapi/internal/controlapi/workflow_objectstore_plugin_test.go index 35f011e889..d32191fd5c 100644 --- a/cmd/ateapi/internal/controlapi/workflow_objectstore_plugin_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_objectstore_plugin_test.go @@ -18,9 +18,9 @@ import ( "context" "testing" - "github.com/agent-substrate/substrate/internal/ateinterceptors" - "github.com/agent-substrate/substrate/internal/resources" - objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "go.ate.dev/substrate/internal/ateinterceptors" + "go.ate.dev/substrate/internal/resources" + objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" diff --git a/cmd/ateapi/internal/controlapi/workflow_pause.go b/cmd/ateapi/internal/controlapi/workflow_pause.go deleted file mode 100644 index 2701afe4c5..0000000000 --- a/cmd/ateapi/internal/controlapi/workflow_pause.go +++ /dev/null @@ -1,300 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package controlapi - -import ( - "context" - "errors" - "fmt" - "log/slog" - "time" - - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - "go.opentelemetry.io/otel/attribute" - "google.golang.org/protobuf/proto" -) - -// PauseActor executes the workflow to pause a running actor. Idempotent: -// a re-entered workflow fast-forwards past the steps a previous attempt -// completed, deriving progress from the persisted actor alone. -func (w *ActorWorkflow) PauseActor(ctx context.Context, actorRef resources.ActorRef) (_ *ateapipb.Actor, err error) { - ctx = preview.ContextWithOptIn(ctx) - start := time.Now() - var actor *ateapipb.Actor - var actorTemplate *ateapipb.ActorTemplate - var wireFidelity string - // Set just before finalize; nil until then, so earlier exits label - // themselves from the record they hold. - var finalAttrs []attribute.KeyValue - - defer func() { - attrs := finalAttrs - if attrs == nil { - attrs = lifecycleOpAttrs(actor, actorTemplate, "", wireFidelity) - } - w.instruments.recordLifecycleOp(ctx, ateattr.OperationPause, start, err, attrs...) - }() - - leaseCtx, lease, err := w.acquireActorLease(ctx, actorRef) - if err != nil { - return nil, err - } - defer lease.Close() - - actor, actorTemplate, err = w.loadActorForPause(leaseCtx, actorRef) - if err != nil { - return nil, err - } - if actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_SUSPENDED && latestSnapshotIsLocalOnly(actor.GetStatus()) { - // Fully paused already: FinalizePaused commits SUSPENDED, the local - // snapshot's locality, and the cleared worker assignment in a single - // update, so there is nothing left to do. - // This success reports no pool, and cannot: the previous attempt - // released the worker, so the record names none (#957). - return actor, nil - } - var marked *ateapipb.Actor - if marked, err = w.ensureMarkedPausing(leaseCtx, actorRef, actor, actorTemplate); err != nil { - return nil, err - } - actor = marked - if wireFidelity, err = w.ensureAteletPaused(leaseCtx, actorRef, actor, actorTemplate); err != nil { - return nil, err - } - // TODO: There is no difference between suspend and pause for now, but we - // could optimize pause by not detaching. We would need to make sure Resume - // is idempotent. - if err = w.ensureVolumesDetached(leaseCtx, actor, actorTemplate, "DetachVolumesForPause", ateattr.OperationPause); err != nil { - return nil, err - } - // FinalizePaused clears the WorkerAssignment the labels read, so snapshot - // them here, as crash.go does for the crash counter. - finalAttrs = lifecycleOpAttrs(actor, actorTemplate, "", wireFidelity) - var finalized *ateapipb.Actor - if finalized, err = w.ensurePausedFinalized(leaseCtx, actorRef); err != nil { - return nil, err - } - actor = finalized - return actor, nil -} - -// loadActorForPause fetches the current actor record and its template. -func (w *ActorWorkflow) loadActorForPause(ctx context.Context, actorRef resources.ActorRef) (_ *ateapipb.Actor, _ *ateapipb.ActorTemplate, err error) { - ctx, done := stepSpan(ctx, "LoadActorForPause") - defer func() { err = done(err) }() - - actor, err := w.store.GetActor(ctx, actorRef) - if err != nil { - return nil, nil, err - } - actorTemplate, err := resolveActorTemplate(ctx, w.store, actor) - if err != nil { - return nil, nil, err - } - return actor, actorTemplate, nil -} - -// ensureMarkedPausing transitions a RUNNING actor to SUSPENDING, minting the -// local snapshot name. A pause in progress is a SUSPENDING actor whose latest -// generation is a pending node-local snapshot; a suspend in progress has a -// pending durable one instead. Skips when a previous attempt already marked -// the actor; the persisted name then stays authoritative for the rest of the -// workflow. -func (w *ActorWorkflow) ensureMarkedPausing(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate) (_ *ateapipb.Actor, err error) { - ctx, done := stepSpan(ctx, "MarkPausing") - defer func() { err = done(err) }() - - if actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_SUSPENDING && latestSnapshotIsPendingLocal(actor.GetStatus()) { - markSkipped(ctx, "actor already SUSPENDING for a pause") - return actor, nil - } - // The pause edge only exists from RUNNING. A SUSPENDING actor that is not - // pausing is suspending, and that suspend owns it. - if actor.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_RUNNING { - return nil, apierror.FailedPrecondition("MarkPausing prerequisite not met for Actor: %s (got: %v, want %s)", actorRef, actor.GetStatus().GetState(), ateapipb.ActorState_ACTOR_STATE_RUNNING) - } - // By design a golden actor cannot be paused — it can only be suspended - // (committed). - if actorRef.Atespace == resources.GoldenActorAtespace { - return nil, apierror.FailedPrecondition("actors in atespace %q are golden actors, which cannot be paused", actorRef.Atespace) - } - - snapshotUUID := resources.NewSnapshotName() - storedActor, err := w.store.UpdateActor(ctx, actorRef, store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error { - toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_SUSPENDING - // Increment last_assigned_generation for the new Pause request. - gen := toUpdate.Status.LastAssignedGeneration + 1 - toUpdate.Status.LastAssignedGeneration = gen - snap := newLocalSnapshot( - gen, - actorTemplate.GetSnapshotConfig().GetPreferredFidelity(), - actorTemplate.GetMetadata().GetUid(), - snapshotUUID, - "", - ) - snap.SandboxRuntime = proto.CloneOf(toUpdate.GetStatus().GetWorkerAssignment().GetSandboxRuntime()) - toUpdate.Status.Snapshots = append(toUpdate.Status.Snapshots, snap) - return nil - }) - if err != nil { - if errors.Is(err, store.ErrVersionConflict) { - return nil, apierror.Aborted("concurrent update conflict, please retry") - } - return nil, err - } - logActorStateChanged(ctx, storedActor, ateattr.OperationPause) - return storedActor, nil -} - -// ensureAteletPaused checkpoints the workload locally on the worker node -// under the actor's persisted snapshot name. This is the atelet reentrancy -// seam (#372): the request is keyed by the actor UID, the worker pod UID, and -// the once-minted snapshot name, so a re-entered workflow re-sends the same -// semantic request; once atelet's Checkpoint is idempotent on those keys this -// step becomes fully reentrant with no changes here. -func (w *ActorWorkflow) ensureAteletPaused(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate) (wireFidelity string, err error) { - ctx, done := stepSpan(ctx, "CallAteletPause") - defer func() { err = done(err) }() - - assignment := actor.GetStatus().GetWorkerAssignment() - if assignment == nil { - // Missing active worker pod reference mid-pause indicates corrupted store state. - return "", crashActorAndFail(ctx, w.store, actorRef, ateattr.OperationPause, crashMessageWorkerAssignmentMissing) - } - - ateletConn, err := w.dialer.DialForAteletOnNode(assignment.GetNodeName()) - if err != nil { - return "", fmt.Errorf("while getting atelet conn for node %q: %w", assignment.GetNodeName(), err) - } - client := ateletpb.NewAteletClient(ateletConn) - - workloadSpec, err := workloadSpecFromActorTemplate(actorTemplate, actor, nil) - if err != nil { - return "", err - } - - // Checkpoint does not carry the sandbox config: atelet uses the version the - // actor is currently running (recorded on-node at Run/Restore) and pins it - // into the snapshot manifest. - inProgressSnap := snapshotAtLatestGeneration(actor.GetStatus()) - req := &ateletpb.CheckpointRequest{ - WorkerPodUid: assignment.GetWorkerPodUid(), - Atespace: actor.GetMetadata().GetAtespace(), - ActorName: actor.GetMetadata().GetName(), - ActorTemplateAtespace: actor.GetActorTemplate().GetAtespace(), - ActorTemplateName: actor.GetActorTemplate().GetName(), - Spec: workloadSpec, - Snapshot: &ateletpb.Snapshot{ - SnapshotUuid: inProgressSnap.GetUuid(), - Generation: inProgressSnap.GetGeneration(), - Fidelity: fidelityToAtelet(actorTemplate.GetSnapshotConfig().GetPreferredFidelity()), - }, - UploadConfig: &ateletpb.UploadConfig{ - Mode: ateletpb.UploadMode_UPLOAD_MODE_DEFERRED, - }, - ActorUid: actor.GetMetadata().Uid, - } - wireFidelity = ateattr.SnapshotFidelityValue(req.GetSnapshot().GetFidelity()) - - if _, err = client.Checkpoint(ctx, req); err != nil { - return wireFidelity, handleAteletError(ctx, w.store, actorRef, ateattr.OperationPause, "Checkpoint", false, err) - } - return wireFidelity, nil -} - -// ensurePausedFinalized releases the actor's worker (only when it is still -// owned by this actor), records where the local snapshot lives, and commits -// SUSPENDED with the assignment cleared in a single update. The actor keeps -// its assigned node: a paused actor is a SUSPENDED actor whose latest -// snapshot is node-local, and it resumes on that node. When the worker's -// node name was lost or the latest local snapshot entry is missing, the actor -// can never be resumed from local state, so it commits CRASHED instead and -// fails the step with FailedPrecondition. It re-reads the actor first so an -// out-of-band transition (e.g. the syncer crashing the actor after its worker -// died) is not overwritten: with no assignment left there is nothing to -// finalize. -func (w *ActorWorkflow) ensurePausedFinalized(ctx context.Context, actorRef resources.ActorRef) (_ *ateapipb.Actor, err error) { - ctx, done := stepSpan(ctx, "FinalizePaused") - defer func() { err = done(err) }() - - latestActor, err := w.store.GetActor(ctx, actorRef) - if err != nil { - return nil, err - } - - // 1. Free the worker (if it hasn't been freed yet) - if assignment := latestActor.GetStatus().GetWorkerAssignment(); assignment != nil { - if latestActor.GetStatus().GetAssignedNode() == "" { - // Without a node name we cannot record where the local snapshot lives, - // so the actor can never be resumed (the scheduler would search for a - // worker on an unknown node forever). Crash it instead of committing - // SUSPENDED with a snapshot nothing can find. - return nil, crashActorAndFail(ctx, w.store, actorRef, ateattr.OperationPause, crashMessageLocalSnapshotNodeUnknown) - } - if snapshotAtLatestGeneration(latestActor.GetStatus()) == nil { - return nil, crashActorAndFail(ctx, w.store, actorRef, ateattr.OperationPause, crashMessageLocalSnapshotMissing) - } - - worker, err := w.store.GetWorker(ctx, assignment.GetWorker().GetName()) - if err != nil { - if !errors.Is(err, store.ErrNotFound) { - return nil, fmt.Errorf("while getting worker for release: %w", err) - } - slog.Warn("Worker already gone during finalize pause, skipping release", "worker", assignment.GetWorkerPod()) - } else { - // Drop just this actor's assignment; any other actors the worker - // hosts keep theirs. - released, err := w.store.ReleaseActorFromWorker(ctx, worker.GetMetadata().GetName(), latestActor.GetMetadata().GetUid()) - if err != nil { - if errors.Is(err, store.ErrVersionConflict) { - return nil, apierror.Aborted("concurrent update conflict, please retry") - } - return nil, err - } - if w.workerCache != nil { - w.workerCache.Observe(released) - } - } - - // 2. Clear the actor's assignment, now that the worker is freed. The - // precondition pins the version checked above, so an out-of-band write - // since then fails the update instead of being overwritten. - storedActor, err := w.store.UpdateActor(ctx, actorRef, store.PreconditionFrom(latestActor), func(toUpdate *ateapipb.Actor) error { - toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_SUSPENDED - // Record the node locality of the local checkpoint and clear older local checkpoints. - snap := snapshotAtLatestGeneration(toUpdate.Status) - snap.Locality = toUpdate.GetStatus().GetAssignedNode() - pruneSnapshots(toUpdate.Status, snap) - toUpdate.Status.WorkerAssignment = nil - return nil - }) - if err != nil { - if errors.Is(err, store.ErrVersionConflict) { - return nil, apierror.Aborted("concurrent update conflict, please retry") - } - return nil, err - } - logActorStateChanged(ctx, storedActor, ateattr.OperationPause) - latestActor = storedActor - } - - return latestActor, nil -} diff --git a/cmd/ateapi/internal/controlapi/workflow_pause_test.go b/cmd/ateapi/internal/controlapi/workflow_pause_test.go index 76be4a5088..9b73ffc793 100644 --- a/cmd/ateapi/internal/controlapi/workflow_pause_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_pause_test.go @@ -17,15 +17,17 @@ package controlapi import ( "context" "fmt" + "strings" "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/cmd/ateapi/internal/workercache" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/protobuf/testing/protocmp" ) @@ -71,7 +73,7 @@ func TestPauseActor_Finalized_WorkerGone(t *testing.T) { if got.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_CRASHED { t.Errorf("state = %v, want CRASHED (node name unknown, cannot resume safely)", got.GetStatus().GetState()) } - if msg, want := got.GetStatus().GetCrash().GetMessage(), "pause failed: "+crashMessageLocalSnapshotNodeUnknown; msg != want { + if msg, want := got.GetStatus().GetCrash().GetMessage(), "suspend failed: "+crashMessageLocalSnapshotNodeUnknown; msg != want { t.Errorf("crash message = %q, want %q", msg, want) } if got.GetStatus().GetAssignedNode() != "" { @@ -92,6 +94,9 @@ func TestPauseActor_Finalized_WorkerGone(t *testing.T) { if got := (*records)[0].attrs[string(ateattr.ActorUIDKey)]; got == "" { t.Error("crash record carries no ate.actor.uid") } + if got := (*records)[0].attrs[string(ateattr.ActorOperationNameKey)]; got != ateattr.OperationSuspend { + t.Errorf("crash record %s = %q, want %q", ateattr.ActorOperationNameKey, got, ateattr.OperationSuspend) + } } // TestPauseActor_Finalized_AlreadyCrashed verifies that if the actor was @@ -118,10 +123,11 @@ func TestPauseActor_Finalized_AlreadyCrashed(t *testing.T) { }, }, }) - originalCrash := newActorCrash(ateattr.OperationPause, "original crash reason") + originalCrash := newActorCrash(ateattr.OperationSuspend, "original crash reason") updateDuringCheckpoint(t, atelet, st, actorRef, func(s *ateapipb.ActorStatus) { s.State = ateapipb.ActorState_ACTOR_STATE_CRASHED s.Crash = originalCrash + s.WorkerAssignment = nil }) if _, err := w.PauseActor(ctx, actorRef); apierror.Code(err) != codes.FailedPrecondition { @@ -168,7 +174,7 @@ func TestPauseActor_Finalized_KeepsCrashFromWorkerDelete(t *testing.T) { w, _ := newWireCaptureWorkflow(t, workerDeletedDuringReleaseStore{Interface: st}) _, err := w.PauseActor(ctx, actorRef) - assertConcurrentCrashKept(t, ctx, st, actorRef, err, "pause failed: "+crashMessageWorkerPodGone) + assertConcurrentCrashKept(t, ctx, st, actorRef, err, "suspend failed: "+crashMessageWorkerPodGone) } // TestPauseActor_Finalized_RecordsFidelity verifies pause records the template's @@ -193,6 +199,10 @@ func TestPauseActor_Finalized_RecordsFidelity(t *testing.T) { st := newTestPersistence(t) seedSuspendTemplate(t, ctx, st, &ateapipb.SnapshotConfig{PreferredFidelity: tc.preferred}) w, _ := newWireCaptureWorkflow(t, st) + // Without a watch running, finalization must notify the scheduler itself. + w.workerCache = workercache.New(nil, 0) + var observed *ateapipb.Worker + w.workerCache.AddHandler(func(worker *ateapipb.Worker) { observed = worker }) actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"} wantRuntime := &ateapipb.SandboxRuntime{ @@ -250,6 +260,16 @@ func TestPauseActor_Finalized_RecordsFidelity(t *testing.T) { if a := firstAssignment(t, st, wireTestAssignment().GetWorker().GetName()); a != nil { t.Errorf("worker assignment = %v, want released", a) } + if observed == nil { + t.Fatal("worker cache was not notified of the released capacity") + } + released, err := st.GetWorker(ctx, wireTestAssignment().GetWorker().GetName()) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(released, observed, protocmp.Transform()); diff != "" { + t.Errorf("cached worker differs from released worker (-want +got):\n%s", diff) + } }) } } @@ -291,7 +311,7 @@ func TestPauseActor_Finalized_MissingLocalSnapshotClearsAssignment(t *testing.T) if got.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_CRASHED { t.Errorf("state = %v, want CRASHED", got.GetStatus().GetState()) } - if msg, want := got.GetStatus().GetCrash().GetMessage(), "pause failed: "+crashMessageLocalSnapshotMissing; msg != want { + if msg, want := got.GetStatus().GetCrash().GetMessage(), "suspend failed: "+crashMessageLocalSnapshotMissing; msg != want { t.Errorf("crash message = %q, want %q", msg, want) } if got.GetStatus().GetWorkerAssignment() != nil { @@ -300,6 +320,9 @@ func TestPauseActor_Finalized_MissingLocalSnapshotClearsAssignment(t *testing.T) if len(*records) != 1 { t.Fatalf("got %d crash records, want 1", len(*records)) } + if got := (*records)[0].attrs[string(ateattr.ActorOperationNameKey)]; got != ateattr.OperationSuspend { + t.Errorf("crash record %s = %q, want %q", ateattr.ActorOperationNameKey, got, ateattr.OperationSuspend) + } } // TestPauseActorWorkflow_RejectedAndIdempotentPaths covers the two @@ -401,6 +424,10 @@ func TestPauseActor_MarkedPausing_StateMatrix(t *testing.T) { if got := apierror.Code(err); got != codes.FailedPrecondition { t.Errorf("state %v: code = %v, want FailedPrecondition (err: %v)", seedState, got, err) } + // Only a sync suspend accepts a paused actor, so a pause must not offer it. + if strings.Contains(err.Error(), "node-local snapshot") { + t.Errorf("state %v: err = %v, want no mention of a node-local snapshot", seedState, err) + } if got := stored.GetStatus().GetState(); got != seedState { t.Errorf("state %v: stored state = %v, want it untouched", seedState, got) } @@ -545,7 +572,7 @@ func TestPauseActor_CrashesWhenPausingActorMissingWorkerPod(t *testing.T) { if got.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_CRASHED { t.Errorf("stored state = %v, want %v", got.GetStatus().GetState(), ateapipb.ActorState_ACTOR_STATE_CRASHED) } - if msg, want := got.GetStatus().GetCrash().GetMessage(), "pause failed: "+crashMessageWorkerAssignmentMissing; msg != want { + if msg, want := got.GetStatus().GetCrash().GetMessage(), "suspend failed: "+crashMessageWorkerAssignmentMissing; msg != want { t.Errorf("crash message = %q, want %q", msg, want) } } diff --git a/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments.go b/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments.go index 6116c60591..fc101af121 100644 --- a/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments.go +++ b/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments.go @@ -19,10 +19,10 @@ import ( "errors" "fmt" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // needsAssignmentReconcile reports whether worker's epoch has risen past the diff --git a/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go b/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go index f70e5f76ea..20c9b04e9f 100644 --- a/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go @@ -20,13 +20,13 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/scheduling" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/scheduling" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/cmd/ateapi/internal/workercache" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" ) diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index cb82726ec9..a28938a7be 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -22,16 +22,16 @@ import ( "slices" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/scheduling" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/hardware" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/volume" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/scheduling" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/hardware" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/volume" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/labels" @@ -508,7 +508,7 @@ func (w *ActorWorkflow) assignWorkerAttempt(ctx context.Context, actorRef resour if assignedWorker == nil { pickedWorker, err := w.scheduler.Schedule(ctx, constraints) if errors.Is(err, scheduling.ErrNoCapacity) && relaxable { - constraints.RequiredRuntime = nil + constraints.RequiredRuntime = hardware.Requirement{} pickedWorker, err = w.scheduler.Schedule(ctx, constraints) } if err != nil { @@ -521,10 +521,10 @@ func (w *ActorWorkflow) assignWorkerAttempt(ctx context.Context, actorRef resour assignedWorker = pickedWorker slog.InfoContext(ctx, "Picked worker", slog.Any("worker", pickedWorker.String())) - } else if relaxable && !hardware.WorkerMatches(assignedWorker.GetStatus(), snapRuntime) { + } else if relaxable && !snapRuntime.WorkerMatches(assignedWorker.GetStatus()) { // The stale claim only passed the relaxed check, so bind under it: // admit asks the same question again under the row lock. - constraints.RequiredRuntime = nil + constraints.RequiredRuntime = hardware.Requirement{} } assignment := &ateapipb.ActorAssignment{ @@ -601,7 +601,7 @@ func (w *ActorWorkflow) assignWorkerAttempt(ctx context.Context, actorRef resour // attempt re-runs scheduling. var errWorkerFilledUp = errors.New("picked worker no longer has room") -func workerAssignmentFrom(w *ateapipb.Worker, snapRuntime *ateapipb.SandboxRuntime) *ateapipb.WorkerAssignment { +func workerAssignmentFrom(w *ateapipb.Worker, snapRuntime hardware.Requirement) *ateapipb.WorkerAssignment { return &ateapipb.WorkerAssignment{ Worker: &ateapipb.ObjectRef{Name: w.GetMetadata().GetName()}, WorkerNamespace: w.GetWorkerNamespace(), @@ -610,19 +610,21 @@ func workerAssignmentFrom(w *ateapipb.Worker, snapRuntime *ateapipb.SandboxRunti WorkerPodUid: w.GetWorkerPodUid(), WorkerPodIps: slices.Clone(w.GetIps()), NodeName: w.GetNodeName(), + WorkerSpiffeId: w.GetWorkerSpiffeId(), // The runtime the actor is started or restored under. Snapshots taken // during this assignment are stamped with it, so a worker that later // changes its default does not relabel state captured on the old one. // - // TODO: A restorable_runtimes match records the old runtime here, but - // resolveSandboxAssets still ships the SandboxConfig's default version - // and atelet restores with the assets in the request, so the restore - // runs on the new version: it fails, or the next checkpoint is stamped - // with a runtime that did not produce it. Resolve the assets for the - // matched version once SandboxConfig can name them, or treat a - // restorable-only match as incompatible until it can. Latent for now: - // worker reports no restorable_runtimes (internal/ateworker/register.go). - SandboxRuntime: proto.CloneOf(hardware.RuntimeFor(w.GetStatus(), snapRuntime)), + // TODO: Workers report restorable_runtimes, so a match on one records + // that older runtime here, but resolveSandboxAssets still ships the + // template's SandboxConfig default version and atelet restores with + // the assets in the request: the restore runs on the newer version and + // fails, or its next checkpoint is stamped with a runtime that did not + // produce it. A follow-up replaces the assets in Run and Restore with + // this sandbox_runtime, which atelet resolves to the matching version + // of the worker pool's SandboxConfig; until it lands, keep a config's + // default version the only enabled one. + SandboxRuntime: proto.CloneOf(snapRuntime.RuntimeFor(w.GetStatus())), } } @@ -659,7 +661,7 @@ func schedulingConstraints(actor *ateapipb.Actor, tmpl *ateapipb.ActorTemplate) } c := scheduling.Constraints{ SandboxClass: tmpl.GetSandboxConfig().GetSandboxClass(), - RequiredRuntime: requiredSnapshotRuntime(actor, tmpl), + RequiredRuntime: hardware.NewRequirement(requiredSnapshotRuntime(actor, tmpl)), ActorSelector: labels.SelectorFromSet(labels.Set(actor.GetWorkerSelector().GetMatchLabels())), RequiredNodes: requiredNodes, Limits: limits.Proto(), @@ -673,6 +675,10 @@ func schedulingConstraints(actor *ateapipb.Actor, tmpl *ateapipb.ActorTemplate) // requiredSnapshotRuntime returns the SandboxRuntime stamped on the snapshot // that ResumeActor will restore guest state from, or nil when the resume does // not restore guest state (cold boot, VOLUMES fidelity, or replaced template). +// +// TODO: Once gVisor can restore ROOTFS snapshots, return nil for ROOTFS as +// well: a rootfs upper is files tied to the image, not to the runtime, so only +// MEMORY needs a runtime match. Until then ROOTFS is pinned like MEMORY. func requiredSnapshotRuntime(actor *ateapipb.Actor, tmpl *ateapipb.ActorTemplate) *ateapipb.SandboxRuntime { if localSnap := findLatestLocalSnapshot(actor.GetStatus()); localSnap != nil { if localSnap.GetFidelity() == ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES { @@ -700,7 +706,7 @@ func requiredSnapshotRuntime(actor *ateapipb.Actor, tmpl *ateapipb.ActorTemplate // fidelity downgrade, and nothing invalidates the checkpoint files if the actor // boots elsewhere. Suspending the actor uploads it and lifts the pin. func canRelaxRuntime(actor *ateapipb.Actor, constraints scheduling.Constraints) bool { - if constraints.RequiredRuntime == nil { + if constraints.RequiredRuntime.Runtime() == nil { return false } return findLatestLocalSnapshot(actor.GetStatus()) == nil @@ -710,11 +716,11 @@ func canRelaxRuntime(actor *ateapipb.Actor, constraints scheduling.Constraints) // relaxable — satisfies constraints with RequiredRuntime relaxed. func (w *ActorWorkflow) appliesWithFallback(worker *ateapipb.Worker, constraints scheduling.Constraints, relaxable bool) bool { relaxed := constraints - relaxed.RequiredRuntime = nil + relaxed.RequiredRuntime = hardware.Requirement{} if !w.scheduler.Applies(worker, relaxed) { return false } - return relaxable || hardware.WorkerMatches(worker.GetStatus(), constraints.RequiredRuntime) + return relaxable || constraints.RequiredRuntime.WorkerMatches(worker.GetStatus()) } // ensureVolumesAttached attaches the actor's mounted external volumes to the @@ -832,6 +838,10 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou if runtimeIncompatible { logRuntimeFallback(ctx, actorRef, assignment, src.RequiredRuntime) } + // TODO: Once gVisor can restore ROOTFS snapshots, drop an incompatible + // runtime to ROOTFS, not VOLUMES: only the memory layer is invalid. A + // replaced template still drops to VOLUMES, as the rootfs upper is + // only valid over the image it was written on. scope = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES } tele.WireFidelity = ateattr.SnapshotFidelityValue(scope) diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index 3b2bdf7a7b..a355de1958 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -23,20 +23,21 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/scheduling" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/volume" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/cmd/ateapi/internal/scheduling" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/cmd/ateapi/internal/workercache" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/hardware" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/volume" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/pkg/proto/ateapipb" sdkmetric "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/metricdata" "google.golang.org/grpc" @@ -2367,7 +2368,7 @@ type relaxingTestScheduler struct { } func (s *relaxingTestScheduler) Schedule(_ context.Context, c scheduling.Constraints) (*ateapipb.Worker, error) { - if c.RequiredRuntime != nil { + if c.RequiredRuntime.Runtime() != nil { return nil, scheduling.ErrNoCapacity } return s.relaxedWorker, nil @@ -2683,9 +2684,19 @@ func TestCanRelaxRuntime(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { actor := &ateapipb.Actor{Status: &ateapipb.ActorStatus{Snapshots: tc.snapshots}} - if got := canRelaxRuntime(actor, scheduling.Constraints{RequiredRuntime: tc.required}); got != tc.want { + if got := canRelaxRuntime(actor, scheduling.Constraints{RequiredRuntime: hardware.NewRequirement(tc.required)}); got != tc.want { t.Errorf("canRelaxRuntime = %v, want %v", got, tc.want) } }) } } + +// atenet-router requires the assignment's SPIFFE ID on atunnel's certificate, +// so the worker's has to reach it. +func TestWorkerAssignmentFromCopiesSPIFFEID(t *testing.T) { + const id = "spiffe://cluster.local/ns/workers/sa/substrate-worker" + got := workerAssignmentFrom(&ateapipb.Worker{WorkerSpiffeId: id}, hardware.Requirement{}) + if got.GetWorkerSpiffeId() != id { + t.Errorf("worker SPIFFE ID = %q, want %q", got.GetWorkerSpiffeId(), id) + } +} diff --git a/cmd/ateapi/internal/controlapi/workflow_revert.go b/cmd/ateapi/internal/controlapi/workflow_revert.go index 696a0d3799..ddc7e02552 100644 --- a/cmd/ateapi/internal/controlapi/workflow_revert.go +++ b/cmd/ateapi/internal/controlapi/workflow_revert.go @@ -22,12 +22,12 @@ import ( "slices" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel/attribute" ) diff --git a/cmd/ateapi/internal/controlapi/workflow_revert_test.go b/cmd/ateapi/internal/controlapi/workflow_revert_test.go index ecf052fb4f..32c65a2d70 100644 --- a/cmd/ateapi/internal/controlapi/workflow_revert_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_revert_test.go @@ -18,10 +18,10 @@ import ( "context" "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" ) diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend.go b/cmd/ateapi/internal/controlapi/workflow_suspend.go index b2655daa16..1b1d4b0d22 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend.go @@ -21,24 +21,61 @@ import ( "log/slog" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel/attribute" "google.golang.org/protobuf/proto" ) +// suspendMode selects which of the two suspends the suspend workflow runs. +// Both take the same steps; the mode only decides the shape of the snapshot +// they take. +type suspendMode int + +const ( + // suspendSync makes the snapshot durable before the workflow returns: + // the actor ends SUSPENDED holding a durable snapshot (SuspendActor). + suspendSync suspendMode = iota + // suspendAsync leaves the snapshot on the worker's node, to be uploaded + // later: the actor ends SUSPENDED holding a node-local snapshot and + // pinned to that node (PauseActor). Nothing uploads it yet. + suspendAsync +) + +// uploadMode is when atelet makes the checkpoint durable. +func (m suspendMode) uploadMode() ateletpb.UploadMode { + if m == suspendAsync { + return ateletpb.UploadMode_UPLOAD_MODE_DEFERRED + } + return ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE +} + // SuspendActor executes the workflow to suspend a running or paused actor: // a running actor is checkpointed on its worker, a paused actor's node-local // snapshot is uploaded. Idempotent: a re-entered workflow fast-forwards past // the steps a previous attempt completed, deriving progress from the // persisted actor alone. -func (w *ActorWorkflow) SuspendActor(ctx context.Context, actorRef resources.ActorRef) (_ *ateapipb.Actor, err error) { +func (w *ActorWorkflow) SuspendActor(ctx context.Context, actorRef resources.ActorRef) (*ateapipb.Actor, error) { + ctx = preview.ContextWithOptIn(ctx) + return w.suspendActor(ctx, actorRef, suspendSync) +} + +// PauseActor executes the workflow to pause a running actor: it is +// checkpointed on its worker and the snapshot is kept on that node. +// Idempotent: a re-entered workflow fast-forwards past the steps a previous +// attempt completed, deriving progress from the persisted actor alone. +func (w *ActorWorkflow) PauseActor(ctx context.Context, actorRef resources.ActorRef) (*ateapipb.Actor, error) { ctx = preview.ContextWithOptIn(ctx) + return w.suspendActor(ctx, actorRef, suspendAsync) +} + +// suspendActor runs the suspend workflow behind SuspendActor and PauseActor. +func (w *ActorWorkflow) suspendActor(ctx context.Context, actorRef resources.ActorRef, mode suspendMode) (_ *ateapipb.Actor, err error) { start := time.Now() var actor *ateapipb.Actor var actorTemplate *ateapipb.ActorTemplate @@ -65,31 +102,39 @@ func (w *ActorWorkflow) SuspendActor(ctx context.Context, actorRef resources.Act if err != nil { return nil, err } - if actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_SUSPENDED && !latestSnapshotIsLocalOnly(actor.GetStatus()) { - // Fully suspended already: FinalizeSuspended commits SUSPENDED and the - // cleared worker assignment in a single update, so there is nothing - // left to do. A SUSPENDED actor whose latest snapshot is node-local is - // paused, not suspended, and goes on to upload it. This success - // reports no pool, and cannot: the previous attempt released the - // worker, so the record names none (#957). - return actor, nil + // FinalizeSuspended commits SUSPENDED and the cleared worker assignment in + // a single update, so an actor already holding the snapshot this mode + // leaves behind has nothing left to do. These successes report no pool, + // and cannot: the previous attempt released the worker, so the record + // names none (#957). + if actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_SUSPENDED { + if mode == suspendAsync && latestSnapshotIsLocalOnly(actor.GetStatus()) { + return actor, nil // fully paused + } + if mode == suspendSync && !latestSnapshotIsLocalOnly(actor.GetStatus()) { + return actor, nil // fully suspended + } + // A sync suspend of a paused actor goes on to upload its node-local + // snapshot. An async suspend of a suspended actor is rejected by MarkSuspending. } // Decided before marking: once SUSPENDING is committed, the loaded status // alone can no longer tell the two origins apart. - fromPaused := isPausedOriginSuspend(actor) + // TODO(paused-suspend): remove the paused -> suspended transition before GA. + fromPaused := mode == suspendSync && isPausedOriginSuspend(actor) var marked *ateapipb.Actor - if marked, err = w.ensureMarkedSuspending(leaseCtx, actorRef, actor, actorTemplate); err != nil { + if marked, err = w.ensureMarkedSuspending(leaseCtx, actorRef, actor, actorTemplate, mode); err != nil { return nil, err } actor = marked if fromPaused { wireFidelity, err = w.ensurePausedSnapshotUploaded(leaseCtx, actorRef, actor, actorTemplate) } else { - wireFidelity, err = w.ensureAteletSuspended(leaseCtx, actorRef, actor, actorTemplate) + wireFidelity, err = w.ensureAteletSuspended(leaseCtx, actorRef, actor, actorTemplate, mode) } if err != nil { return nil, err } + // TODO(#2734): An async suspend should skip detaching here, since the actor can resume on the same node. if err = w.ensureVolumesDetached(leaseCtx, actor, actorTemplate, "DetachVolumes", ateattr.OperationSuspend); err != nil { return nil, err } @@ -97,7 +142,7 @@ func (w *ActorWorkflow) SuspendActor(ctx context.Context, actorRef resources.Act // them here, as crash.go does for the crash counter. finalAttrs = lifecycleOpAttrs(actor, actorTemplate, "", wireFidelity) var finalized *ateapipb.Actor - if finalized, err = w.ensureSuspendedFinalized(leaseCtx, actorRef); err != nil { + if finalized, err = w.ensureSuspendedFinalized(leaseCtx, actorRef, mode); err != nil { return nil, err } actor = finalized @@ -121,28 +166,44 @@ func (w *ActorWorkflow) loadActorForSuspend(ctx context.Context, actorRef resour } // ensureMarkedSuspending transitions a RUNNING or paused actor to -// SUSPENDING, minting the in-progress snapshot location and recording the -// actor version the snapshot will capture. Skips when a previous attempt -// already marked the actor; the persisted location and source version then -// stay authoritative for the rest of the workflow. -func (w *ActorWorkflow) ensureMarkedSuspending(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate) (_ *ateapipb.Actor, err error) { +// SUSPENDING, minting the in-progress snapshot and recording the actor +// version the snapshot will capture. A sync suspend mints a pending durable +// snapshot; an async one mints a node-local snapshot with no locality yet. +// Skips when a previous attempt of the same mode already marked the actor; +// the persisted snapshot and source version then stay authoritative for the +// rest of the workflow. A mark left by the other mode is rejected: that +// workflow owns the actor until it finishes. +func (w *ActorWorkflow) ensureMarkedSuspending(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate, mode suspendMode) (_ *ateapipb.Actor, err error) { ctx, done := stepSpan(ctx, "MarkSuspending") defer func() { err = done(err) }() if actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_SUSPENDING { - if latestSnapshotIsPendingLocal(actor.GetStatus()) { - // The pause workflow owns the actor until it finishes; a suspend - // can run once the actor is paused. - return nil, apierror.FailedPrecondition("MarkSuspending prerequisite not met for Actor: %s (a pause is in progress)", actorRef) + // An async mark's latest snapshot is node-local with no locality yet, + // a sync mark's has a pending durable copy. + if latestSnapshotIsPendingLocal(actor.GetStatus()) != (mode == suspendAsync) { + inProgress := "Pause" + if mode == suspendAsync { + inProgress = "Suspend" + } + return nil, apierror.FailedPrecondition("MarkSuspending prerequisite not met for Actor: %s (a previous %s is in progress)", actorRef, inProgress) } markSkipped(ctx, "actor already SUSPENDING") return actor, nil } state := actor.GetStatus().GetState() - paused := isPausedOriginSuspend(actor) + // Only a sync suspend takes a paused actor. + paused := mode == suspendSync && isPausedOriginSuspend(actor) // TODO(paused-suspend) if state != ateapipb.ActorState_ACTOR_STATE_RUNNING && !paused { + if mode == suspendAsync { + return nil, apierror.FailedPrecondition("MarkSuspending prerequisite not met for Actor: %s (got: %v, want %s)", actorRef, state, ateapipb.ActorState_ACTOR_STATE_RUNNING) + } return nil, apierror.FailedPrecondition("MarkSuspending prerequisite not met for Actor: %s (got: %v, want %s or %s with a node-local snapshot)", actorRef, state, ateapipb.ActorState_ACTOR_STATE_RUNNING, ateapipb.ActorState_ACTOR_STATE_SUSPENDED) } + // Golden actors are only suspended, never paused. + // TODO: Remove this check: the suspend workflow should not special-case golden actors. + if mode == suspendAsync && actorRef.Atespace == resources.GoldenActorAtespace { + return nil, apierror.FailedPrecondition("actors in atespace %q are golden actors, which cannot be paused", actorRef.Atespace) + } // A paused actor's durable snapshot is an upload of its local checkpoint, // so it keeps that checkpoint's name; a running actor's is a new one. var snapshotName string @@ -154,10 +215,14 @@ func (w *ActorWorkflow) ensureMarkedSuspending(ctx context.Context, actorRef res } // Fail here rather than at checkpoint time if the template's location - // cannot produce a usable URI: nothing has been written yet. - uri, err := newInProgressSnapshotURI(actorTemplate, actor, snapshotName) - if err != nil { - return nil, err + // cannot produce a usable URI: nothing has been written yet. An async + // suspend keeps its snapshot on the node, so it needs no URI. + var uri resources.SnapshotURI + if mode == suspendSync { + // TODO: Always populate the in-progress URI. + if uri, err = newInProgressSnapshotURI(actorTemplate, actor, snapshotName); err != nil { + return nil, err + } } storedActor, err := w.store.UpdateActor(ctx, actorRef, store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error { toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_SUSPENDING @@ -176,18 +241,25 @@ func (w *ActorWorkflow) ensureMarkedSuspending(ctx context.Context, actorRef res return nil } // Increment last_assigned_generation for the new Suspend request and create - // a pending snapshot for that generation. + // a pending snapshot for that generation: durable for a sync suspend, + // node-local with no locality until finalize for an async one. gen := toUpdate.Status.LastAssignedGeneration + 1 toUpdate.Status.LastAssignedGeneration = gen - snap := newDurableSnapshot( - gen, - ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, - preferredFidelity(actorRef.Atespace, actorTemplate), - actorTemplate.GetMetadata().GetUid(), - uri.Name(), - uri.String(), - ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING, - ) + + fidelity := preferredFidelity(actorRef.Atespace, actorTemplate) + templateUID := actorTemplate.GetMetadata().GetUid() + snap := newLocalSnapshot(gen, fidelity, templateUID, snapshotName, "") + if mode == suspendSync { + snap = newDurableSnapshot( + gen, + ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, + fidelity, + templateUID, + uri.Name(), + uri.String(), + ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING, + ) + } snap.SandboxRuntime = proto.CloneOf(toUpdate.GetStatus().GetWorkerAssignment().GetSandboxRuntime()) toUpdate.Status.Snapshots = append(toUpdate.Status.Snapshots, snap) return nil @@ -213,6 +285,7 @@ func preferredFidelity(atespace string, tmpl *ateapipb.ActorTemplate) ateapipb.S return tmpl.GetSnapshotConfig().GetPreferredFidelity() } +// TODO(paused-suspend): remove with the paused -> suspended transition. // isPausedOriginSuspend reports whether the suspend must upload a paused // actor's node-local snapshot instead of checkpointing a running workload. // A local snapshot alone does not mean paused-origin: resume restores from it @@ -232,12 +305,14 @@ func isPausedOriginSuspend(actor *ateapipb.Actor) bool { } // ensureAteletSuspended checkpoints the workload to the actor's persisted -// in-progress snapshot location. This is the atelet reentrancy seam (#372): +// in-progress snapshot location. A sync suspend has atelet upload it before +// returning; an async suspend has atelet keep it on the node. +// This is the atelet reentrancy seam (#372): // the request is keyed by the actor UID, the worker pod UID, and the // once-minted snapshot location, so a re-entered workflow re-sends the same // semantic request; once atelet's Checkpoint is idempotent on those keys this // step becomes fully reentrant with no changes here. -func (w *ActorWorkflow) ensureAteletSuspended(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate) (wireFidelity string, err error) { +func (w *ActorWorkflow) ensureAteletSuspended(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate, mode suspendMode) (wireFidelity string, err error) { ctx, done := stepSpan(ctx, "CallAteletSuspend") defer func() { err = done(err) }() @@ -263,6 +338,11 @@ func (w *ActorWorkflow) ensureAteletSuspended(ctx context.Context, actorRef reso // into the snapshot manifest. inProgressSnap := snapshotAtLatestGeneration(actor.GetStatus()) fidelity := fidelityToAtelet(preferredFidelity(actor.GetMetadata().GetAtespace(), actorTemplate)) + // An async suspend's snapshot has no durable copy, so it stays on the node. + var object *ateletpb.ObjectStorage + if uri := inProgressSnap.GetDurableSnapshot().GetObject().GetSnapshotUri(); uri != "" { + object = &ateletpb.ObjectStorage{SnapshotUri: uri} + } req := &ateletpb.CheckpointRequest{ WorkerPodUid: assignment.GetWorkerPodUid(), Atespace: actor.GetMetadata().GetAtespace(), @@ -274,16 +354,10 @@ func (w *ActorWorkflow) ensureAteletSuspended(ctx context.Context, actorRef reso SnapshotUuid: inProgressSnap.GetUuid(), Generation: inProgressSnap.GetGeneration(), Fidelity: fidelity, - Object: &ateletpb.ObjectStorage{ - SnapshotUri: inProgressSnap.GetDurableSnapshot().GetObject().GetSnapshotUri(), - }, + Object: object, }, - // FinalizeSuspended marks this snapshot COMPLETED, so it must be - // durable before Checkpoint returns. IMMEDIATE temporarily keeps no - // local copy either (see UPLOAD_MODE_IMMEDIATE), which matches - // FinalizeSuspended dropping the actor's local snapshots. UploadConfig: &ateletpb.UploadConfig{ - Mode: ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE, + Mode: mode.uploadMode(), }, ActorUid: actor.GetMetadata().Uid, } @@ -295,6 +369,7 @@ func (w *ActorWorkflow) ensureAteletSuspended(ctx context.Context, actorRef reso return wireFidelity, nil } +// TODO(paused-suspend): remove with the paused -> suspended transition. // ensurePausedSnapshotUploaded suspends a paused actor by telling the atelet // on the node holding the local pause snapshot to upload it to the actor's // persisted in-progress snapshot location; no workload runs, so there is no @@ -311,8 +386,8 @@ func (w *ActorWorkflow) ensurePausedSnapshotUploaded(ctx context.Context, actorR } nodeName := actor.GetStatus().GetAssignedNode() if nodeName == "" { - // Without the node the snapshot can never be found (mirrors - // FinalizePaused, which crashes rather than record an unknown node). + // Without the node the snapshot can never be found (mirrors an async + // FinalizeSuspended, which crashes rather than record an unknown node). return "", crashActorAndFail(ctx, w.store, actorRef, ateattr.OperationSuspend, crashMessageLocalSnapshotNodeUnknown) } @@ -363,7 +438,7 @@ func newInProgressSnapshotURI(actorTemplate *ateapipb.ActorTemplate, actor *atea // ensureVolumesDetached detaches the actor's mounted external volumes from // its worker node. Detachment is idempotent, so a re-entered workflow safely -// runs it again. spanName distinguishes the suspend and pause steps in +// runs it again. spanName distinguishes the suspend and revert steps in // traces; op labels the volume metrics. // TODO replace re-execution with a proper check on the volumes' attach state. func (w *ActorWorkflow) ensureVolumesDetached(ctx context.Context, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate, spanName, op string) (err error) { @@ -380,14 +455,16 @@ func (w *ActorWorkflow) ensureVolumesDetached(ctx context.Context, actor *ateapi // ensureSuspendedFinalized releases the actor's worker (only when it is still // owned by this actor), records the in-progress snapshot as the actor's -// durable snapshot, and commits SUSPENDED with the assignment cleared in a -// single update, then releases the durable snapshot that update replaced. -// When the latest snapshot has no durable snapshot to record, it commits -// CRASHED instead and fails the step. +// durable snapshot (sync) or node-local snapshot (async), and commits +// SUSPENDED with the assignment cleared in a single update, then releases the +// durable snapshot that update replaced. +// When the latest snapshot cannot be recorded, it commits CRASHED instead and +// fails the step. // It re-reads the actor first so an out-of-band transition (e.g. the syncer -// crashing the actor after its worker died) is not overwritten: with no -// assignment left there is nothing to finalize. -func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef resources.ActorRef) (_ *ateapipb.Actor, err error) { +// crashing the actor after its worker died) is not overwritten: an actor that +// is no longer SUSPENDING is left as it is and the step fails with +// FailedPrecondition. +func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef resources.ActorRef, mode suspendMode) (_ *ateapipb.Actor, err error) { ctx, done := stepSpan(ctx, "FinalizeSuspended") defer func() { err = done(err) }() @@ -413,21 +490,38 @@ func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef r if err != nil { return nil, err } - if snapshotAtLatestGeneration(latestActor.GetStatus()).GetDurableSnapshot() == nil { - return nil, crashActorAndFail(ctx, w.store, actorRef, ateattr.OperationSuspend, crashMessageDurableSnapshotMissing) + if state := latestActor.GetStatus().GetState(); state != ateapipb.ActorState_ACTOR_STATE_SUSPENDING { + return nil, apierror.FailedPrecondition("FinalizeSuspended prerequisite not met for Actor: %s (got: %v, want %s)", actorRef, state, ateapipb.ActorState_ACTOR_STATE_SUSPENDING) + } + switch mode { + case suspendSync: + if snapshotAtLatestGeneration(latestActor.GetStatus()).GetDurableSnapshot() == nil { + return nil, crashActorAndFail(ctx, w.store, actorRef, ateattr.OperationSuspend, crashMessageDurableSnapshotMissing) + } + case suspendAsync: + // A local snapshot on an unknown node can never be found by the scheduler. + if latestActor.GetStatus().GetAssignedNode() == "" { + return nil, crashActorAndFail(ctx, w.store, actorRef, ateattr.OperationSuspend, crashMessageLocalSnapshotNodeUnknown) + } + if snapshotAtLatestGeneration(latestActor.GetStatus()) == nil { + return nil, crashActorAndFail(ctx, w.store, actorRef, ateattr.OperationSuspend, crashMessageLocalSnapshotMissing) + } } // 1. Free the worker (if it hasn't been freed yet) if latestActor.GetStatus().GetWorkerAssignment() != nil { t = time.Now() - _, _, err := releaseWorker(ctx, w.store, latestActor) + _, released, err := releaseWorker(ctx, w.store, latestActor) dReleaseWorker = time.Since(t) if err != nil { return nil, err } + if w.workerCache != nil && released != nil { + w.workerCache.Observe(released) + } } - // 2. Finalize the actor: record its new durable snapshot and mark it SUSPENDED. This + // 2. Finalize the actor: record its new snapshot and mark it SUSPENDED. This // must run even with no worker assignment (nothing to free), or the actor // would be left SUSPENDING forever with the workflow reporting success. // The precondition pins the version checked above, so an out-of-band write @@ -435,14 +529,22 @@ func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef r t = time.Now() storedActor, err := w.store.UpdateActor(ctx, actorRef, store.PreconditionFrom(latestActor), func(toUpdate *ateapipb.Actor) error { toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_SUSPENDED - inProgressSt := snapshotAtLatestGeneration(toUpdate.Status).GetDurableSnapshot() - inProgressSt.Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED - // Remove all local snapshots since they are superseded by the new durable snapshot. - // TODO: This will change when we merge pause and suspend and allow for both local - // and durable snapshots to be kept. - pruneSnapshots(toUpdate.Status, nil) + snap := snapshotAtLatestGeneration(toUpdate.Status) + switch mode { + case suspendSync: + snap.GetDurableSnapshot().Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + // Remove all local snapshots since they are superseded by the new durable snapshot. + // TODO: This will change when an async suspend uploads its snapshot + // and both local and durable snapshots are kept. + pruneSnapshots(toUpdate.Status, nil) + toUpdate.Status.AssignedNode = "" + case suspendAsync: + // Record the node locality of the local checkpoint and clear older local checkpoints. + // AssignedNode is kept: a paused actor resumes on that node. + snap.Locality = toUpdate.GetStatus().GetAssignedNode() + pruneSnapshots(toUpdate.Status, snap) + } toUpdate.Status.WorkerAssignment = nil - toUpdate.Status.AssignedNode = "" return nil }) dUpdateActor = time.Since(t) diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go index 8bed842237..46d8e0eb88 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go @@ -21,15 +21,16 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/cmd/ateapi/internal/workercache" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/testing/protocmp" @@ -821,6 +822,10 @@ func TestSuspendActor_Finalized_ReleasesOnlyOwnWorker(t *testing.T) { persistence := newTestPersistence(t) seedSuspendTemplate(t, ctx, persistence, &ateapipb.SnapshotConfig{StorageLocation: testStorageLocation}) w, _ := newWireCaptureWorkflow(t, persistence) + // Without a watch running, finalization must notify the scheduler itself. + w.workerCache = workercache.New(nil, 0) + var observed *ateapipb.Worker + w.workerCache.AddHandler(func(worker *ateapipb.Worker) { observed = worker }) actor := &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "shared"}, ActorTemplate: suspendTestTemplateRef(), @@ -853,6 +858,23 @@ func TestSuspendActor_Finalized_ReleasesOnlyOwnWorker(t *testing.T) { if released := stored == nil; released != tt.wantReleased { t.Errorf("worker released = %t, want %t (assignment: %v)", released, tt.wantReleased, stored) } + if !tt.wantReleased { + if observed != nil { + t.Error("worker cache notified even though no assignment was released") + } + return + } + + if observed == nil { + t.Fatal("worker cache was not notified of the released capacity") + } + released, err := persistence.GetWorker(ctx, wireTestAssignment().GetWorker().GetName()) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(released, observed, protocmp.Transform()); diff != "" { + t.Errorf("cached worker differs from released worker (-want +got):\n%s", diff) + } }) } } @@ -1315,18 +1337,10 @@ func TestSuspendAndPause_CharacteristicMatrix(t *testing.T) { } } -// TestSuspendAndPause_FinalizeWithoutAssignment pins where the two finalize -// steps differ when the DeleteWorker workflow crashes the actor, clearing its -// worker assignment, between the atelet call and finalize. Both sides pin -// bugs tracked in #2676: the suspend commits SUSPENDED over the crash, and -// the pause keeps the crash but returns OK for an actor left CRASHED. The fix -// changes the sync wantState to CRASHED and both codes to FailedPrecondition, -// replacing the err != nil fatal below. +// TestSuspendAndPause_FinalizeWithoutAssignment verifies that both workflows +// preserve an out-of-band crash and return FailedPrecondition when the crash +// clears the worker assignment between the atelet call and finalize's read. func TestSuspendAndPause_FinalizeWithoutAssignment(t *testing.T) { - wantState := map[string]ateapipb.ActorState{ - "sync": ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - "async": ateapipb.ActorState_ACTOR_STATE_CRASHED, - } for _, mode := range suspendModes { t.Run(mode.name, func(t *testing.T) { ctx := context.Background() @@ -1343,21 +1357,29 @@ func TestSuspendAndPause_FinalizeWithoutAssignment(t *testing.T) { WorkerAssignment: wireTestAssignment(), }, }) + // What a worker delete or the syncer commits: CRASHED with the + // crash recorded and the assignment cleared, landing after the + // atelet call and before finalize re-reads the actor. + originalCrash := newActorCrash(ateattr.OperationDelete, "original crash reason") updateDuringCheckpoint(t, atelet, persistence, actorRef, func(s *ateapipb.ActorStatus) { s.State = ateapipb.ActorState_ACTOR_STATE_CRASHED + s.Crash = originalCrash s.WorkerAssignment = nil }) returned, err := mode.run(w, ctx, actorRef) - // #2676: both modes should return FailedPrecondition here. - if err != nil { - t.Fatalf("%s: %v", mode.name, err) + if got := apierror.Code(err); got != codes.FailedPrecondition { + t.Fatalf("%s: code = %v (err %v), want FailedPrecondition", mode.name, got, err) + } + if returned != nil { + t.Errorf("%s: returned actor = %v, want nil", mode.name, returned) } - if got := returned.GetStatus().GetState(); got != wantState[mode.name] { - t.Errorf("returned state = %v, want %v", got, wantState[mode.name]) + stored := mustReadActor(t, ctx, persistence, actorRef) + if got := stored.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_CRASHED { + t.Errorf("%s: stored state = %v, want CRASHED", mode.name, got) } - if got := mustReadActor(t, ctx, persistence, actorRef).GetStatus().GetState(); got != wantState[mode.name] { - t.Errorf("stored state = %v, want %v", got, wantState[mode.name]) + if got, want := stored.GetStatus().GetCrash().GetMessage(), originalCrash.GetMessage(); got != want { + t.Errorf("%s: crash message = %q, want %q kept", mode.name, got, want) } }) } diff --git a/cmd/ateapi/internal/controlapi/workflow_tag.go b/cmd/ateapi/internal/controlapi/workflow_tag.go index 384e0084da..690d52bc84 100644 --- a/cmd/ateapi/internal/controlapi/workflow_tag.go +++ b/cmd/ateapi/internal/controlapi/workflow_tag.go @@ -19,11 +19,11 @@ import ( "errors" "fmt" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" ) diff --git a/cmd/ateapi/internal/controlapi/workflow_tag_test.go b/cmd/ateapi/internal/controlapi/workflow_tag_test.go index c2de668b39..425bd24bd4 100644 --- a/cmd/ateapi/internal/controlapi/workflow_tag_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_tag_test.go @@ -21,13 +21,13 @@ import ( "sync" "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/objectstore/objectstoretest" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/objectstore/objectstoretest" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/protobuf/testing/protocmp" ) diff --git a/cmd/ateapi/internal/controlapi/workflow_template_delete.go b/cmd/ateapi/internal/controlapi/workflow_template_delete.go index 91fce8361b..553ee0585a 100644 --- a/cmd/ateapi/internal/controlapi/workflow_template_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_template_delete.go @@ -19,11 +19,11 @@ import ( "errors" "fmt" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" ) diff --git a/cmd/ateapi/internal/controlapi/workflow_test.go b/cmd/ateapi/internal/controlapi/workflow_test.go index 61c53f42b0..b40faa7a06 100644 --- a/cmd/ateapi/internal/controlapi/workflow_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_test.go @@ -18,12 +18,12 @@ import ( "context" "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" - "github.com/agent-substrate/substrate/internal/actorevent" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/internal/actorevent" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // TestActorStateChangeRecords drives each transition that commits a new state @@ -38,6 +38,7 @@ func TestActorStateChangeRecords(t *testing.T) { tests := []struct { name string seedState ateapipb.ActorState + seedOpts []func(*ateapipb.Actor) // transition runs the workflow step that commits the new state. transition func(t *testing.T, w *ActorWorkflow, ref resources.ActorRef, actor *ateapipb.Actor, tmpl *ateapipb.ActorTemplate) wantOp string @@ -60,10 +61,29 @@ func TestActorStateChangeRecords(t *testing.T) { transition: func(t *testing.T, w *ActorWorkflow, ref resources.ActorRef, actor *ateapipb.Actor, _ *ateapipb.ActorTemplate) { markOnly(t, w, ref, actor, w.PauseActor) }, - wantOp: ateattr.OperationPause, + wantOp: ateattr.OperationSuspend, wantState: ateattr.ActorStateSuspending, wantStored: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, }, + { + name: "finalize pause marks the actor suspended", + seedState: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, + seedOpts: []func(*ateapipb.Actor){ + withPauseInProgress(), + func(a *ateapipb.Actor) { + a.Status.AssignedNode = "node-1" + a.Status.WorkerAssignment = wireTestAssignment() + }, + }, + transition: func(t *testing.T, w *ActorWorkflow, ref resources.ActorRef, _ *ateapipb.Actor, _ *ateapipb.ActorTemplate) { + if _, err := w.PauseActor(context.Background(), ref); err != nil { + t.Fatalf("PauseActor: %v", err) + } + }, + wantOp: ateattr.OperationSuspend, + wantState: ateattr.ActorStateSuspended, + wantStored: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, + }, { name: "delete marks the actor deleting", seedState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, @@ -96,7 +116,7 @@ func TestActorStateChangeRecords(t *testing.T) { } actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} - seedWorkflowActor(t, ctx, persistence, actorRef, tmplAtespace, tmplName, tt.seedState) + seedWorkflowActor(t, ctx, persistence, actorRef, tmplAtespace, tmplName, tt.seedState, tt.seedOpts...) actor, err := persistence.GetActor(ctx, actorRef) if err != nil { t.Fatalf("get actor: %v", err) @@ -106,7 +126,7 @@ func TestActorStateChangeRecords(t *testing.T) { t.Fatalf("get template: %v", err) } - w := newMarkOnlyWorkflow(persistence) + w, _ := newWireCaptureWorkflow(t, persistence) tt.transition(t, w, actorRef, actor, tmpl) // The logs exporter is on, so the record goes to OTLP only. diff --git a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go index 4e1f95c86d..eb68bc16c5 100644 --- a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go @@ -21,15 +21,15 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/objectstore/objectstoretest" - "github.com/agent-substrate/substrate/internal/objectstoreplugin/objectstoreplugintest" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/uuid" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/objectstore/objectstoretest" + "go.ate.dev/substrate/internal/objectstoreplugin/objectstoreplugintest" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" sdkmetric "go.opentelemetry.io/otel/sdk/metric" "google.golang.org/grpc/codes" ) diff --git a/cmd/ateapi/internal/controlapi/workflow_worker_delete.go b/cmd/ateapi/internal/controlapi/workflow_worker_delete.go index eddf952f6c..5aeec6555d 100644 --- a/cmd/ateapi/internal/controlapi/workflow_worker_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_worker_delete.go @@ -20,12 +20,12 @@ import ( "fmt" "log/slog" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // DeleteWorker executes the workflow to deregister a Worker. The caller reaches @@ -209,9 +209,6 @@ func (w *WorkerWorkflow) crashBoundActor(ctx context.Context, worker *ateapipb.W opName = ateattr.OperationResume case ateapipb.ActorState_ACTOR_STATE_SUSPENDING: opName = ateattr.OperationSuspend - if latestSnapshotIsPendingLocal(actor.GetStatus()) { - opName = ateattr.OperationPause - } case ateapipb.ActorState_ACTOR_STATE_REVERTING: opName = ateattr.OperationRevert case ateapipb.ActorState_ACTOR_STATE_DELETING: diff --git a/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go index 5c1493d4db..0bd5037174 100644 --- a/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go @@ -20,12 +20,12 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" sdkmetric "go.opentelemetry.io/otel/sdk/metric" "google.golang.org/grpc/codes" ) @@ -159,11 +159,8 @@ func TestDeleteWorkerWorkflow_ReleasesBoundActor(t *testing.T) { // crashed is released again without being counted twice. func TestDeleteWorkerWorkflow_ReleasedActorStateTransitions(t *testing.T) { tests := []struct { - name string - start ateapipb.ActorState - // pausing seeds the pause-in-progress shape (a pending node-local - // snapshot), which labels a SUSPENDING crash as a pause. - pausing bool + name string + start ateapipb.ActorState wantState ateapipb.ActorState wantOp string wantMetric bool @@ -171,7 +168,6 @@ func TestDeleteWorkerWorkflow_ReleasedActorStateTransitions(t *testing.T) { {name: "running becomes crashed", start: ateapipb.ActorState_ACTOR_STATE_RUNNING, wantState: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantOp: ateattr.OperationUnknown, wantMetric: true}, {name: "resuming becomes crashed", start: ateapipb.ActorState_ACTOR_STATE_RESUMING, wantState: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantOp: ateattr.OperationResume, wantMetric: true}, {name: "suspending becomes crashed", start: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, wantState: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantOp: ateattr.OperationSuspend, wantMetric: true}, - {name: "pausing becomes crashed", start: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, pausing: true, wantState: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantOp: ateattr.OperationPause, wantMetric: true}, {name: "reverting becomes crashed", start: ateapipb.ActorState_ACTOR_STATE_REVERTING, wantState: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantOp: ateattr.OperationRevert, wantMetric: true}, {name: "deleting becomes crashed", start: ateapipb.ActorState_ACTOR_STATE_DELETING, wantState: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantOp: ateattr.OperationDelete, wantMetric: true}, {name: "suspended stays suspended", start: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, wantState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, wantMetric: false}, @@ -187,11 +183,7 @@ func TestDeleteWorkerWorkflow_ReleasedActorStateTransitions(t *testing.T) { ctx := context.Background() wf, persistence := newWorkerDeleteWorkflow(t) seedAPIWorker(t, ctx, persistence, validWorker(apiWorkerName)) - var opts []func(*ateapipb.Actor) - if tc.pausing { - opts = append(opts, withPauseInProgress()) - } - actor := seedAPIActor(t, ctx, persistence, tc.start, opts...) + actor := seedAPIActor(t, ctx, persistence, tc.start) assignAPIWorker(t, ctx, persistence, apiWorkerName, actor.GetMetadata().GetUid()) if _, err := wf.DeleteWorker(ctx, apiWorkerName, store.DeletePreconditions{}); err != nil { diff --git a/cmd/ateapi/internal/controlapi/workload_spec.go b/cmd/ateapi/internal/controlapi/workload_spec.go index bed3c3f602..790d6b086a 100644 --- a/cmd/ateapi/internal/controlapi/workload_spec.go +++ b/cmd/ateapi/internal/controlapi/workload_spec.go @@ -17,8 +17,8 @@ package controlapi import ( "fmt" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "k8s.io/apimachinery/pkg/api/resource" ) diff --git a/cmd/ateapi/internal/controlapi/workload_spec_test.go b/cmd/ateapi/internal/controlapi/workload_spec_test.go index a1b640ef0f..f5adef117a 100644 --- a/cmd/ateapi/internal/controlapi/workload_spec_test.go +++ b/cmd/ateapi/internal/controlapi/workload_spec_test.go @@ -17,9 +17,9 @@ package controlapi import ( "testing" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/testing/protocmp" ) diff --git a/cmd/ateapi/internal/defaults/defaults.go b/cmd/ateapi/internal/defaults/defaults.go index ff31f792d4..73849186a5 100644 --- a/cmd/ateapi/internal/defaults/defaults.go +++ b/cmd/ateapi/internal/defaults/defaults.go @@ -21,7 +21,7 @@ package defaults import ( "fmt" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" ) diff --git a/cmd/ateapi/internal/defaults/defaults_test.go b/cmd/ateapi/internal/defaults/defaults_test.go index 5ccb5ba159..9c82226087 100644 --- a/cmd/ateapi/internal/defaults/defaults_test.go +++ b/cmd/ateapi/internal/defaults/defaults_test.go @@ -17,8 +17,8 @@ package defaults import ( "testing" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/testing/protocmp" diff --git a/cmd/ateapi/internal/scheduling/scheduling.go b/cmd/ateapi/internal/scheduling/scheduling.go index d820911dca..070d2bceb2 100644 --- a/cmd/ateapi/internal/scheduling/scheduling.go +++ b/cmd/ateapi/internal/scheduling/scheduling.go @@ -22,9 +22,9 @@ import ( "math/rand" "slices" - "github.com/agent-substrate/substrate/internal/hardware" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/hardware" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "k8s.io/apimachinery/pkg/labels" ) @@ -34,10 +34,11 @@ type Constraints struct { // portable across sandbox classes, so this is never relaxed. SandboxClass string - // RequiredRuntime, when non-nil, restricts placement to workers whose - // DefaultRuntime or RestorableRuntimes can restore a snapshot stamped with - // this runtime. - RequiredRuntime *ateapipb.SandboxRuntime + // RequiredRuntime, when it names a runtime, restricts placement to + // workers whose DefaultRuntime or RestorableRuntimes can restore a + // snapshot stamped with it. Prepared once per Schedule, since Applies + // asks it of every candidate. + RequiredRuntime hardware.Requirement // TemplateSelector and ActorSelector must both match the worker's labels. TemplateSelector labels.Selector @@ -232,7 +233,7 @@ func (s *scheduler) Applies(worker *ateapipb.Worker, constraints Constraints) bo return false } - if !hardware.WorkerMatches(worker.GetStatus(), constraints.RequiredRuntime) { + if !constraints.RequiredRuntime.WorkerMatches(worker.GetStatus()) { return false } diff --git a/cmd/ateapi/internal/scheduling/scheduling_test.go b/cmd/ateapi/internal/scheduling/scheduling_test.go index 28a0b5b73c..d94e50d5f9 100644 --- a/cmd/ateapi/internal/scheduling/scheduling_test.go +++ b/cmd/ateapi/internal/scheduling/scheduling_test.go @@ -19,8 +19,9 @@ import ( "errors" "testing" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/hardware" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "k8s.io/apimachinery/pkg/labels" ) @@ -331,7 +332,7 @@ func TestSchedule(t *testing.T) { worker("w-arm", "gvisor", "node-a", tierTwo, withRuntimes(sandboxRuntime("gvisor", "arm64"))), worker("w-amd", "gvisor", "node-b", tierTwo, withRuntimes(sandboxRuntime("gvisor", "amd64"))), }, - constraints: Constraints{SandboxClass: "gvisor", RequiredRuntime: sandboxRuntime("gvisor", "amd64")}, + constraints: Constraints{SandboxClass: "gvisor", RequiredRuntime: hardware.NewRequirement(sandboxRuntime("gvisor", "amd64"))}, wantPod: "w-amd", }, { @@ -340,7 +341,7 @@ func TestSchedule(t *testing.T) { worker("w-upgraded", "gvisor", "node-a", tierTwo, withRuntimes(sandboxRuntime("gvisor", "amd64", "new"), sandboxRuntime("gvisor", "amd64", "old"))), }, - constraints: Constraints{SandboxClass: "gvisor", RequiredRuntime: sandboxRuntime("gvisor", "amd64", "old")}, + constraints: Constraints{SandboxClass: "gvisor", RequiredRuntime: hardware.NewRequirement(sandboxRuntime("gvisor", "amd64", "old"))}, wantPod: "w-upgraded", }, { @@ -348,7 +349,7 @@ func TestSchedule(t *testing.T) { fleet: fleet{ worker("w-arm", "gvisor", "node-a", tierTwo, withRuntimes(sandboxRuntime("gvisor", "arm64"))), }, - constraints: Constraints{SandboxClass: "gvisor", RequiredRuntime: sandboxRuntime("gvisor", "amd64")}, + constraints: Constraints{SandboxClass: "gvisor", RequiredRuntime: hardware.NewRequirement(sandboxRuntime("gvisor", "amd64"))}, }, } @@ -458,20 +459,20 @@ func TestApplies(t *testing.T) { { name: "matches required runtime via default runtime", worker: worker("w", "gvisor", "node-a", nil, withRuntimes(sandboxRuntime("gvisor", "amd64", "old"))), - constraints: Constraints{SandboxClass: "gvisor", RequiredRuntime: sandboxRuntime("gvisor", "amd64", "old")}, + constraints: Constraints{SandboxClass: "gvisor", RequiredRuntime: hardware.NewRequirement(sandboxRuntime("gvisor", "amd64", "old"))}, want: true, }, { name: "matches required runtime via restorable runtimes", worker: worker("w", "gvisor", "node-a", nil, withRuntimes(sandboxRuntime("gvisor", "amd64", "new"), sandboxRuntime("gvisor", "amd64", "old"))), - constraints: Constraints{SandboxClass: "gvisor", RequiredRuntime: sandboxRuntime("gvisor", "amd64", "old")}, + constraints: Constraints{SandboxClass: "gvisor", RequiredRuntime: hardware.NewRequirement(sandboxRuntime("gvisor", "amd64", "old"))}, want: true, }, { name: "skips worker when required runtime does not match", worker: worker("w", "gvisor", "node-a", nil, withRuntimes(sandboxRuntime("gvisor", "arm64"))), - constraints: Constraints{SandboxClass: "gvisor", RequiredRuntime: sandboxRuntime("gvisor", "amd64")}, + constraints: Constraints{SandboxClass: "gvisor", RequiredRuntime: hardware.NewRequirement(sandboxRuntime("gvisor", "amd64"))}, want: false, }, } diff --git a/cmd/ateapi/internal/store/atepg/access_policy.go b/cmd/ateapi/internal/store/atepg/access_policy.go index e09880f33d..93759d49dd 100644 --- a/cmd/ateapi/internal/store/atepg/access_policy.go +++ b/cmd/ateapi/internal/store/atepg/access_policy.go @@ -19,9 +19,9 @@ import ( "errors" "fmt" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/jackc/pgx/v5" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" ) diff --git a/cmd/ateapi/internal/store/atepg/access_policy_test.go b/cmd/ateapi/internal/store/atepg/access_policy_test.go index 48abda0138..e2e89775b6 100644 --- a/cmd/ateapi/internal/store/atepg/access_policy_test.go +++ b/cmd/ateapi/internal/store/atepg/access_policy_test.go @@ -20,10 +20,10 @@ import ( "sync" "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/authz" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/cmd/ateapi/internal/authz" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/testing/protocmp" ) diff --git a/cmd/ateapi/internal/store/atepg/actor.go b/cmd/ateapi/internal/store/atepg/actor.go index 538d263511..f370f911a2 100644 --- a/cmd/ateapi/internal/store/atepg/actor.go +++ b/cmd/ateapi/internal/store/atepg/actor.go @@ -19,10 +19,10 @@ import ( "errors" "fmt" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/jackc/pgx/v5" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" ) diff --git a/cmd/ateapi/internal/store/atepg/actor_template.go b/cmd/ateapi/internal/store/atepg/actor_template.go index 8a480ade6e..d49f6edee0 100644 --- a/cmd/ateapi/internal/store/atepg/actor_template.go +++ b/cmd/ateapi/internal/store/atepg/actor_template.go @@ -19,10 +19,10 @@ import ( "errors" "fmt" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/jackc/pgx/v5" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" ) diff --git a/cmd/ateapi/internal/store/atepg/actor_template_test.go b/cmd/ateapi/internal/store/atepg/actor_template_test.go index 6560c1d699..48954008fe 100644 --- a/cmd/ateapi/internal/store/atepg/actor_template_test.go +++ b/cmd/ateapi/internal/store/atepg/actor_template_test.go @@ -20,10 +20,10 @@ import ( "fmt" "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/testing/protocmp" ) diff --git a/cmd/ateapi/internal/store/atepg/actor_test.go b/cmd/ateapi/internal/store/atepg/actor_test.go index 337a758663..713f91aa30 100644 --- a/cmd/ateapi/internal/store/atepg/actor_test.go +++ b/cmd/ateapi/internal/store/atepg/actor_test.go @@ -20,9 +20,9 @@ import ( "fmt" "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) func TestUpdateActor_ConcurrentWriteReturnsConflict(t *testing.T) { diff --git a/cmd/ateapi/internal/store/atepg/atepg.go b/cmd/ateapi/internal/store/atepg/atepg.go index d5945f1f70..a84c7218ec 100644 --- a/cmd/ateapi/internal/store/atepg/atepg.go +++ b/cmd/ateapi/internal/store/atepg/atepg.go @@ -30,14 +30,14 @@ import ( "sync" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/authz" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/defaults" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" + "go.ate.dev/substrate/cmd/ateapi/internal/authz" + "go.ate.dev/substrate/cmd/ateapi/internal/defaults" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" ) diff --git a/cmd/ateapi/internal/store/atepg/atepg_test.go b/cmd/ateapi/internal/store/atepg/atepg_test.go index 6c0e6c0bb0..a1063ba6bd 100644 --- a/cmd/ateapi/internal/store/atepg/atepg_test.go +++ b/cmd/ateapi/internal/store/atepg/atepg_test.go @@ -19,8 +19,8 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // A listing fails as a whole on one undecodable row, so the error has to name diff --git a/cmd/ateapi/internal/store/atepg/atespace.go b/cmd/ateapi/internal/store/atepg/atespace.go index 9d18c70329..c948b74850 100644 --- a/cmd/ateapi/internal/store/atepg/atespace.go +++ b/cmd/ateapi/internal/store/atepg/atespace.go @@ -19,9 +19,9 @@ import ( "errors" "fmt" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/jackc/pgx/v5" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" ) diff --git a/cmd/ateapi/internal/store/atepg/cert_auth_test.go b/cmd/ateapi/internal/store/atepg/cert_auth_test.go index c333f44307..d20f819f54 100644 --- a/cmd/ateapi/internal/store/atepg/cert_auth_test.go +++ b/cmd/ateapi/internal/store/atepg/cert_auth_test.go @@ -32,13 +32,13 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/localca" - "github.com/agent-substrate/substrate/pkg/postgressetup" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "github.com/testcontainers/testcontainers-go" tcexec "github.com/testcontainers/testcontainers-go/exec" "github.com/testcontainers/testcontainers-go/modules/postgres" + "go.ate.dev/substrate/internal/localca" + "go.ate.dev/substrate/pkg/postgressetup" "sigs.k8s.io/yaml" ) diff --git a/cmd/ateapi/internal/store/atepg/common_test.go b/cmd/ateapi/internal/store/atepg/common_test.go index f85e8741a6..accb18c49f 100644 --- a/cmd/ateapi/internal/store/atepg/common_test.go +++ b/cmd/ateapi/internal/store/atepg/common_test.go @@ -18,9 +18,9 @@ import ( "context" "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/authz" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/authz" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // clearAll truncates every table so the next test starts from an empty store @@ -59,6 +59,7 @@ func setTestPolicyManager(t *testing.T, p *Persistence) { ControllerSPIFFEID: installdefaults.ControllerSPIFFEID(installdefaults.SystemNamespace), EgressGatewaySPIFFEID: installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace), IngressRouterSPIFFEID: installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), + AteletSPIFFEID: installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), }) if err != nil { t.Fatalf("authz.New failed: %v", err) diff --git a/cmd/ateapi/internal/store/atepg/contract_test.go b/cmd/ateapi/internal/store/atepg/contract_test.go index 5aefaf1d0b..d5ea2cf7b9 100644 --- a/cmd/ateapi/internal/store/atepg/contract_test.go +++ b/cmd/ateapi/internal/store/atepg/contract_test.go @@ -17,8 +17,8 @@ package atepg import ( "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storecontract" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storecontract" ) // setupPostgresStore adapts the concrete setup helper to the interface-typed diff --git a/cmd/ateapi/internal/store/atepg/egress_policy.go b/cmd/ateapi/internal/store/atepg/egress_policy.go index 70c679656b..763c8ff37e 100644 --- a/cmd/ateapi/internal/store/atepg/egress_policy.go +++ b/cmd/ateapi/internal/store/atepg/egress_policy.go @@ -19,10 +19,10 @@ import ( "errors" "fmt" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/jackc/pgx/v5" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" ) diff --git a/cmd/ateapi/internal/store/atepg/lease.go b/cmd/ateapi/internal/store/atepg/lease.go index 4303cec0fe..f5aa783b64 100644 --- a/cmd/ateapi/internal/store/atepg/lease.go +++ b/cmd/ateapi/internal/store/atepg/lease.go @@ -21,9 +21,9 @@ import ( "log/slog" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/google/uuid" "github.com/jackc/pgx/v5" + "go.ate.dev/substrate/cmd/ateapi/internal/store" ) // defaultLeaseTTL is how long a lease may go unrenewed before another client diff --git a/cmd/ateapi/internal/store/atepg/lease_test.go b/cmd/ateapi/internal/store/atepg/lease_test.go index a68859b909..c3d65b09eb 100644 --- a/cmd/ateapi/internal/store/atepg/lease_test.go +++ b/cmd/ateapi/internal/store/atepg/lease_test.go @@ -21,7 +21,7 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/store" ) // countLeases returns how many lease rows exist for key. diff --git a/cmd/ateapi/internal/store/atepg/main_test.go b/cmd/ateapi/internal/store/atepg/main_test.go index 8009525698..a0ca3b1bf8 100644 --- a/cmd/ateapi/internal/store/atepg/main_test.go +++ b/cmd/ateapi/internal/store/atepg/main_test.go @@ -25,7 +25,7 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "github.com/testcontainers/testcontainers-go/modules/postgres" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/dockerenv" + "go.ate.dev/substrate/cmd/ateapi/internal/store/dockerenv" ) // One Postgres container serves every test in this package; each test gets diff --git a/cmd/ateapi/internal/store/atepg/outbox.go b/cmd/ateapi/internal/store/atepg/outbox.go index ca9bdca410..3d1e71de45 100644 --- a/cmd/ateapi/internal/store/atepg/outbox.go +++ b/cmd/ateapi/internal/store/atepg/outbox.go @@ -28,9 +28,9 @@ import ( "strings" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/jackc/pgx/v5" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" ) diff --git a/cmd/ateapi/internal/store/atepg/outbox_test.go b/cmd/ateapi/internal/store/atepg/outbox_test.go index 1e133660e2..9bbcee22a6 100644 --- a/cmd/ateapi/internal/store/atepg/outbox_test.go +++ b/cmd/ateapi/internal/store/atepg/outbox_test.go @@ -31,8 +31,8 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/testing/protocmp" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // TestConnect_DedicatedWatchPool covers the auxiliary-pool path only Connect diff --git a/cmd/ateapi/internal/store/atepg/pagetoken.go b/cmd/ateapi/internal/store/atepg/pagetoken.go index ebf56b5ae2..837a84c98b 100644 --- a/cmd/ateapi/internal/store/atepg/pagetoken.go +++ b/cmd/ateapi/internal/store/atepg/pagetoken.go @@ -19,7 +19,7 @@ import ( "encoding/json" "fmt" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/store" ) // pageTokenVersion guards against decoding a token produced by an incompatible diff --git a/cmd/ateapi/internal/store/atepg/partition_test.go b/cmd/ateapi/internal/store/atepg/partition_test.go index 20ba51fd29..3da1a7497b 100644 --- a/cmd/ateapi/internal/store/atepg/partition_test.go +++ b/cmd/ateapi/internal/store/atepg/partition_test.go @@ -26,8 +26,8 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storecontract" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storecontract" ) // TestActorsTablePartitionable exists to keep it possible to partition the diff --git a/cmd/ateapi/internal/store/atepg/postgressetup_test.go b/cmd/ateapi/internal/store/atepg/postgressetup_test.go index b3cc113668..512d704616 100644 --- a/cmd/ateapi/internal/store/atepg/postgressetup_test.go +++ b/cmd/ateapi/internal/store/atepg/postgressetup_test.go @@ -25,7 +25,7 @@ import ( "github.com/jackc/pgx/v5/pgxpool" tcexec "github.com/testcontainers/testcontainers-go/exec" - "github.com/agent-substrate/substrate/pkg/postgressetup" + "go.ate.dev/substrate/pkg/postgressetup" ) func TestPostgresSetupScript(t *testing.T) { diff --git a/cmd/ateapi/internal/store/atepg/tag.go b/cmd/ateapi/internal/store/atepg/tag.go index 2a42f7fe8e..794c9fada7 100644 --- a/cmd/ateapi/internal/store/atepg/tag.go +++ b/cmd/ateapi/internal/store/atepg/tag.go @@ -19,10 +19,10 @@ import ( "errors" "fmt" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/jackc/pgx/v5" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" ) diff --git a/cmd/ateapi/internal/store/atepg/tag_test.go b/cmd/ateapi/internal/store/atepg/tag_test.go index 9d4682dfc2..315fa8b68b 100644 --- a/cmd/ateapi/internal/store/atepg/tag_test.go +++ b/cmd/ateapi/internal/store/atepg/tag_test.go @@ -20,10 +20,10 @@ import ( "fmt" "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/testing/protocmp" ) diff --git a/cmd/ateapi/internal/store/atepg/worker.go b/cmd/ateapi/internal/store/atepg/worker.go index a7428e1100..4e86bfed49 100644 --- a/cmd/ateapi/internal/store/atepg/worker.go +++ b/cmd/ateapi/internal/store/atepg/worker.go @@ -19,9 +19,9 @@ import ( "errors" "fmt" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/jackc/pgx/v5" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" ) diff --git a/cmd/ateapi/internal/store/atepg/worker_assignment.go b/cmd/ateapi/internal/store/atepg/worker_assignment.go index 4e85ae595a..b7e49f489f 100644 --- a/cmd/ateapi/internal/store/atepg/worker_assignment.go +++ b/cmd/ateapi/internal/store/atepg/worker_assignment.go @@ -19,10 +19,10 @@ import ( "errors" "fmt" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/jackc/pgx/v5" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" ) diff --git a/cmd/ateapi/internal/store/atepg/worker_assignment_test.go b/cmd/ateapi/internal/store/atepg/worker_assignment_test.go index 7a26a5e26d..ae3ea6dec2 100644 --- a/cmd/ateapi/internal/store/atepg/worker_assignment_test.go +++ b/cmd/ateapi/internal/store/atepg/worker_assignment_test.go @@ -19,8 +19,8 @@ import ( "errors" "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // TestSaveWorker_RejectsAStaleWrite proves the precondition saveWorker states diff --git a/cmd/ateapi/internal/store/store.go b/cmd/ateapi/internal/store/store.go index cc494082c1..254bf64f44 100644 --- a/cmd/ateapi/internal/store/store.go +++ b/cmd/ateapi/internal/store/store.go @@ -21,8 +21,8 @@ import ( "fmt" "sync" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) var ( diff --git a/cmd/ateapi/internal/store/store_test.go b/cmd/ateapi/internal/store/store_test.go index 477dc06f2e..b5646d4dfc 100644 --- a/cmd/ateapi/internal/store/store_test.go +++ b/cmd/ateapi/internal/store/store_test.go @@ -18,7 +18,7 @@ import ( "errors" "testing" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) const ( diff --git a/cmd/ateapi/internal/store/storecontract/contract.go b/cmd/ateapi/internal/store/storecontract/contract.go index 2093146326..847e7cc926 100644 --- a/cmd/ateapi/internal/store/storecontract/contract.go +++ b/cmd/ateapi/internal/store/storecontract/contract.go @@ -35,9 +35,9 @@ import ( "google.golang.org/protobuf/testing/protocmp" "google.golang.org/protobuf/types/known/timestamppb" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // testAtespace is the atespace used by tests that create a single actor. diff --git a/cmd/ateapi/internal/store/storetest/storetest.go b/cmd/ateapi/internal/store/storetest/storetest.go index 96d7e17428..3385bb4112 100644 --- a/cmd/ateapi/internal/store/storetest/storetest.go +++ b/cmd/ateapi/internal/store/storetest/storetest.go @@ -29,14 +29,14 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/authz" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/atepg" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/dockerenv" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/jackc/pgx/v5/pgxpool" "github.com/testcontainers/testcontainers-go/modules/postgres" + "go.ate.dev/substrate/cmd/ateapi/internal/authz" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/store/atepg" + "go.ate.dev/substrate/cmd/ateapi/internal/store/dockerenv" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) var ( @@ -103,6 +103,7 @@ func DefaultSystemPrincipals() authz.SystemPrincipals { ControllerSPIFFEID: installdefaults.ControllerSPIFFEID(installdefaults.SystemNamespace), EgressGatewaySPIFFEID: installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace), IngressRouterSPIFFEID: installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), + AteletSPIFFEID: installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), } } diff --git a/cmd/ateapi/internal/workercache/workercache.go b/cmd/ateapi/internal/workercache/workercache.go index cef58c2407..8ab910b4b1 100644 --- a/cmd/ateapi/internal/workercache/workercache.go +++ b/cmd/ateapi/internal/workercache/workercache.go @@ -26,8 +26,8 @@ import ( "sync/atomic" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/pkg/proto/ateapipb" "k8s.io/apimachinery/pkg/util/wait" ) diff --git a/cmd/ateapi/internal/workercache/workercache_test.go b/cmd/ateapi/internal/workercache/workercache_test.go index 0fc35c4e43..bc6d69ad2f 100644 --- a/cmd/ateapi/internal/workercache/workercache_test.go +++ b/cmd/ateapi/internal/workercache/workercache_test.go @@ -21,11 +21,11 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/workercache" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/testing/protocmp" "k8s.io/apimachinery/pkg/util/wait" ) diff --git a/cmd/ateapi/internal/workerservice/certificate.go b/cmd/ateapi/internal/workerservice/certificate.go index c6737bf419..7d75515811 100644 --- a/cmd/ateapi/internal/workerservice/certificate.go +++ b/cmd/ateapi/internal/workerservice/certificate.go @@ -19,17 +19,18 @@ import ( "crypto/x509" "errors" "fmt" + "log/slog" "net/url" "path" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/apivalidation" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/ateletauth" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/substratex509" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/apivalidation" + "go.ate.dev/substrate/cmd/ateapi/internal/ateletauth" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/substratex509" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // MintWorkerActorCertificate mints a Substrate-issued SPIFFE certificate that asserts @@ -40,23 +41,39 @@ func (s *Server) MintWorkerActorCertificate(ctx context.Context, req *ateapipb.M } // TODO(identity): This check should be handled by OpenFGA. - if _, err := ateletauth.Authenticate(ctx, s.ateletSPIFFEID); err != nil { + caller, err := ateletauth.Authenticate(ctx, s.ateletSPIFFEID) + if err != nil { return nil, err } - // TODO(authz): Authorization layer needs to check whether the caller has - // the mintActorCertificate permission/relation with this actor. This - // could be an atelet (via the relationship of the atelet running the - // actor), or the egress gateway (via a cluster-level grant?) + // TODO(authz): The egress gateway may also need certificates for actors + // (via a cluster-level grant?). - // Verify that this actor exists in the store. It doesn't need to be - // running, since we may need to issue certificates during actor boot / resume. dbActor, err := s.store.GetActor(ctx, resources.ActorRefFromObjectRef(req.GetActor())) if errors.Is(err, store.ErrNotFound) { return nil, apierror.NotFound("actor not found") } else if err != nil { return nil, fmt.Errorf("while retrieving actor: %w", err) } + // Only the assigned worker may mint, via its node's atelet, checked before the + // UID so other callers get NotFound. Unlike MintWorkerActorJwt, a paused actor + // gets NotFound too: it has no worker to bind the certificate to. + if err := s.checkActorOnNode(ctx, dbActor, caller.NodeName, "certificate"); err != nil { + return nil, err + } + // The status must name the requesting worker. When it names one, + // checkActorOnNode has confirmed in the store's worker assignments that + // this worker holds the actor. When it names none (a paused actor, or a + // resume that has bound a worker but not yet recorded it), checkActorOnNode + // read assigned_node alone, and no worker runs the actor yet. + if assigned := dbActor.GetStatus().GetWorkerAssignment().GetWorker().GetName(); assigned == "" || assigned != req.GetWorker().GetName() { + slog.WarnContext(ctx, "Refusing a worker-for-actor certificate for an actor not assigned to the calling worker", + slog.String("actor", resources.ActorRefFromActor(dbActor).String()), + slog.String("assigned_worker", assigned), + slog.String("caller_worker", req.GetWorker().GetName()), + slog.String("caller_node", caller.NodeName)) + return nil, apierror.NotFound("actor not found") + } if dbActor.GetMetadata().GetUid() != req.GetActorUid() { return nil, apierror.Aborted("conflict; actor has been deleted and recreated") } diff --git a/cmd/ateapi/internal/workerservice/certificate_test.go b/cmd/ateapi/internal/workerservice/certificate_test.go index 5ff978b5aa..afcbb1ea08 100644 --- a/cmd/ateapi/internal/workerservice/certificate_test.go +++ b/cmd/ateapi/internal/workerservice/certificate_test.go @@ -20,17 +20,18 @@ import ( "crypto/elliptic" "crypto/rand" "crypto/x509" + "errors" "testing" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/ateletauth/ateletauthtest" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/localca" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/substratex509" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/cmd/ateapi/internal/ateletauth/ateletauthtest" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/localca" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/substratex509" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" ) @@ -138,6 +139,7 @@ func TestMintWorkerActorCertificate(t *testing.T) { }, ActorUid: actor.GetMetadata().GetUid(), CertificateSigningRequest: csr, + Worker: &ateapipb.ObjectRef{Name: "worker-1"}, }) if err != nil { t.Fatalf("MintWorkerActorCertificate() failed: %v", err) @@ -182,77 +184,331 @@ func TestMintWorkerActorCertificate_Errors(t *testing.T) { csr, _ := generateTestCSR(t) authed := ateletauthtest.ContextWith(ateletauthtest.CertOn(t, "node-1")) + otherNode := ateletauthtest.ContextWith(ateletauthtest.CertOn(t, "node-2")) + valid := func() *ateapipb.MintWorkerActorCertificateRequest { + return &ateapipb.MintWorkerActorCertificateRequest{ + Actor: &ateapipb.ObjectRef{Atespace: "team-a", Name: "my-actor"}, + ActorUid: actor.GetMetadata().GetUid(), + CertificateSigningRequest: csr, + Worker: &ateapipb.ObjectRef{Name: "worker-1"}, + } + } + const ( + staleUID = "7d1c0b2a-9e4f-4c3d-8a6b-5f2e1d0c9b8a" + otherActorUID = "9a8b7c6d-5e4f-4a3b-8c2d-1e0f9a8b7c6d" + ) + // onWorker is an actor status naming worker-1, which runs on node-1. + onWorker := func(state ateapipb.ActorState) *ateapipb.ActorStatus { + return &ateapipb.ActorStatus{State: state, WorkerAssignment: &ateapipb.WorkerAssignment{Worker: &ateapipb.ObjectRef{Name: "worker-1"}}} + } + // pausingOnWorker is a pause in progress: SUSPENDING on worker-1 with a + // latest snapshot that is node-local and not yet captured. + pausingOnWorker := func() *ateapipb.ActorStatus { + status := onWorker(ateapipb.ActorState_ACTOR_STATE_SUSPENDING) + status.AssignedNode = "node-1" + status.LastAssignedGeneration = 1 + status.Snapshots = []*ateapipb.Snapshot{{Uuid: "snap-1", Generation: 1, Owner: ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR}} + return status + } + released := map[string]string{} + rebooked := map[string]string{otherActorUID: "worker-1"} tests := []struct { - name string - ctx context.Context - req *ateapipb.MintWorkerActorCertificateRequest - want codes.Code + name string + ctx context.Context + mutate func(*ateapipb.MintWorkerActorCertificateRequest) + // status, when set, replaces the actor's status for the case. + status *ateapipb.ActorStatus + // assignments, when set, replaces the store's worker assignments + // (actor UID to worker) for the case. By default worker-1 holds the + // actor. + assignments map[string]string + // assignmentErr, when set, fails the store's worker assignment + // lookup. + assignmentErr error + want codes.Code }{ { - name: "unauthenticated", - ctx: ateletauthtest.ContextWith(nil), - req: &ateapipb.MintWorkerActorCertificateRequest{ - Actor: &ateapipb.ObjectRef{Atespace: "team-a", Name: "my-actor"}, - ActorUid: actor.GetMetadata().GetUid(), - CertificateSigningRequest: csr, - }, - want: codes.Unauthenticated, + name: "running actor on its worker's node", + ctx: authed, + status: onWorker(ateapipb.ActorState_ACTOR_STATE_RUNNING), + want: codes.OK, }, { - name: "missing actor", - ctx: authed, - req: &ateapipb.MintWorkerActorCertificateRequest{ - ActorUid: actor.GetMetadata().GetUid(), - CertificateSigningRequest: csr, - }, - want: codes.InvalidArgument, + // atunnel mints while the actor resumes, before it runs. + name: "resuming actor on its worker's node", + ctx: authed, + status: onWorker(ateapipb.ActorState_ACTOR_STATE_RESUMING), + want: codes.OK, }, { - name: "missing actor uid", - ctx: authed, - req: &ateapipb.MintWorkerActorCertificateRequest{ - Actor: &ateapipb.ObjectRef{Atespace: "team-a", Name: "my-actor"}, - CertificateSigningRequest: csr, - }, - want: codes.InvalidArgument, + name: "running actor being suspended on its worker's node", + ctx: authed, + status: onWorker(ateapipb.ActorState_ACTOR_STATE_SUSPENDING), + want: codes.OK, }, { - name: "missing csr", - ctx: authed, - req: &ateapipb.MintWorkerActorCertificateRequest{ - Actor: &ateapipb.ObjectRef{Atespace: "team-a", Name: "my-actor"}, - ActorUid: actor.GetMetadata().GetUid(), - }, - want: codes.InvalidArgument, + name: "running actor being paused on its worker's node", + ctx: authed, + status: pausingOnWorker(), + want: codes.OK, }, { - name: "actor not found", - ctx: authed, - req: &ateapipb.MintWorkerActorCertificateRequest{ - Actor: &ateapipb.ObjectRef{Atespace: "team-a", Name: "nonexistent"}, - ActorUid: actor.GetMetadata().GetUid(), - CertificateSigningRequest: csr, - }, + // Pause, suspend and crash release the worker before they clear the + // actor's status, which keeps naming the worker until they do. + name: "running actor whose worker was released", + ctx: authed, + status: onWorker(ateapipb.ActorState_ACTOR_STATE_RUNNING), + assignments: released, + want: codes.NotFound, + }, + { + // A resume that finds its worker ineligible releases it, then + // crashes the actor. + name: "resuming actor whose worker was released", + ctx: authed, + status: onWorker(ateapipb.ActorState_ACTOR_STATE_RESUMING), + assignments: released, + want: codes.NotFound, + }, + { + name: "suspending actor whose worker was released", + ctx: authed, + status: onWorker(ateapipb.ActorState_ACTOR_STATE_SUSPENDING), + assignments: released, + want: codes.NotFound, + }, + { + name: "pausing actor whose worker was released", + ctx: authed, + status: pausingOnWorker(), + assignments: released, + want: codes.NotFound, + }, + { + // The released worker took another actor. Its node hosts that one, + // not this one. + name: "running actor whose worker was rebooked for another actor", + ctx: authed, + status: onWorker(ateapipb.ActorState_ACTOR_STATE_RUNNING), + assignments: rebooked, + want: codes.NotFound, + }, + { + name: "resuming actor whose worker was rebooked for another actor", + ctx: authed, + status: onWorker(ateapipb.ActorState_ACTOR_STATE_RESUMING), + assignments: rebooked, + want: codes.NotFound, + }, + { + name: "suspending actor whose worker was rebooked for another actor", + ctx: authed, + status: onWorker(ateapipb.ActorState_ACTOR_STATE_SUSPENDING), + assignments: rebooked, + want: codes.NotFound, + }, + { + name: "pausing actor whose worker was rebooked for another actor", + ctx: authed, + status: pausingOnWorker(), + assignments: rebooked, + want: codes.NotFound, + }, + { + // The store binds the actor to a different worker than its status + // names, so the status's worker does not host it. + name: "actor held by a worker other than the one its status names", + ctx: authed, + status: onWorker(ateapipb.ActorState_ACTOR_STATE_RUNNING), + assignments: map[string]string{actor.GetMetadata().GetUid(): "worker-2"}, + want: codes.NotFound, + }, + { + // A failed lookup refuses the mint rather than trusting the status. + name: "worker assignment lookup fails", + ctx: authed, + status: onWorker(ateapipb.ActorState_ACTOR_STATE_RUNNING), + assignmentErr: errors.New("connection reset"), + want: codes.Internal, + }, + { + // The actor's worker runs on node-1 and the caller proves it is on node-2. + name: "actor hosted on another node", + ctx: otherNode, want: codes.NotFound, }, { - name: "actor uid mismatch", + // A foreign node must not learn that the actor exists from the UID + // check answering Aborted: the node check runs first. + name: "actor hosted on another node, stale uid", + ctx: otherNode, + mutate: func(r *ateapipb.MintWorkerActorCertificateRequest) { r.ActorUid = staleUID }, + want: codes.NotFound, + }, + { + // worker-2 runs on the actor's node, but the actor is not its own. + name: "another worker on the actor's node", + ctx: authed, + mutate: func(r *ateapipb.MintWorkerActorCertificateRequest) { r.Worker.Name = "worker-2" }, + want: codes.NotFound, + }, + { + // The worker check also runs before the UID check. + name: "another worker on the actor's node, stale uid", ctx: authed, - req: &ateapipb.MintWorkerActorCertificateRequest{ - Actor: &ateapipb.ObjectRef{Atespace: "team-a", Name: "my-actor"}, - ActorUid: "00000000-0000-0000-0000-000000000000", - CertificateSigningRequest: csr, + mutate: func(r *ateapipb.MintWorkerActorCertificateRequest) { + r.Worker.Name = "worker-2" + r.ActorUid = staleUID }, - want: codes.Aborted, + want: codes.NotFound, + }, + { + // MintWorkerActorJwt serves a paused actor's checkpoint node. No + // worker runs a paused actor, so no worker gets its certificate. + name: "paused actor on this node", + ctx: authed, + status: pausedStatus(ateapipb.ActorState_ACTOR_STATE_SUSPENDED, "node-1"), + assignments: released, + want: codes.NotFound, + }, + { + name: "paused actor being suspended on this node", + ctx: authed, + status: pausedStatus(ateapipb.ActorState_ACTOR_STATE_SUSPENDING, "node-1"), + assignments: released, + want: codes.NotFound, + }, + { + // Resume binds the worker in the store before it writes the + // assignment to the actor's status, and starts the actor only + // after that. In between, the worker holds the store's assignment + // but runs nothing, and HostingNode trusts assigned_node without + // reading the store, so only the status comparison refuses. + name: "paused actor bound to a worker on this node before its status names the worker", + ctx: authed, + status: pausedStatus(ateapipb.ActorState_ACTOR_STATE_SUSPENDED, "node-1"), + assignments: map[string]string{actor.GetMetadata().GetUid(): "worker-1"}, + want: codes.NotFound, + }, + { + name: "paused actor on another node", + ctx: authed, + status: pausedStatus(ateapipb.ActorState_ACTOR_STATE_SUSPENDED, "node-2"), + assignments: released, + want: codes.NotFound, + }, + { + // An empty worker name must not match a paused actor's empty + // assignment. + name: "paused actor, empty worker name", + ctx: authed, + status: pausedStatus(ateapipb.ActorState_ACTOR_STATE_SUSPENDED, "node-1"), + assignments: released, + mutate: func(r *ateapipb.MintWorkerActorCertificateRequest) { r.Worker.Name = "" }, + want: codes.InvalidArgument, + }, + { + // A crash clears the worker but leaves assigned_node on the last + // node. Nothing is hosted there, so nothing may be minted. + name: "crashed actor on its last node", + ctx: authed, + status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_CRASHED, AssignedNode: "node-1"}, + want: codes.NotFound, + }, + { + // A revert releases the worker but keeps the assignment until it + // finalizes. The node no longer runs the actor. + name: "reverting actor with a stale assignment", + ctx: authed, + status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_REVERTING, WorkerAssignment: &ateapipb.WorkerAssignment{Worker: &ateapipb.ObjectRef{Name: "worker-1"}}}, + want: codes.NotFound, + }, + { + name: "actor hosted by no node", + ctx: authed, + status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}, + want: codes.NotFound, + }, + { + // The assignment still binds worker-gone, but its Worker record + // is gone. + name: "actor assigned to a worker the store has lost", + ctx: authed, + status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_RUNNING, WorkerAssignment: &ateapipb.WorkerAssignment{Worker: &ateapipb.ObjectRef{Name: "worker-gone"}}}, + assignments: map[string]string{actor.GetMetadata().GetUid(): "worker-gone"}, + want: codes.NotFound, + }, + { + name: "unauthenticated", + ctx: ateletauthtest.ContextWith(nil), + want: codes.Unauthenticated, + }, + { + name: "missing actor", + ctx: authed, + mutate: func(r *ateapipb.MintWorkerActorCertificateRequest) { r.Actor = nil }, + want: codes.InvalidArgument, + }, + { + name: "missing actor uid", + ctx: authed, + mutate: func(r *ateapipb.MintWorkerActorCertificateRequest) { r.ActorUid = "" }, + want: codes.InvalidArgument, + }, + { + name: "missing csr", + ctx: authed, + mutate: func(r *ateapipb.MintWorkerActorCertificateRequest) { r.CertificateSigningRequest = nil }, + want: codes.InvalidArgument, + }, + { + name: "missing worker", + ctx: authed, + mutate: func(r *ateapipb.MintWorkerActorCertificateRequest) { r.Worker = nil }, + want: codes.InvalidArgument, + }, + { + name: "worker with an atespace", + ctx: authed, + mutate: func(r *ateapipb.MintWorkerActorCertificateRequest) { r.Worker.Atespace = "team-a" }, + want: codes.InvalidArgument, + }, + { + name: "actor not found", + ctx: authed, + mutate: func(r *ateapipb.MintWorkerActorCertificateRequest) { r.Actor.Name = "nonexistent" }, + want: codes.NotFound, + }, + { + // The assigned worker asks for an incarnation that was deleted and + // recreated. + name: "stale actor uid", + ctx: authed, + mutate: func(r *ateapipb.MintWorkerActorCertificateRequest) { r.ActorUid = staleUID }, + want: codes.Aborted, }, } - + hostedStatus := actor.GetStatus() + hostedAssignments := st.assignments for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - _, err := s.MintWorkerActorCertificate(tc.ctx, tc.req) + actor.Status = hostedStatus + if tc.status != nil { + actor.Status = tc.status + } + st.assignments = hostedAssignments + if tc.assignments != nil { + st.assignments = tc.assignments + } + st.assignmentErr = tc.assignmentErr + req := valid() + if tc.mutate != nil { + tc.mutate(req) + } + _, err := s.MintWorkerActorCertificate(tc.ctx, req) if got := apierror.Code(err); got != tc.want { - t.Errorf("code = %v (err %v), want %v", got, err, tc.want) + t.Errorf("MintWorkerActorCertificate() code = %v, want %v (err: %v)", got, tc.want, err) } }) } diff --git a/cmd/ateapi/internal/workerservice/jwt.go b/cmd/ateapi/internal/workerservice/jwt.go index be85b4666b..4889c43c8c 100644 --- a/cmd/ateapi/internal/workerservice/jwt.go +++ b/cmd/ateapi/internal/workerservice/jwt.go @@ -22,14 +22,14 @@ import ( "log/slog" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/apivalidation" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/ateletauth" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/controlapi" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/actoridjwt" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/apivalidation" + "go.ate.dev/substrate/cmd/ateapi/internal/ateletauth" + "go.ate.dev/substrate/cmd/ateapi/internal/controlapi" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/actoridjwt" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -59,7 +59,7 @@ func (s *Server) MintWorkerActorJwt(ctx context.Context, req *ateapipb.MintWorke // The node check comes first so a node that does not host the actor gets // NotFound whatever UID it sends, and cannot use the UID check to learn // that the actor exists. - if err := s.checkActorOnNode(ctx, dbActor, caller.NodeName); err != nil { + if err := s.checkActorOnNode(ctx, dbActor, caller.NodeName, "JWT"); err != nil { return nil, err } // A stale UID is Aborted, as in MintWorkerActorCertificate. @@ -101,14 +101,14 @@ func (s *Server) MintWorkerActorJwt(ctx context.Context, req *ateapipb.MintWorke // checkActorOnNode refuses a mint from an atelet whose node does not host the // actor, as RequestActorSuspend does. Authoritative state decides, never the // request. The refusal is NotFound, so a node learns nothing about actors it -// does not host. -func (s *Server) checkActorOnNode(ctx context.Context, actor *ateapipb.Actor, callerNode string) error { +// does not host. credential names what the caller mints, for the refusal log. +func (s *Server) checkActorOnNode(ctx context.Context, actor *ateapipb.Actor, callerNode, credential string) error { hostingNode, err := controlapi.HostingNode(ctx, s.store, actor) if err != nil { return err } if hostingNode == "" || hostingNode != callerNode { - slog.WarnContext(ctx, "Refusing a worker-for-actor JWT for an actor the caller's node does not host", + slog.WarnContext(ctx, "Refusing a worker-for-actor "+credential+" for an actor the caller's node does not host", slog.String("actor", resources.ActorRefFromActor(actor).String()), slog.String("hosting_node", hostingNode), slog.String("caller_node", callerNode)) diff --git a/cmd/ateapi/internal/workerservice/jwt_test.go b/cmd/ateapi/internal/workerservice/jwt_test.go index 4f17f2b459..e3cb3e47dc 100644 --- a/cmd/ateapi/internal/workerservice/jwt_test.go +++ b/cmd/ateapi/internal/workerservice/jwt_test.go @@ -15,19 +15,21 @@ package workerservice import ( + "bytes" "context" "encoding/base64" "encoding/json" "errors" + "log/slog" "strings" "testing" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/ateletauth/ateletauthtest" - "github.com/agent-substrate/substrate/internal/actoridjwt" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/localjwtauthority" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/ateletauth/ateletauthtest" + "go.ate.dev/substrate/internal/actoridjwt" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/localjwtauthority" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" ) @@ -409,3 +411,62 @@ func TestMintWorkerActorJwt_Errors(t *testing.T) { }) } } + +// TestNodeRefusalLogNamesCredential checks that each mint's node refusal logs +// the credential that RPC mints. +func TestNodeRefusalLogNamesCredential(t *testing.T) { + var buf bytes.Buffer + prev := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil))) + t.Cleanup(func() { slog.SetDefault(prev) }) + + st, actor := newFakeStoreWithActor("team-a", "my-actor", "3b9f1e77-2c4d-4a80-91be-6d5c8f0a7e21") + csr, _ := generateTestCSR(t) + s := New(st, &fakeSuspender{}, testAteletSPIFFEID, newTestCAPool(t), testActorJWTIssuer, newTestJWTPool(t)) + // The actor's worker runs on node-1. + otherNode := ateletauthtest.ContextWith(ateletauthtest.CertOn(t, "node-2")) + actorRef := &ateapipb.ObjectRef{Atespace: "team-a", Name: "my-actor"} + + tests := []struct { + name string + mint func() error + want string + }{ + { + name: "MintWorkerActorJwt", + mint: func() error { + _, err := s.MintWorkerActorJwt(otherNode, &ateapipb.MintWorkerActorJwtRequest{ + Actor: actorRef, + ActorUid: actor.GetMetadata().GetUid(), + Audiences: []string{"snapshot-plugin"}, + }) + return err + }, + want: "Refusing a worker-for-actor JWT for an actor the caller's node does not host", + }, + { + name: "MintWorkerActorCertificate", + mint: func() error { + _, err := s.MintWorkerActorCertificate(otherNode, &ateapipb.MintWorkerActorCertificateRequest{ + Actor: actorRef, + ActorUid: actor.GetMetadata().GetUid(), + CertificateSigningRequest: csr, + Worker: &ateapipb.ObjectRef{Name: "worker-1"}, + }) + return err + }, + want: "Refusing a worker-for-actor certificate for an actor the caller's node does not host", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + buf.Reset() + if got := apierror.Code(tc.mint()); got != codes.NotFound { + t.Fatalf("code = %v, want %v", got, codes.NotFound) + } + if !strings.Contains(buf.String(), tc.want) { + t.Errorf("log = %q, want it to contain %q", buf.String(), tc.want) + } + }) + } +} diff --git a/cmd/ateapi/internal/workerservice/main_test.go b/cmd/ateapi/internal/workerservice/main_test.go index 04d3bc84b9..906f13b8b5 100644 --- a/cmd/ateapi/internal/workerservice/main_test.go +++ b/cmd/ateapi/internal/workerservice/main_test.go @@ -17,7 +17,7 @@ package workerservice import ( "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" ) func TestMain(m *testing.M) { diff --git a/cmd/ateapi/internal/workerservice/register.go b/cmd/ateapi/internal/workerservice/register.go index e17f090548..625e8c7e6b 100644 --- a/cmd/ateapi/internal/workerservice/register.go +++ b/cmd/ateapi/internal/workerservice/register.go @@ -22,12 +22,12 @@ import ( "log/slog" "slices" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/apivalidation" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/ateletauth" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/apivalidation" + "go.ate.dev/substrate/cmd/ateapi/internal/ateletauth" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" ) diff --git a/cmd/ateapi/internal/workerservice/register_test.go b/cmd/ateapi/internal/workerservice/register_test.go index 2209d25668..2dec5e845d 100644 --- a/cmd/ateapi/internal/workerservice/register_test.go +++ b/cmd/ateapi/internal/workerservice/register_test.go @@ -23,11 +23,11 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" "google.golang.org/protobuf/testing/protocmp" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/ateletauth/ateletauthtest" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/ateletauth/ateletauthtest" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" ) diff --git a/cmd/ateapi/internal/workerservice/report.go b/cmd/ateapi/internal/workerservice/report.go new file mode 100644 index 0000000000..15888f5b11 --- /dev/null +++ b/cmd/ateapi/internal/workerservice/report.go @@ -0,0 +1,43 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package workerservice + +import ( + "context" + + "go.ate.dev/substrate/cmd/ateapi/internal/apivalidation" + "go.ate.dev/substrate/cmd/ateapi/internal/ateletauth" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" +) + +// ReportActorSnapshotState records the snapshot state that the calling atelet +// reports for an Actor on its node. +func (s *Server) ReportActorSnapshotState(ctx context.Context, req *ateapipb.ReportActorSnapshotStateRequest) (*ateapipb.ReportActorSnapshotStateResponse, error) { + // Authz doesn't cover WorkerService, so authenticate before reading the + // request. + caller, err := ateletauth.Authenticate(ctx, s.ateletSPIFFEID) + if err != nil { + return nil, err + } + if errs := apivalidation.ValidateReportActorSnapshotStateRequest(ctx, req); len(errs) > 0 { + return nil, resources.ToAPIError(errs) + } + actorRef := resources.ActorRefFromObjectRef(req.GetActor()) + if err := s.suspender.ApplyActorSnapshotState(ctx, actorRef, req.GetActorUid(), caller.NodeName, req.GetState()); err != nil { + return nil, err + } + return &ateapipb.ReportActorSnapshotStateResponse{}, nil +} diff --git a/cmd/ateapi/internal/workerservice/report_test.go b/cmd/ateapi/internal/workerservice/report_test.go new file mode 100644 index 0000000000..e11447e7e3 --- /dev/null +++ b/cmd/ateapi/internal/workerservice/report_test.go @@ -0,0 +1,106 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package workerservice + +import ( + "context" + "testing" + + "google.golang.org/grpc/codes" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" + + "go.ate.dev/substrate/cmd/ateapi/internal/ateletauth/ateletauthtest" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" +) + +const reportActorUID = "5f2c1a90-7b34-4e6d-8a11-0c3e9d5b7f42" + +func reportRequest() *ateapipb.ReportActorSnapshotStateRequest { + durable := &ateapipb.SnapshotReport{ + Uuid: "snap-1", + Generation: 1, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: "gs://bucket/actors/a/snap-1"}, + LocalCommitTime: timestamppb.Now(), + } + return &ateapipb.ReportActorSnapshotStateRequest{ + Actor: &ateapipb.ObjectRef{Atespace: suspendAtespace, Name: suspendActor}, + ActorUid: reportActorUID, + State: &ateapipb.ActorSnapshotState{LatestDurable: durable}, + } +} + +// A report reaches the control plane with the node from the caller's +// certificate. The handler reads nothing from the store. +func TestReportActorSnapshotState(t *testing.T) { + suspender := &fakeSuspender{} + s := New(nil, suspender, testAteletSPIFFEID, nil, "", nil) + + req := reportRequest() + if _, err := s.ReportActorSnapshotState(ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)), req); err != nil { + t.Fatalf("ReportActorSnapshotState() failed: %v", err) + } + if len(suspender.reports) != 1 { + t.Fatalf("got %d applied reports, want 1", len(suspender.reports)) + } + got := suspender.reports[0] + if want := (resources.ActorRef{Atespace: suspendAtespace, Name: suspendActor}); got.actorRef != want { + t.Errorf("actorRef = %v, want %v", got.actorRef, want) + } + if got.actorUID != reportActorUID || got.node != testNode { + t.Errorf("applied (uid %q, node %q), want (%q, %q)", got.actorUID, got.node, reportActorUID, testNode) + } + if !proto.Equal(got.state, req.GetState()) { + t.Errorf("applied state = %v, want %v", got.state, req.GetState()) + } +} + +// A caller that isn't atelet, or a malformed request, doesn't reach the control +// plane. A refusal from the control plane reaches atelet unchanged. +func TestReportActorSnapshotState_Refused(t *testing.T) { + onNode := ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)) + invalid := reportRequest() + invalid.State = nil + + tests := []struct { + name string + ctx context.Context + req *ateapipb.ReportActorSnapshotStateRequest + controlErr error + want codes.Code + wantApplied bool + }{ + {name: "unauthenticated", ctx: ateletauthtest.ContextWith(nil), req: reportRequest(), want: codes.Unauthenticated}, + {name: "not atelet", ctx: ateletauthtest.ContextWith(ateletauthtest.CertIn(t, "other-namespace", testNode)), req: reportRequest(), want: codes.PermissionDenied}, + {name: "invalid request", ctx: onNode, req: invalid, want: codes.InvalidArgument}, + {name: "control plane refuses", ctx: onNode, req: reportRequest(), controlErr: apierror.NotFound("Actor not found"), want: codes.NotFound, wantApplied: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + suspender := &fakeSuspender{reportErr: tt.controlErr} + s := New(nil, suspender, testAteletSPIFFEID, nil, "", nil) + _, err := s.ReportActorSnapshotState(tt.ctx, tt.req) + if got := apierror.Code(err); got != tt.want { + t.Fatalf("ReportActorSnapshotState() error = %v, want code %v", err, tt.want) + } + if applied := len(suspender.reports) > 0; applied != tt.wantApplied { + t.Errorf("report reached the control plane = %v, want %v", applied, tt.wantApplied) + } + }) + } +} diff --git a/cmd/ateapi/internal/workerservice/service.go b/cmd/ateapi/internal/workerservice/service.go index 435b1113ff..09ae137522 100644 --- a/cmd/ateapi/internal/workerservice/service.go +++ b/cmd/ateapi/internal/workerservice/service.go @@ -19,10 +19,11 @@ package workerservice import ( "context" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/localca" - "github.com/agent-substrate/substrate/internal/localjwtauthority" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/localca" + "go.ate.dev/substrate/internal/localjwtauthority" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // Server implements ateapipb.WorkerServiceServer. @@ -54,6 +55,7 @@ type Server struct { // drives. *controlapi.RPCService satisfies it. type actorSuspender interface { SuspendActor(ctx context.Context, req *ateapipb.SuspendActorRequest) (*ateapipb.SuspendActorResponse, error) + ApplyActorSnapshotState(ctx context.Context, actorRef resources.ActorRef, actorUID, node string, state *ateapipb.ActorSnapshotState) error } var _ ateapipb.WorkerServiceServer = (*Server)(nil) diff --git a/cmd/ateapi/internal/workerservice/service_test.go b/cmd/ateapi/internal/workerservice/service_test.go index 8e26beebdc..0c29c10103 100644 --- a/cmd/ateapi/internal/workerservice/service_test.go +++ b/cmd/ateapi/internal/workerservice/service_test.go @@ -18,9 +18,10 @@ import ( "context" "testing" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // The Worker every test in this package calls about, and the node its atelet @@ -41,6 +42,9 @@ type fakeSuspender struct { calls []*ateapipb.SuspendActorRequest resp *ateapipb.SuspendActorResponse err error + + reports []appliedReport + reportErr error } func (f *fakeSuspender) SuspendActor(_ context.Context, req *ateapipb.SuspendActorRequest) (*ateapipb.SuspendActorResponse, error) { @@ -48,6 +52,19 @@ func (f *fakeSuspender) SuspendActor(_ context.Context, req *ateapipb.SuspendAct return f.resp, f.err } +func (f *fakeSuspender) ApplyActorSnapshotState(_ context.Context, actorRef resources.ActorRef, actorUID, node string, state *ateapipb.ActorSnapshotState) error { + f.reports = append(f.reports, appliedReport{actorRef: actorRef, actorUID: actorUID, node: node, state: state}) + return f.reportErr +} + +// appliedReport is one ApplyActorSnapshotState call. +type appliedReport struct { + actorRef resources.ActorRef + actorUID string + node string + state *ateapipb.ActorSnapshotState +} + // seedReportedWorker registers a Worker on nodeName that has already reported // capacity and its default runtime: identity from the pod, the way the syncer // writes it, plus the result of an earlier registration. A Worker that has diff --git a/cmd/ateapi/internal/workerservice/suspend.go b/cmd/ateapi/internal/workerservice/suspend.go index 702e6f6b51..35d1a4c3f7 100644 --- a/cmd/ateapi/internal/workerservice/suspend.go +++ b/cmd/ateapi/internal/workerservice/suspend.go @@ -20,12 +20,12 @@ import ( "fmt" "log/slog" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/apivalidation" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/ateletauth" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/apivalidation" + "go.ate.dev/substrate/cmd/ateapi/internal/ateletauth" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // RequestActorSuspend suspends an Actor on behalf of the Worker hosting it. The diff --git a/cmd/ateapi/internal/workerservice/suspend_test.go b/cmd/ateapi/internal/workerservice/suspend_test.go index 7fa4c4a679..83866ec88b 100644 --- a/cmd/ateapi/internal/workerservice/suspend_test.go +++ b/cmd/ateapi/internal/workerservice/suspend_test.go @@ -22,12 +22,12 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/protobuf/testing/protocmp" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/ateletauth/ateletauthtest" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/ateapi/internal/ateletauth/ateletauthtest" + "go.ate.dev/substrate/cmd/ateapi/internal/store" + "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) const ( diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index e7fb5b610d..ee8343dc5f 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -27,31 +27,32 @@ import ( "syscall" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/apiauthn" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/authz" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/controlapi" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/atepg" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/workerservice" - "github.com/agent-substrate/substrate/internal/ateinterceptors" - "github.com/agent-substrate/substrate/internal/credbundle" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/internal/localca" - "github.com/agent-substrate/substrate/internal/localjwtauthority" - "github.com/agent-substrate/substrate/internal/oidcdiscovery" - "github.com/agent-substrate/substrate/internal/oidcjwt" - "github.com/agent-substrate/substrate/internal/pluginsocket" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/serverboot" - "github.com/agent-substrate/substrate/internal/version" - "github.com/agent-substrate/substrate/internal/volume" - "github.com/agent-substrate/substrate/pkg/client/clientset/versioned" - "github.com/agent-substrate/substrate/pkg/client/informers/externalversions" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" "github.com/jackc/pgx/v5/pgconn" "github.com/spf13/pflag" + "go.ate.dev/substrate/cmd/ateapi/internal/apiauthn" + "go.ate.dev/substrate/cmd/ateapi/internal/apiconfig" + "go.ate.dev/substrate/cmd/ateapi/internal/authz" + "go.ate.dev/substrate/cmd/ateapi/internal/controlapi" + "go.ate.dev/substrate/cmd/ateapi/internal/store/atepg" + "go.ate.dev/substrate/cmd/ateapi/internal/workercache" + "go.ate.dev/substrate/cmd/ateapi/internal/workerservice" + "go.ate.dev/substrate/internal/ateinterceptors" + "go.ate.dev/substrate/internal/credbundle" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/localca" + "go.ate.dev/substrate/internal/localjwtauthority" + "go.ate.dev/substrate/internal/oidcdiscovery" + "go.ate.dev/substrate/internal/oidcjwt" + "go.ate.dev/substrate/internal/pluginsocket" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/serverboot" + "go.ate.dev/substrate/internal/version" + "go.ate.dev/substrate/internal/volume" + "go.ate.dev/substrate/pkg/client/clientset/versioned" + "go.ate.dev/substrate/pkg/client/informers/externalversions" + "go.ate.dev/substrate/pkg/proto/ateapipb" + objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.opentelemetry.io/otel" "google.golang.org/grpc" @@ -70,6 +71,8 @@ const maxRPCDeadline = 10 * time.Minute const minResyncInterval = 250 * time.Millisecond var ( + configFile = pflag.String("config-file", "", "API server config file") + listenAddr = pflag.String("grpc-listen-addr", ":443", "Address and port the gRPC server should listen on.") metricsListenAddr = pflag.String("metrics-listen-addr", ":9090", "Address and port the prometheus metrics server should listen on.") healthListenAddr = pflag.String("health-listen-addr", ":9091", "Address and port the /readyz and /healthz probe server should listen on.") @@ -112,30 +115,47 @@ var ( actorWorkflowDeadline = pflag.Duration("actor-workflow-deadline", 5*time.Minute, "Maximum wall-clock duration of a single Resume/Suspend workflow; raise it for slow image registries.") showVersion = pflag.Bool("version", false, "Print version and exit.") - logLevelFlag = pflag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.") previewFlags = pflag.StringSlice("preview", nil, "Preview gates to enable.") ) func main() { pflag.Parse() + if *showVersion { fmt.Println(version.String()) return } + ctx := context.Background() + serverboot.InitLogger() - if err := serverboot.SetLogLevel(*logLevelFlag); err != nil { - serverboot.Fatal(ctx, "Invalid --log-level", err) + + // Load unified config. + if *configFile == "" { + serverboot.Fatal(ctx, "--config-file must be specified", nil) + } + + config, err := apiconfig.LoadConfig(*configFile) + if err != nil { + serverboot.Fatal(ctx, "Error while loading config file", err) + } + + if err := serverboot.SetLogLevel(config.GetLogging().GetLevel()); err != nil { + serverboot.Fatal(ctx, "Invalid log level in config file", err) } + if err := loadFlagsFromEnv(); err != nil { serverboot.Fatal(ctx, "Invalid PostgreSQL configuration", err) } + if err := preview.Init(*previewFlags...); err != nil { serverboot.Fatal(ctx, "Invalid --preview", err) } + if err := rejectStorageEnv(); err != nil { serverboot.Fatal(ctx, "Storage settings moved to the objectstore-plugin sidecar", err) } + slog.InfoContext(ctx, "ateapi starting", slog.String("version", version.Version)) if *templateResyncInterval < minResyncInterval { serverboot.Fatal(ctx, "Invalid --template-resync-interval", fmt.Errorf("must be at least %s", minResyncInterval)) @@ -213,6 +233,18 @@ func main() { // AccessPolicy, so the enforced API would be unusable. serverboot.Fatal(ctx, "Invalid flags", fmt.Errorf("--authz-bootstrap-owners must list at least one principal when --experimental-enable-authz is set")) } + // atelet shares ateapi's namespace in every supported deployment topology, + // so we read it from Kubernetes' downward API rather than expose a flag. + ateletNamespace := installdefaults.NamespaceFromPodEnv() + // An empty ServiceAccount would not fail here: path.Join drops the empty + // segment, yielding an identity that parses but matches nothing, so every + // atelet dial would be rejected with no hint at the cause. + if *ateletServiceAccount == "" { + serverboot.Fatal(ctx, "Invalid flags", fmt.Errorf("--atelet-service-account must not be empty")) + } + ateletSPIFFEID := installdefaults.SPIFFEID(ateletNamespace, *ateletServiceAccount) + slog.InfoContext(ctx, "Resolved atelet namespace", slog.String("atelet-namespace", ateletNamespace), slog.String("atelet-spiffe-id", ateletSPIFFEID)) + fgaServer, err := authz.NewOpenFGAServer(pool) if err != nil { serverboot.Fatal(ctx, "Failed to create OpenFGA server", err) @@ -222,6 +254,7 @@ func main() { ControllerSPIFFEID: *controllerSPIFFEID, EgressGatewaySPIFFEID: *egressGatewaySPIFFEID, IngressRouterSPIFFEID: *ingressRouterSPIFFEID, + AteletSPIFFEID: ateletSPIFFEID, }) if err != nil { serverboot.Fatal(ctx, "Failed to initialize OpenFGA authz", err) @@ -248,18 +281,6 @@ func main() { sandboxConfigLister := ateFactory.Api().V1alpha1().SandboxConfigs().Lister() csiDriverConfigLister := ateFactory.Api().V1alpha1().CSIDriverConfigs().Lister() - // atelet shares ateapi's namespace in every supported deployment topology, - // so we read it from Kubernetes' downward API rather than expose a flag. - ateletNamespace := installdefaults.NamespaceFromPodEnv() - // An empty ServiceAccount would not fail here: path.Join drops the empty - // segment, yielding an identity that parses but matches nothing, so every - // atelet dial would be rejected with no hint at the cause. - if *ateletServiceAccount == "" { - serverboot.Fatal(ctx, "Invalid flags", fmt.Errorf("--atelet-service-account must not be empty")) - } - ateletSPIFFEID := installdefaults.SPIFFEID(ateletNamespace, *ateletServiceAccount) - slog.InfoContext(ctx, "Resolved atelet namespace", slog.String("atelet-namespace", ateletNamespace), slog.String("atelet-spiffe-id", ateletSPIFFEID)) - ateletPodInformerFactory, ateletPodInformer := controlapi.AteletInformer(clientset, ateletNamespace) scInformerFactory := informers.NewSharedInformerFactory(clientset, 0) storageClassLister := scInformerFactory.Storage().V1().StorageClasses().Lister() @@ -453,9 +474,6 @@ func loadFlagsFromEnv() error { *postgresPoolMaxConns = int32(value) } } - if v := os.Getenv("ATE_API_EXPERIMENTAL_ENABLE_AUTHZ"); v != "" && !pflag.CommandLine.Changed("experimental-enable-authz") { - *experimentalEnableAuthz = (v == "true" || v == "1") - } return nil } @@ -650,7 +668,7 @@ func buildJWTProviders(ctx context.Context, cfg *apiauthn.AuthenticationConfig) if err != nil { return apiauthn.ServerConfig{}, fmt.Errorf("initialize JWT provider %q: %w", providerCfg.Name, err) } - verifier := oidcjwt.NewVerifier(providerCfg.Issuer, providerCfg.Audiences, httpClient) + verifier := oidcjwt.NewVerifier(providerCfg.Issuer, providerCfg.Audiences, oidcjwt.HTTPKeySource(providerCfg.Issuer, httpClient)) serverCfg.JWTProviders = append(serverCfg.JWTProviders, apiauthn.JWTProvider{ Name: providerCfg.Name, Issuer: providerCfg.Issuer, diff --git a/cmd/ateapi/main_test.go b/cmd/ateapi/main_test.go index 83050c3e84..1231be9215 100644 --- a/cmd/ateapi/main_test.go +++ b/cmd/ateapi/main_test.go @@ -49,24 +49,20 @@ func TestConnectStoreRequiresPostgresReadWriteConnectionString(t *testing.T) { func TestLoadFlagsFromEnvResolvesPostgresSourcesOnce(t *testing.T) { oldRuntime, oldDDL := *postgresReadWriteConnectionString, *postgresOwnerConnectionString oldRuntimeRole, oldDDLRole := *postgresReadWriteRole, *postgresOwnerRole - oldAuthz := *experimentalEnableAuthz t.Cleanup(func() { *postgresReadWriteConnectionString = oldRuntime *postgresOwnerConnectionString = oldDDL *postgresReadWriteRole = oldRuntimeRole *postgresOwnerRole = oldDDLRole - *experimentalEnableAuthz = oldAuthz }) *postgresReadWriteConnectionString = "@env" *postgresOwnerConnectionString = "@env" *postgresReadWriteRole = "@env" *postgresOwnerRole = "@env" - *experimentalEnableAuthz = false t.Setenv("ATE_API_POSTGRES_READ_WRITE_CONNECTION_STRING", "runtime-a") t.Setenv("ATE_API_POSTGRES_OWNER_CONNECTION_STRING", "ddl-a") t.Setenv("ATE_API_POSTGRES_READ_WRITE_ROLE", "runtime-role") t.Setenv("ATE_API_POSTGRES_OWNER_ROLE", "ddl-role") - t.Setenv("ATE_API_EXPERIMENTAL_ENABLE_AUTHZ", "true") if err := loadFlagsFromEnv(); err != nil { t.Fatal(err) @@ -75,9 +71,6 @@ func TestLoadFlagsFromEnvResolvesPostgresSourcesOnce(t *testing.T) { *postgresReadWriteRole != "runtime-role" || *postgresOwnerRole != "ddl-role" { t.Fatalf("resolved values = %q, %q, %q, %q", *postgresReadWriteConnectionString, *postgresOwnerConnectionString, *postgresReadWriteRole, *postgresOwnerRole) } - if !*experimentalEnableAuthz { - t.Fatal("authorization environment flag was not resolved alongside PostgreSQL settings") - } t.Setenv("ATE_API_POSTGRES_READ_WRITE_CONNECTION_STRING", "runtime-b") t.Setenv("ATE_API_POSTGRES_OWNER_CONNECTION_STRING", "ddl-b") if err := loadFlagsFromEnv(); err != nil { diff --git a/cmd/atecontroller/internal/controllers/egressmitmtrust_controller.go b/cmd/atecontroller/internal/controllers/egressmitmtrust_controller.go index dae71830aa..2829539756 100644 --- a/cmd/atecontroller/internal/controllers/egressmitmtrust_controller.go +++ b/cmd/atecontroller/internal/controllers/egressmitmtrust_controller.go @@ -20,8 +20,8 @@ import ( "fmt" "strings" - "github.com/agent-substrate/substrate/internal/clustertrustbundle" - "github.com/agent-substrate/substrate/internal/localca" + "go.ate.dev/substrate/internal/clustertrustbundle" + "go.ate.dev/substrate/internal/localca" certsv1 "k8s.io/api/certificates/v1" certsv1beta1 "k8s.io/api/certificates/v1beta1" corev1 "k8s.io/api/core/v1" diff --git a/cmd/atecontroller/internal/controllers/egressmitmtrust_controller_test.go b/cmd/atecontroller/internal/controllers/egressmitmtrust_controller_test.go index 43e1e6ddcf..aa818267ff 100644 --- a/cmd/atecontroller/internal/controllers/egressmitmtrust_controller_test.go +++ b/cmd/atecontroller/internal/controllers/egressmitmtrust_controller_test.go @@ -21,9 +21,9 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/clustertrustbundle" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/internal/localca" + "go.ate.dev/substrate/internal/clustertrustbundle" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/localca" certsv1 "k8s.io/api/certificates/v1" certsv1beta1 "k8s.io/api/certificates/v1beta1" corev1 "k8s.io/api/core/v1" diff --git a/cmd/atecontroller/internal/controllers/networkpolicy_controller.go b/cmd/atecontroller/internal/controllers/networkpolicy_controller.go index 22c65bbbdd..a1f6b13a19 100644 --- a/cmd/atecontroller/internal/controllers/networkpolicy_controller.go +++ b/cmd/atecontroller/internal/controllers/networkpolicy_controller.go @@ -27,8 +27,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" - "github.com/agent-substrate/substrate/internal/resources" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/internal/resources" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" ) const ( diff --git a/cmd/atecontroller/internal/controllers/networkpolicy_controller_test.go b/cmd/atecontroller/internal/controllers/networkpolicy_controller_test.go index 9d939ea33c..2e07170943 100644 --- a/cmd/atecontroller/internal/controllers/networkpolicy_controller_test.go +++ b/cmd/atecontroller/internal/controllers/networkpolicy_controller_test.go @@ -21,8 +21,8 @@ import ( networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/types" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/resources" ) func TestWorkerPoolCreatesNetworkPolicy(t *testing.T) { diff --git a/cmd/atecontroller/internal/controllers/sandboxconfig_protection_controller.go b/cmd/atecontroller/internal/controllers/sandboxconfig_protection_controller.go index ac0f937b39..355caeb90c 100644 --- a/cmd/atecontroller/internal/controllers/sandboxconfig_protection_controller.go +++ b/cmd/atecontroller/internal/controllers/sandboxconfig_protection_controller.go @@ -25,7 +25,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" ) // workerPoolSandboxConfigIndex names the cache index from a SandboxConfig's diff --git a/cmd/atecontroller/internal/controllers/sandboxconfig_protection_controller_test.go b/cmd/atecontroller/internal/controllers/sandboxconfig_protection_controller_test.go index bd2bf5648f..0357664891 100644 --- a/cmd/atecontroller/internal/controllers/sandboxconfig_protection_controller_test.go +++ b/cmd/atecontroller/internal/controllers/sandboxconfig_protection_controller_test.go @@ -29,7 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" ) // TestSandboxConfigProtectionReconcile checks that the reconciler never adds diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply.go b/cmd/atecontroller/internal/controllers/workerpool_apply.go index 728e106a9a..4238d19f9c 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply.go @@ -26,12 +26,12 @@ import ( corev1ac "k8s.io/client-go/applyconfigurations/core/v1" metav1ac "k8s.io/client-go/applyconfigurations/meta/v1" - "github.com/agent-substrate/substrate/internal/deviceplugin" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/workercapacity" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/internal/deviceplugin" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/workercapacity" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" ) // workerOTelResourceAttributes mirrors atelet.yaml; service.instance.id is the pod @@ -551,7 +551,7 @@ func applyWorkerPoolPodTemplate( podSpecAC.NodeSelector = map[string]string{} podSpecAC.Tolerations = []corev1ac.TolerationApplyConfiguration{} podSpecAC.WithPriorityClassName("") - podSpecAC.WithServiceAccountName("default") + podSpecAC.WithServiceAccountName(installdefaults.WorkerServiceAccount) podSpecAC.WithAffinity(corev1ac.Affinity()) resourcesAC := corev1ac.ResourceRequirements() containerAC.WithResources(resourcesAC) diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go index 237811ba14..80f28e239d 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go @@ -30,13 +30,13 @@ import ( metav1ac "k8s.io/client-go/applyconfigurations/meta/v1" "k8s.io/utils/ptr" - "github.com/agent-substrate/substrate/internal/deviceplugin" - "github.com/agent-substrate/substrate/internal/egresspolicy" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/workercapacity" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/internal/deviceplugin" + "go.ate.dev/substrate/internal/egresspolicy" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/workercapacity" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" ) func TestBuildDeploymentApplyConfig(t *testing.T) { @@ -905,7 +905,7 @@ func expectedDeploymentApplyConfig(mutatePodSpec func(*corev1ac.PodSpecApplyConf sandboxClassTolerationAC(atev1alpha1.SandboxClassGvisor), } podSpecAC.WithPriorityClassName("") - podSpecAC.WithServiceAccountName("default") + podSpecAC.WithServiceAccountName("ateworker") podSpecAC.WithAffinity(corev1ac.Affinity()) podSpecAC.WithTerminationGracePeriodSeconds(workerTerminationGracePeriodSeconds) if mutatePodSpec != nil { diff --git a/cmd/atecontroller/internal/controllers/workerpool_controller.go b/cmd/atecontroller/internal/controllers/workerpool_controller.go index 873f5b5126..96cb8d4f1b 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_controller.go +++ b/cmd/atecontroller/internal/controllers/workerpool_controller.go @@ -29,6 +29,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + corev1ac "k8s.io/client-go/applyconfigurations/core/v1" "k8s.io/client-go/tools/events" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -37,8 +38,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "github.com/agent-substrate/substrate/internal/ateattr" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/installdefaults" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" ) const ( @@ -97,6 +99,7 @@ type WorkerPoolReconciler struct { //+kubebuilder:rbac:groups=ate.dev,resources=workerpools/status,verbs=get;update;patch //+kubebuilder:rbac:groups=ate.dev,resources=workerpools/finalizers,verbs=update //+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=create;patch //+kubebuilder:rbac:groups=ate.dev,resources=sandboxconfigs,verbs=get;list;watch;patch //+kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch @@ -317,6 +320,11 @@ func (r *WorkerPoolReconciler) recordSandboxConfigEvents(wp *atev1alpha1.WorkerP } func (r *WorkerPoolReconciler) applyDeployment(ctx context.Context, wp *atev1alpha1.WorkerPool) error { + if wp.Spec.Template == nil || wp.Spec.Template.ServiceAccountName == nil { + if err := r.applyWorkerServiceAccount(ctx, wp.Namespace); err != nil { + return err + } + } depAC := buildDeploymentApplyConfig(wp, workerOTelSettings{ Endpoint: r.OTelEndpoint, MetricExportInterval: r.OTelMetricExportInterval, @@ -331,6 +339,16 @@ func (r *WorkerPoolReconciler) applyDeployment(ctx context.Context, wp *atev1alp return nil } +// applyWorkerServiceAccount creates the default ServiceAccount worker pods run +// as. The pools in a namespace share it, so none of them owns it. +func (r *WorkerPoolReconciler) applyWorkerServiceAccount(ctx context.Context, namespace string) error { + saAC := corev1ac.ServiceAccount(installdefaults.WorkerServiceAccount, namespace) + if err := r.Apply(ctx, saAC, client.FieldOwner(workerPoolFieldOwner), client.ForceOwnership); err != nil { + return fmt.Errorf("failed to apply worker ServiceAccount: %w", err) + } + return nil +} + // syncStatus writes wp's status: replica counts and selector from dep (left // as they are when dep is nil) and sandboxClasses from classes. func (r *WorkerPoolReconciler) syncStatus(ctx context.Context, wp *atev1alpha1.WorkerPool, dep *appsv1.Deployment, classes []atev1alpha1.WorkerPoolSandboxClassStatus) error { diff --git a/cmd/atecontroller/internal/controllers/workerpool_controller_test.go b/cmd/atecontroller/internal/controllers/workerpool_controller_test.go index f7e97579c1..4760d13f5e 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_controller_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_controller_test.go @@ -43,10 +43,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/internal/testenv" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/testenv" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" ) var ( @@ -390,6 +390,66 @@ func sampleWorkerPoolPodTemplate() *atev1alpha1.WorkerPoolPodTemplate { } } +// TestWorkerPoolRunsWorkersAsWorkerServiceAccount checks that a pool naming no +// ServiceAccount runs its workers as the default one, which the controller +// creates. +func TestWorkerPoolRunsWorkersAsWorkerServiceAccount(t *testing.T) { + t.Parallel() + ctx := t.Context() + wp := makeWorkerPool("test-worker-sa", "default", 1, "worker:v1") + if err := k8sClient.Create(ctx, wp); err != nil { + t.Fatalf("create WorkerPool: %v", err) + } + deleteOnCleanup(t, wp) + + eventually(t, func(ctx context.Context) (bool, error) { + dep, err := getDeployment(ctx, wp) + if err != nil { + return false, nil + } + return dep.Spec.Template.Spec.ServiceAccountName == installdefaults.WorkerServiceAccount, nil + }) + // Applied before the Deployment, so it exists by now. + sa := &corev1.ServiceAccount{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: installdefaults.WorkerServiceAccount, Namespace: wp.Namespace}, sa); err != nil { + t.Fatalf("get worker ServiceAccount: %v", err) + } + // Pools share it, so deleting one must not delete it. + if len(sa.OwnerReferences) != 0 { + t.Errorf("worker ServiceAccount has owners %v, want none", sa.OwnerReferences) + } +} + +// TestWorkerPoolWithServiceAccountSkipsDefault checks that a pool naming its own +// ServiceAccount does not get the default one created. +func TestWorkerPoolWithServiceAccountSkipsDefault(t *testing.T) { + t.Parallel() + ctx := t.Context() + // A namespace of its own, so no other pool creates the default account. + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test-custom-worker-sa"}} + if err := client.IgnoreAlreadyExists(k8sClient.Create(ctx, ns)); err != nil { + t.Fatalf("create namespace: %v", err) + } + wp := makeWorkerPool("test-custom-sa", ns.Name, 1, "worker:v1") + wp.Spec.Template = &atev1alpha1.WorkerPoolPodTemplate{ServiceAccountName: ptr.To("substrate-worker")} + if err := k8sClient.Create(ctx, wp); err != nil { + t.Fatalf("create WorkerPool: %v", err) + } + deleteOnCleanup(t, wp) + + eventually(t, func(ctx context.Context) (bool, error) { + dep, err := getDeployment(ctx, wp) + if err != nil { + return false, nil + } + return dep.Spec.Template.Spec.ServiceAccountName == "substrate-worker", nil + }) + err := k8sClient.Get(ctx, types.NamespacedName{Name: installdefaults.WorkerServiceAccount, Namespace: ns.Name}, &corev1.ServiceAccount{}) + if !k8errors.IsNotFound(err) { + t.Errorf("get default worker ServiceAccount: %v, want NotFound", err) + } +} + // TestWorkerPoolPodTemplatePropagation verifies that template fields propagate // to the managed Deployment pod template. func TestWorkerPoolPodTemplatePropagation(t *testing.T) { @@ -547,7 +607,7 @@ func TestWorkerPoolPodTemplateClearAll(t *testing.T) { len(podSpec.Tolerations) == 1 && podSpec.Tolerations[0].Key == sandboxClassTaintKey && podSpec.PriorityClassName == "" && - podSpec.ServiceAccountName == "default" && + podSpec.ServiceAccountName == "ateworker" && (podSpec.Affinity == nil || podSpec.Affinity.NodeAffinity == nil) && len(container.Resources.Limits) == 0 && len(container.Resources.Requests) == 0, nil diff --git a/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig.go b/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig.go index e9752713f9..1a23edd376 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig.go +++ b/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig.go @@ -22,7 +22,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" ) // resolveLatestSandboxConfig picks the SandboxConfig the spec of one sandbox diff --git a/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig_test.go b/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig_test.go index a1ac3b49f4..f01ba39756 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig_test.go @@ -42,7 +42,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/reconcile" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" ) // testGvisorDefault is the gvisor class default TestMain creates. diff --git a/cmd/atecontroller/internal/workersync/fakecontrol_test.go b/cmd/atecontroller/internal/workersync/fakecontrol_test.go index b614a9ae31..a225348d83 100644 --- a/cmd/atecontroller/internal/workersync/fakecontrol_test.go +++ b/cmd/atecontroller/internal/workersync/fakecontrol_test.go @@ -22,7 +22,7 @@ import ( "strconv" "sync" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" diff --git a/cmd/atecontroller/internal/workersync/syncer.go b/cmd/atecontroller/internal/workersync/syncer.go index 09eb6a3d3c..b38595d078 100644 --- a/cmd/atecontroller/internal/workersync/syncer.go +++ b/cmd/atecontroller/internal/workersync/syncer.go @@ -24,8 +24,9 @@ import ( "slices" "time" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/installdefaults" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" corev1 "k8s.io/api/core/v1" @@ -333,6 +334,7 @@ func (s *WorkerPoolSyncer) createOrUpdateWorker(ctx context.Context, key workerK Ips: podIPs(pod), WorkerPodUid: string(pod.UID), NodeName: pod.Spec.NodeName, + WorkerSpiffeId: workerSPIFFEID(pod), SandboxClass: string(pool.Spec.DefaultSandboxClass()), Labels: pool.GetLabels(), Epoch: podEpoch(pod), @@ -477,6 +479,11 @@ func podIPs(pod *corev1.Pod) []string { return ips } +// workerSPIFFEID returns the SPIFFE ID the podidentity signer issues to pod. +func workerSPIFFEID(pod *corev1.Pod) string { + return installdefaults.SPIFFEID(pod.Namespace, pod.Spec.ServiceAccountName) +} + // markWorkerDraining transitions a worker to STATE_DRAINING so the scheduler // stops routing new actors to it while its pod is Terminating. DrainWorker is // idempotent, so a worker already draining costs nothing. If the worker is diff --git a/cmd/atecontroller/internal/workersync/syncer_restart_test.go b/cmd/atecontroller/internal/workersync/syncer_restart_test.go index dd049d018f..ec540dd996 100644 --- a/cmd/atecontroller/internal/workersync/syncer_restart_test.go +++ b/cmd/atecontroller/internal/workersync/syncer_restart_test.go @@ -18,7 +18,7 @@ import ( "context" "testing" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" corev1 "k8s.io/api/core/v1" ) diff --git a/cmd/atecontroller/internal/workersync/syncer_test.go b/cmd/atecontroller/internal/workersync/syncer_test.go index 8d8b6d4494..dcc37ab113 100644 --- a/cmd/atecontroller/internal/workersync/syncer_test.go +++ b/cmd/atecontroller/internal/workersync/syncer_test.go @@ -23,10 +23,10 @@ import ( "testing" "time" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - atefake "github.com/agent-substrate/substrate/pkg/client/clientset/versioned/fake" - "github.com/agent-substrate/substrate/pkg/client/informers/externalversions" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + atefake "go.ate.dev/substrate/pkg/client/clientset/versioned/fake" + "go.ate.dev/substrate/pkg/client/informers/externalversions" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" corev1 "k8s.io/api/core/v1" @@ -555,6 +555,24 @@ func TestSyncer_CreateWorker_DualStack(t *testing.T) { } } +// atenet-router requires atunnel's certificate to carry the SPIFFE ID the +// worker registers with. +func TestSyncer_CreateWorker_SPIFFEID(t *testing.T) { + ctx := context.Background() + ns, podName, poolName := "ns-syncer-sa", "worker-unit-sa", "pool-sa" + api := newFakeControl() + s, pods, _ := setupReconcileTest(t, api, workerPool(ns, poolName, "gvisor", nil)) + pod := workerPod(ns, podName, poolName, testPodUID, "10.0.0.1") + pod.Spec.ServiceAccountName = "substrate-worker" + key := seedPod(t, pods, pod) + + mustReconcile(t, ctx, s, key) + want := "spiffe://cluster.local/ns/ns-syncer-sa/sa/substrate-worker" + if got := api.get(testPodUID).GetWorkerSpiffeId(); got != want { + t.Fatalf("worker SPIFFE ID = %q, want %q", got, want) + } +} + // A concurrent write between the syncer's read and its update loses the version // precondition. The key requeues, and the retry must re-read rather than replay // its stale copy — which is what lets the concurrent change and the syncer's own diff --git a/cmd/atecontroller/main.go b/cmd/atecontroller/main.go index 7053be27c4..e7345b4fc9 100644 --- a/cmd/atecontroller/main.go +++ b/cmd/atecontroller/main.go @@ -19,20 +19,20 @@ import ( "log/slog" "os" - "github.com/agent-substrate/substrate/cmd/atecontroller/internal/controllers" - "github.com/agent-substrate/substrate/cmd/atecontroller/internal/workersync" - "github.com/agent-substrate/substrate/internal/ateapiauth" - "github.com/agent-substrate/substrate/internal/egresspolicy" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/serverboot" - "github.com/agent-substrate/substrate/internal/version" - clientv1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/client/clientset/versioned" - "github.com/agent-substrate/substrate/pkg/client/informers/externalversions" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/go-logr/logr" "github.com/spf13/pflag" + "go.ate.dev/substrate/cmd/atecontroller/internal/controllers" + "go.ate.dev/substrate/cmd/atecontroller/internal/workersync" + "go.ate.dev/substrate/internal/ateapiauth" + "go.ate.dev/substrate/internal/egresspolicy" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/serverboot" + "go.ate.dev/substrate/internal/version" + clientv1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/pkg/client/clientset/versioned" + "go.ate.dev/substrate/pkg/client/informers/externalversions" + "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" corev1 "k8s.io/api/core/v1" diff --git a/cmd/atelet/deviceplugin.go b/cmd/atelet/deviceplugin.go index d4b3cce8b0..c63c42da47 100644 --- a/cmd/atelet/deviceplugin.go +++ b/cmd/atelet/deviceplugin.go @@ -22,7 +22,7 @@ import ( pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" - "github.com/agent-substrate/substrate/internal/deviceplugin" + "go.ate.dev/substrate/internal/deviceplugin" ) // hostDevRoot is where the node's /dev is mounted into atelet (see diff --git a/cmd/atelet/durabledir_test.go b/cmd/atelet/durabledir_test.go new file mode 100644 index 0000000000..90fce1215c --- /dev/null +++ b/cmd/atelet/durabledir_test.go @@ -0,0 +1,41 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "path/filepath" + "testing" +) + +// Chmod, not MkdirAll's mode: the umask narrows a fresh dir. +func TestPrepareDurableDirVolume(t *testing.T) { + path := filepath.Join(t.TempDir(), "vol") + if err := prepareDurableDirVolume(path); err != nil { + t.Fatal(err) + } + if got := dirMode(t, path); got != 0o777 { + t.Errorf("mode = %o, want 777", got) + } +} + +func dirMode(t *testing.T, path string) os.FileMode { + t.Helper() + fi, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + return fi.Mode().Perm() +} diff --git a/cmd/atelet/egresspolicy.go b/cmd/atelet/egresspolicy.go index 97769f38f7..a3456cd352 100644 --- a/cmd/atelet/egresspolicy.go +++ b/cmd/atelet/egresspolicy.go @@ -15,8 +15,8 @@ package main import ( - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // toAteletEgressPolicy copies the destinations each rule of the control diff --git a/cmd/atelet/egresspolicy_test.go b/cmd/atelet/egresspolicy_test.go index a1dcec0744..477a0e64c5 100644 --- a/cmd/atelet/egresspolicy_test.go +++ b/cmd/atelet/egresspolicy_test.go @@ -17,9 +17,9 @@ package main import ( "testing" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/testing/protocmp" ) diff --git a/cmd/atelet/helm_test.go b/cmd/atelet/helm_test.go index 9929e9e81c..78e1f01246 100644 --- a/cmd/atelet/helm_test.go +++ b/cmd/atelet/helm_test.go @@ -22,7 +22,7 @@ import ( "strings" "testing" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" diff --git a/cmd/atelet/imagegc.go b/cmd/atelet/imagegc.go index af81d261d4..0325c9b74a 100644 --- a/cmd/atelet/imagegc.go +++ b/cmd/atelet/imagegc.go @@ -35,9 +35,9 @@ import ( "strings" "time" - "github.com/agent-substrate/substrate/internal/imagecache" - "github.com/agent-substrate/substrate/internal/nodepath" "github.com/spf13/pflag" + "go.ate.dev/substrate/internal/imagecache" + "go.ate.dev/substrate/internal/nodepath" "golang.org/x/sys/unix" ) diff --git a/cmd/atelet/imagegc_test.go b/cmd/atelet/imagegc_test.go index 07ac664ec8..5555768af1 100644 --- a/cmd/atelet/imagegc_test.go +++ b/cmd/atelet/imagegc_test.go @@ -25,8 +25,8 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/imagecache" - "github.com/agent-substrate/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/imagecache" + "go.ate.dev/substrate/internal/nodepath" ) func TestImageCacheGCTarget(t *testing.T) { diff --git a/cmd/atelet/imagevolume_test.go b/cmd/atelet/imagevolume_test.go index d39c42332c..387688a823 100644 --- a/cmd/atelet/imagevolume_test.go +++ b/cmd/atelet/imagevolume_test.go @@ -26,8 +26,6 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/internal/imagecache" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/registry" v1 "github.com/google/go-containerregistry/pkg/v1" @@ -35,6 +33,8 @@ import ( "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/tarball" + "go.ate.dev/substrate/internal/imagecache" + "go.ate.dev/substrate/internal/proto/ateletpb" ) // imageVolumeTestRegistry starts an in-memory OCI registry. Its 127.0.0.1 host diff --git a/cmd/atelet/internal/apivalidation/doc.go b/cmd/atelet/internal/apivalidation/doc.go index 42a40299a0..bbb6bca873 100644 --- a/cmd/atelet/internal/apivalidation/doc.go +++ b/cmd/atelet/internal/apivalidation/doc.go @@ -19,7 +19,7 @@ // work. // // +k8s:validation-gen=TypesWithSuffix=Request -// +k8s:validation-gen-input=github.com/agent-substrate/substrate/internal/proto/ateletpb +// +k8s:validation-gen-input=go.ate.dev/substrate/internal/proto/ateletpb // +k8s:validation-gen-scheme-registry=nil // +k8s:validation-gen-deep-equal-func=ateDeepEqual package apivalidation diff --git a/cmd/atelet/internal/apivalidation/validate.go b/cmd/atelet/internal/apivalidation/validate.go index 63f9f143b2..17097a8959 100644 --- a/cmd/atelet/internal/apivalidation/validate.go +++ b/cmd/atelet/internal/apivalidation/validate.go @@ -18,8 +18,8 @@ import ( "context" "fmt" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" "k8s.io/apimachinery/pkg/api/operation" "k8s.io/apimachinery/pkg/util/validation/field" ) @@ -65,6 +65,11 @@ func ValidatePromoteSnapshotRequest(ctx context.Context, req *ateletpb.PromoteSn return Validate_PromoteSnapshotRequest(ctx, operation.Operation{Type: operation.Create}, nil, req, nil) } +// ValidateDetachActorRequest runs the generated validation for req. +func ValidateDetachActorRequest(ctx context.Context, req *ateletpb.DetachActorRequest) field.ErrorList { + return Validate_DetachActorRequest(ctx, operation.Operation{Type: operation.Create}, nil, req, nil) +} + // ValidateRegisterWorkerRequest runs the generated validation for req. func ValidateRegisterWorkerRequest(ctx context.Context, req *ateletpb.RegisterWorkerRequest) field.ErrorList { return Validate_RegisterWorkerRequest(ctx, operation.Operation{Type: operation.Create}, nil, req, nil) diff --git a/cmd/atelet/internal/apivalidation/validation_test.go b/cmd/atelet/internal/apivalidation/validation_test.go index d84212e8cb..2e81af37e9 100644 --- a/cmd/atelet/internal/apivalidation/validation_test.go +++ b/cmd/atelet/internal/apivalidation/validation_test.go @@ -20,8 +20,8 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" "k8s.io/apimachinery/pkg/api/operation" "k8s.io/apimachinery/pkg/util/validation/field" ) @@ -1068,6 +1068,58 @@ func TestValidateTerminateRequest(t *testing.T) { } } +func TestValidateDetachActorRequest(t *testing.T) { + valid := func(mutate ...func(*ateletpb.DetachActorRequest)) *ateletpb.DetachActorRequest { + r := &ateletpb.DetachActorRequest{ + Atespace: "team-a", + ActorName: "actor-1", + ActorUid: "01234567-89ab-cdef-0123-456789abcdef", + } + for _, m := range mutate { + m(r) + } + return r + } + + tests := []struct { + name string + obj *ateletpb.DetachActorRequest + want field.ErrorList + }{{ + name: "valid", + obj: valid(), + }, { + name: "missing atespace", + obj: valid(func(r *ateletpb.DetachActorRequest) { r.Atespace = "" }), + want: field.ErrorList{field.Required(field.NewPath("atespace"), "")}, + }, { + name: "invalid atespace: uppercase", + obj: valid(func(r *ateletpb.DetachActorRequest) { r.Atespace = "Team-A" }), + want: field.ErrorList{field.Invalid(field.NewPath("atespace"), nil, "").WithOrigin("format=k8s-short-name")}, + }, { + name: "missing actor_name", + obj: valid(func(r *ateletpb.DetachActorRequest) { r.ActorName = "" }), + want: field.ErrorList{field.Required(field.NewPath("actor_name"), "")}, + }, { + name: "invalid actor_name: path escape", + obj: valid(func(r *ateletpb.DetachActorRequest) { r.ActorName = "../escape" }), + want: field.ErrorList{field.Invalid(field.NewPath("actor_name"), nil, "").WithOrigin("format=k8s-short-name")}, + }, { + name: "missing actor_uid", + obj: valid(func(r *ateletpb.DetachActorRequest) { r.ActorUid = "" }), + want: field.ErrorList{field.Required(field.NewPath("actor_uid"), "")}, + }, { + name: "invalid actor_uid: not a uuid", + obj: valid(func(r *ateletpb.DetachActorRequest) { r.ActorUid = "not-a-uuid" }), + want: field.ErrorList{field.Invalid(field.NewPath("actor_uid"), nil, "").WithOrigin("format=k8s-uuid")}, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assertValidateErr(t, ValidateDetachActorRequest(context.Background(), tt.obj), tt.want) + }) + } +} + func TestValidateRegisterWorkerRequest(t *testing.T) { testRuntime := func() *ateletpb.SandboxRuntime { return &ateletpb.SandboxRuntime{ diff --git a/cmd/atelet/internal/apivalidation/workloadspec.go b/cmd/atelet/internal/apivalidation/workloadspec.go index c0e3f94693..e3cfc456cf 100644 --- a/cmd/atelet/internal/apivalidation/workloadspec.go +++ b/cmd/atelet/internal/apivalidation/workloadspec.go @@ -22,8 +22,8 @@ package apivalidation import ( "context" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" "k8s.io/apimachinery/pkg/api/operation" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" diff --git a/cmd/atelet/internal/apivalidation/zz_generated.validation.go b/cmd/atelet/internal/apivalidation/zz_generated.validation.go index 8cbdbae51c..333cac6d1c 100644 --- a/cmd/atelet/internal/apivalidation/zz_generated.validation.go +++ b/cmd/atelet/internal/apivalidation/zz_generated.validation.go @@ -22,7 +22,7 @@ package apivalidation import ( context "context" - ateletpb "github.com/agent-substrate/substrate/internal/proto/ateletpb" + ateletpb "go.ate.dev/substrate/internal/proto/ateletpb" operation "k8s.io/apimachinery/pkg/api/operation" safe "k8s.io/apimachinery/pkg/api/safe" validate "k8s.io/apimachinery/pkg/api/validate" @@ -1102,6 +1102,111 @@ func Validate_Container( return errs } +// Validate_DetachActorRequest validates an instance of DetachActorRequest according +// to declarative validation rules in the API schema. +func Validate_DetachActorRequest( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateletpb.DetachActorRequest) (errs field.ErrorList) { + + { // field ateletpb.DetachActorRequest.Atespace + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.DetachActorRequest) *string { + return &oldObj.Atespace + }) + errs = append(errs, fn(fldPath.Child("atespace"), &obj.Atespace, oldVal, oldObj != nil)...) + } + + { // field ateletpb.DetachActorRequest.ActorName + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.DetachActorRequest) *string { + return &oldObj.ActorName + }) + errs = append(errs, fn(fldPath.Child("actor_name"), &obj.ActorName, oldVal, oldObj != nil)...) + } + + { // field ateletpb.DetachActorRequest.ActorUid + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.UUID(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.DetachActorRequest) *string { + return &oldObj.ActorUid + }) + errs = append(errs, fn(fldPath.Child("actor_uid"), &obj.ActorUid, oldVal, oldObj != nil)...) + } + + return errs +} + // Validate_EgressGateway validates an instance of EgressGateway according // to declarative validation rules in the API schema. func Validate_EgressGateway( @@ -3510,7 +3615,7 @@ func Validate_Snapshot( return errs } -var unionMembershipFor_github_com_agent_substrate_substrate_internal_proto_ateletpb_SystemInfoDataSource_ = validate.NewUnionMembership(validate.NewUnionMember("actor_metadata"), validate.NewUnionMember("trust_bundle")) +var unionMembershipFor_go_ate_dev_substrate_internal_proto_ateletpb_SystemInfoDataSource_ = validate.NewUnionMembership(validate.NewUnionMember("actor_metadata"), validate.NewUnionMember("trust_bundle")) // Validate_SystemInfoDataSource validates an instance of SystemInfoDataSource according // to declarative validation rules in the API schema. @@ -3518,7 +3623,7 @@ func Validate_SystemInfoDataSource( ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *ateletpb.SystemInfoDataSource) (errs field.ErrorList) { - if e := validate.Union(ctx, op, fldPath, obj, oldObj, unionMembershipFor_github_com_agent_substrate_substrate_internal_proto_ateletpb_SystemInfoDataSource_, + if e := validate.Union(ctx, op, fldPath, obj, oldObj, unionMembershipFor_go_ate_dev_substrate_internal_proto_ateletpb_SystemInfoDataSource_, func(obj *ateletpb.SystemInfoDataSource) bool { if obj == nil { return false @@ -4110,7 +4215,7 @@ func Validate_VersionedSandboxCompat( return errs } -var unionMembershipFor_github_com_agent_substrate_substrate_internal_proto_ateletpb_Volume_ = validate.NewUnionMembership(validate.NewUnionMember("durable_dir"), validate.NewUnionMember("external"), validate.NewUnionMember("system_info"), validate.NewUnionMember("image")) +var unionMembershipFor_go_ate_dev_substrate_internal_proto_ateletpb_Volume_ = validate.NewUnionMembership(validate.NewUnionMember("durable_dir"), validate.NewUnionMember("external"), validate.NewUnionMember("system_info"), validate.NewUnionMember("image")) // Validate_Volume validates an instance of Volume according // to declarative validation rules in the API schema. @@ -4118,7 +4223,7 @@ func Validate_Volume( ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *ateletpb.Volume) (errs field.ErrorList) { - if e := validate.Union(ctx, op, fldPath, obj, oldObj, unionMembershipFor_github_com_agent_substrate_substrate_internal_proto_ateletpb_Volume_, + if e := validate.Union(ctx, op, fldPath, obj, oldObj, unionMembershipFor_go_ate_dev_substrate_internal_proto_ateletpb_Volume_, func(obj *ateletpb.Volume) bool { if obj == nil { return false diff --git a/cmd/atelet/internal/ateletpath/ateletpath.go b/cmd/atelet/internal/ateletpath/ateletpath.go index fdae2ec031..283c94307f 100644 --- a/cmd/atelet/internal/ateletpath/ateletpath.go +++ b/cmd/atelet/internal/ateletpath/ateletpath.go @@ -19,8 +19,8 @@ package ateletpath import ( "path/filepath" - "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/proto/ateworkerpb" ) var ( diff --git a/cmd/atelet/internal/filecache/getfilecopyto.go b/cmd/atelet/internal/filecache/getfilecopyto.go index d3c49acc99..1f82de67c5 100644 --- a/cmd/atelet/internal/filecache/getfilecopyto.go +++ b/cmd/atelet/internal/filecache/getfilecopyto.go @@ -22,7 +22,7 @@ import ( "os" "time" - "github.com/agent-substrate/substrate/cmd/atelet/internal/sparsefile" + "go.ate.dev/substrate/cmd/atelet/internal/sparsefile" ) // GetFileCopyTo materializes the artifact identified by key at dst as a diff --git a/cmd/atelet/internal/filecache/getfileto_test.go b/cmd/atelet/internal/filecache/getfileto_test.go index 25c0d993d6..168641ed84 100644 --- a/cmd/atelet/internal/filecache/getfileto_test.go +++ b/cmd/atelet/internal/filecache/getfileto_test.go @@ -365,6 +365,9 @@ func TestGetFileToHitTouchesLastUse(t *testing.T) { } func TestGetFileToPublishedCopyIsWriteProtected(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root ignores file permissions, so the write cannot fail") + } s := newTestStore(t) key := URIKey("gs://bucket/golden", "mem.img") fetch, _ := countingFetcher("precious") diff --git a/cmd/atelet/internal/snapshottracker/actor.go b/cmd/atelet/internal/snapshottracker/actor.go index ec4ecbb724..b7be0e9c36 100644 --- a/cmd/atelet/internal/snapshottracker/actor.go +++ b/cmd/atelet/internal/snapshottracker/actor.go @@ -18,7 +18,7 @@ import ( "encoding/json" "time" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/proto/ateletpb" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" diff --git a/cmd/atelet/internal/snapshottracker/background_test.go b/cmd/atelet/internal/snapshottracker/background_test.go index 4c4f40c005..eeb0842794 100644 --- a/cmd/atelet/internal/snapshottracker/background_test.go +++ b/cmd/atelet/internal/snapshottracker/background_test.go @@ -23,9 +23,9 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/atelet/internal/ateletpath" - "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/proto/ateletpb" "google.golang.org/protobuf/proto" clocktesting "k8s.io/utils/clock/testing" ) diff --git a/cmd/atelet/internal/snapshottracker/lifecycle.go b/cmd/atelet/internal/snapshottracker/lifecycle.go index fde83307b7..3a35b3ddd2 100644 --- a/cmd/atelet/internal/snapshottracker/lifecycle.go +++ b/cmd/atelet/internal/snapshottracker/lifecycle.go @@ -15,7 +15,7 @@ package snapshottracker import ( - "github.com/agent-substrate/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/proto/ateletpb" ) // CommitLocal records snap, just stored locally under UPLOAD_MODE_DEFERRED, as diff --git a/cmd/atelet/internal/snapshottracker/state.go b/cmd/atelet/internal/snapshottracker/state.go index 1d9eef3d92..1184f6ec3b 100644 --- a/cmd/atelet/internal/snapshottracker/state.go +++ b/cmd/atelet/internal/snapshottracker/state.go @@ -21,9 +21,9 @@ import ( "log/slog" "os" - "github.com/agent-substrate/substrate/cmd/atelet/internal/ateletpath" - "github.com/agent-substrate/substrate/cmd/atelet/internal/atomicfile" - "github.com/agent-substrate/substrate/internal/nodepath" + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/cmd/atelet/internal/atomicfile" + "go.ate.dev/substrate/internal/nodepath" ) // saveLocked writes what atelet tracks of the actor to its snapshot state diff --git a/cmd/atelet/internal/snapshottracker/tracker.go b/cmd/atelet/internal/snapshottracker/tracker.go index 50a4045296..a973f22eb0 100644 --- a/cmd/atelet/internal/snapshottracker/tracker.go +++ b/cmd/atelet/internal/snapshottracker/tracker.go @@ -20,8 +20,8 @@ import ( "context" "sync" - "github.com/agent-substrate/substrate/internal/actorlock" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/actorlock" + "go.ate.dev/substrate/internal/proto/ateletpb" "k8s.io/client-go/util/workqueue" "k8s.io/utils/clock" ) diff --git a/cmd/atelet/internal/snapshottracker/tracker_test.go b/cmd/atelet/internal/snapshottracker/tracker_test.go index c8e2ec72d0..997bc0e04e 100644 --- a/cmd/atelet/internal/snapshottracker/tracker_test.go +++ b/cmd/atelet/internal/snapshottracker/tracker_test.go @@ -22,9 +22,9 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/atelet/internal/ateletpath" - "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/proto/ateletpb" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" clocktesting "k8s.io/utils/clock/testing" diff --git a/cmd/atelet/internal/trustbundle/trustbundle.go b/cmd/atelet/internal/trustbundle/trustbundle.go index 7abbea9ee8..9c1d00fc76 100644 --- a/cmd/atelet/internal/trustbundle/trustbundle.go +++ b/cmd/atelet/internal/trustbundle/trustbundle.go @@ -21,7 +21,7 @@ import ( "fmt" "math/rand/v2" - "github.com/agent-substrate/substrate/internal/pemutil" + "go.ate.dev/substrate/internal/pemutil" certsv1 "k8s.io/api/certificates/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" ) diff --git a/cmd/atelet/lifecycle_test.go b/cmd/atelet/lifecycle_test.go index 12ce8befb0..cdf83b1087 100644 --- a/cmd/atelet/lifecycle_test.go +++ b/cmd/atelet/lifecycle_test.go @@ -27,14 +27,14 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/cmd/atelet/internal/ateletpath" - "github.com/agent-substrate/substrate/cmd/atelet/internal/snapshottracker" - "github.com/agent-substrate/substrate/cmd/atelet/internal/trustbundle" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/cmd/atelet/internal/snapshottracker" + "go.ate.dev/substrate/cmd/atelet/internal/trustbundle" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/resources" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/protobuf/proto" diff --git a/cmd/atelet/local_checkpoints.go b/cmd/atelet/local_checkpoints.go index 1e2ff89b25..642122f700 100644 --- a/cmd/atelet/local_checkpoints.go +++ b/cmd/atelet/local_checkpoints.go @@ -22,7 +22,7 @@ import ( "os" "path/filepath" - "github.com/agent-substrate/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" ) // pruneLocalCheckpoints removes every local snapshot of the actor. A missing diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 489d952a9a..82c54cabb2 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -30,41 +30,41 @@ import ( "syscall" "time" - "github.com/agent-substrate/substrate/cmd/atelet/internal/apivalidation" - "github.com/agent-substrate/substrate/cmd/atelet/internal/ateletpath" - "github.com/agent-substrate/substrate/cmd/atelet/internal/credentialprovider" - "github.com/agent-substrate/substrate/cmd/atelet/internal/snapshottracker" - "github.com/agent-substrate/substrate/cmd/atelet/internal/sparsefile" - "github.com/agent-substrate/substrate/cmd/atelet/internal/trustbundle" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateapiauth" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/ateinterceptors" - "github.com/agent-substrate/substrate/internal/atelet" - "github.com/agent-substrate/substrate/internal/clustertrustbundle" - "github.com/agent-substrate/substrate/internal/credbundle" - "github.com/agent-substrate/substrate/internal/imagecache" - "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/ocispec" - "github.com/agent-substrate/substrate/internal/otlprelay" - "github.com/agent-substrate/substrate/internal/pluginsocket" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/serverboot" - "github.com/agent-substrate/substrate/internal/substratex509" - "github.com/agent-substrate/substrate/internal/version" - "github.com/agent-substrate/substrate/internal/volume" - "github.com/agent-substrate/substrate/internal/volume/csi" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/client/clientset/versioned" - "github.com/agent-substrate/substrate/pkg/client/informers/externalversions" - "github.com/agent-substrate/substrate/pkg/objectstorage" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" "github.com/google/go-containerregistry/pkg/authn" "github.com/spf13/pflag" + "go.ate.dev/substrate/cmd/atelet/internal/apivalidation" + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/cmd/atelet/internal/credentialprovider" + "go.ate.dev/substrate/cmd/atelet/internal/snapshottracker" + "go.ate.dev/substrate/cmd/atelet/internal/sparsefile" + "go.ate.dev/substrate/cmd/atelet/internal/trustbundle" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateapiauth" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/ateinterceptors" + "go.ate.dev/substrate/internal/atelet" + "go.ate.dev/substrate/internal/clustertrustbundle" + "go.ate.dev/substrate/internal/credbundle" + "go.ate.dev/substrate/internal/imagecache" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/ocispec" + "go.ate.dev/substrate/internal/otlprelay" + "go.ate.dev/substrate/internal/pluginsocket" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/serverboot" + "go.ate.dev/substrate/internal/substratex509" + "go.ate.dev/substrate/internal/version" + "go.ate.dev/substrate/internal/volume" + "go.ate.dev/substrate/internal/volume/csi" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/pkg/client/clientset/versioned" + "go.ate.dev/substrate/pkg/client/informers/externalversions" + "go.ate.dev/substrate/pkg/objectstorage" + "go.ate.dev/substrate/pkg/proto/ateapipb" + objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/metric" @@ -432,15 +432,16 @@ func main() { if _, err := startSandboxAssetPrewarm(ctx, sandboxConfigInformer, wmService, imageCache, microvmNodeCapable(hostDevRoot)); err != nil { slog.ErrorContext(ctx, "Sandbox asset prewarm disabled", slog.Any("err", err)) } - // Keep a sandbox versions file in each local worker's directory: the - // runtime hashes of the enabled versions of its pool's SandboxConfig, + // Keep a sandbox runtimes file in each local worker's directory: the + // runtimes of the enabled versions of its pool's SandboxConfig, // rewritten when the config changes. if _, err := startWorkerSandboxSyncer(ctx, workerPodFactory.Core().V1().Pods(), ateFactory.Api().V1alpha1().WorkerPools(), ateFactory.Api().V1alpha1().SandboxConfigs(), + wmService, nodepath.WorkersDir()); err != nil { - slog.ErrorContext(ctx, "Worker sandbox versions syncer disabled", slog.Any("err", err)) + slog.ErrorContext(ctx, "Worker sandbox runtimes syncer disabled", slog.Any("err", err)) } // Start after the informers are registered: a factory only runs // informers that exist when Start is called. @@ -1606,6 +1607,20 @@ func (s *Atelet) downloadExternalCheckpoint(ctx context.Context, snapshotURI str return nil } +// prepareDurableDirVolume creates a durable-dir volume's host directory as +// 0777, as Kubernetes creates an emptyDir: one actor's containers can run as +// different users. Chmod, because MkdirAll applies the umask. A restore then +// re-applies the mode the snapshot recorded. +func prepareDurableDirVolume(path string) error { + if err := os.MkdirAll(path, 0o777); err != nil { + return fmt.Errorf("while creating %q: %w", path, err) + } + if err := os.Chmod(path, 0o777); err != nil { + return fmt.Errorf("while setting mode of %q: %w", path, err) + } + return nil +} + // prepareOCIBundles pulls images and assembles OCI bundles for the pause // container and every application container in spec, in parallel. pauseImage // comes from the sandbox record, not the workload spec: it is sandbox @@ -1623,9 +1638,8 @@ func (s *Atelet) prepareOCIBundles( // Prepare host folders for volume types that need them. for _, vol := range spec.GetVolumes() { if vol.GetDurableDir() != nil { - volPath := ateletpath.DurableDirVolumeMountPoint(actorUID, vol.GetName()) - if err := os.MkdirAll(volPath, 0o700); err != nil { - return nil, fmt.Errorf("while creating %q: %w", volPath, err) + if err := prepareDurableDirVolume(ateletpath.DurableDirVolumeMountPoint(actorUID, vol.GetName())); err != nil { + return nil, err } } } diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index 4939569c84..1bc2e6941e 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -33,20 +33,20 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/atelet/internal/ateletpath" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/atelet" - "github.com/agent-substrate/substrate/internal/imagecache" - "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/ocispec" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/serverboot" - "github.com/agent-substrate/substrate/pkg/objectstorage" "github.com/google/go-cmp/cmp" "github.com/spf13/pflag" + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/atelet" + "go.ate.dev/substrate/internal/imagecache" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/ocispec" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/serverboot" + "go.ate.dev/substrate/pkg/objectstorage" "golang.org/x/sys/unix" "google.golang.org/grpc" "google.golang.org/grpc/codes" diff --git a/cmd/atelet/metrics.go b/cmd/atelet/metrics.go index 7433497b8b..d7d65c2359 100644 --- a/cmd/atelet/metrics.go +++ b/cmd/atelet/metrics.go @@ -22,9 +22,9 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" ) const ( diff --git a/cmd/atelet/metrics_test.go b/cmd/atelet/metrics_test.go index 4792a8f44c..51508cd6ee 100644 --- a/cmd/atelet/metrics_test.go +++ b/cmd/atelet/metrics_test.go @@ -26,9 +26,9 @@ import ( sdkmetric "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/metricdata" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" ) const ( diff --git a/cmd/atelet/oci.go b/cmd/atelet/oci.go index cc39c97d34..32e4f39125 100644 --- a/cmd/atelet/oci.go +++ b/cmd/atelet/oci.go @@ -22,10 +22,10 @@ import ( "sort" "strings" - "github.com/agent-substrate/substrate/cmd/atelet/internal/ateletpath" - "github.com/agent-substrate/substrate/internal/imagecache" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" v1 "github.com/google/go-containerregistry/pkg/v1" + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/internal/imagecache" + "go.ate.dev/substrate/internal/proto/ateletpb" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "golang.org/x/sync/errgroup" @@ -109,7 +109,7 @@ func prepareOCIDirectory(ctx context.Context, imageCache *imagecache.Store, acto g.Go(func() error { var err error if img, err = imageCache.EnsureImage(gctx, ref); err != nil { - return fmt.Errorf("in imageCache.EnsureImage: %w", err) + return fmt.Errorf("in imageCache.EnsureImage(%q): %w", ref, err) } return nil }) @@ -163,9 +163,10 @@ func resolveImageVolumes(ctx context.Context, imageCache *imagecache.Store, volu g, gctx := errgroup.WithContext(ctx) for i, vol := range wanted { g.Go(func() error { - img, err := imageCache.EnsureImage(gctx, vol.GetImage().GetReference()) + volRef := vol.GetImage().GetReference() + img, err := imageCache.EnsureImage(gctx, volRef) if err != nil { - return fmt.Errorf("in imageCache.EnsureImage for volume %q: %w", vol.GetName(), err) + return fmt.Errorf("in imageCache.EnsureImage(%q) for volume %q: %w", volRef, vol.GetName(), err) } out[i] = imagecache.ImageVolumeOverlay{ Name: vol.GetName(), diff --git a/cmd/atelet/oci_test.go b/cmd/atelet/oci_test.go index 988534b4a0..c76d2d0d03 100644 --- a/cmd/atelet/oci_test.go +++ b/cmd/atelet/oci_test.go @@ -18,8 +18,8 @@ import ( "slices" "testing" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" v1 "github.com/google/go-containerregistry/pkg/v1" + "go.ate.dev/substrate/internal/proto/ateletpb" ) func TestResolveActorEnv(t *testing.T) { diff --git a/cmd/atelet/rpc_logging_test.go b/cmd/atelet/rpc_logging_test.go index f505f04b41..74f52c604a 100644 --- a/cmd/atelet/rpc_logging_test.go +++ b/cmd/atelet/rpc_logging_test.go @@ -22,10 +22,10 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/protoredact" - "github.com/agent-substrate/substrate/internal/serverboot" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/protoredact" + "go.ate.dev/substrate/internal/serverboot" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" diff --git a/cmd/atelet/sandbox_assets.go b/cmd/atelet/sandbox_assets.go index f4ec42da35..12b177a398 100644 --- a/cmd/atelet/sandbox_assets.go +++ b/cmd/atelet/sandbox_assets.go @@ -39,15 +39,15 @@ import ( "syscall" "time" - "github.com/agent-substrate/substrate/cmd/atelet/internal/ateletpath" - "github.com/agent-substrate/substrate/cmd/atelet/internal/atomicfile" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/objectstorage" - objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/cmd/atelet/internal/atomicfile" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/pkg/objectstorage" + objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -184,13 +184,19 @@ func (s *Atelet) ensureSandboxAssets(ctx context.Context, rec *sandboxAssetsReco return paths, nil } +// runscAssetNames are the assets that carry the gVisor `runsc` binary, most +// preferred first. +var runscAssetNames = []string{gvisorAssetName, runscAssetName} + // runscPathFor returns the local path of the gVisor `runsc` binary from a // fetched asset-path map, or "" if the runtime has none (e.g. micro-VM). func runscPathFor(paths map[string]string) string { - if p := paths[gvisorAssetName]; p != "" { - return p + for _, name := range runscAssetNames { + if p := paths[name]; p != "" { + return p + } } - return paths[runscAssetName] + return "" } // fetchAsset downloads one content-addressed asset (verifying its sha256) into diff --git a/cmd/atelet/sandbox_assets_plugin_test.go b/cmd/atelet/sandbox_assets_plugin_test.go index 1610706dec..f1521bc563 100644 --- a/cmd/atelet/sandbox_assets_plugin_test.go +++ b/cmd/atelet/sandbox_assets_plugin_test.go @@ -30,16 +30,16 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/atelet/internal/ateletpath" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateinterceptors" - "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/objectstoreplugin" - "github.com/agent-substrate/substrate/internal/objectstoreplugin/objectstoreplugintest" - "github.com/agent-substrate/substrate/internal/pluginsocket" - "github.com/agent-substrate/substrate/internal/serverboot" - "github.com/agent-substrate/substrate/pkg/objectstorage" - objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateinterceptors" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/objectstoreplugin" + "go.ate.dev/substrate/internal/objectstoreplugin/objectstoreplugintest" + "go.ate.dev/substrate/internal/pluginsocket" + "go.ate.dev/substrate/internal/serverboot" + "go.ate.dev/substrate/pkg/objectstorage" + objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/health" diff --git a/cmd/atelet/sandbox_assets_test.go b/cmd/atelet/sandbox_assets_test.go index 2e2d65bf8a..a99e709eda 100644 --- a/cmd/atelet/sandbox_assets_test.go +++ b/cmd/atelet/sandbox_assets_test.go @@ -25,9 +25,9 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/google/go-cmp/cmp" "github.com/klauspost/compress/zstd" + "go.ate.dev/substrate/internal/proto/ateletpb" ) func TestExtractTarArchive(t *testing.T) { diff --git a/cmd/atelet/sandbox_prewarm.go b/cmd/atelet/sandbox_prewarm.go index ab174bc1e0..12a722c4a2 100644 --- a/cmd/atelet/sandbox_prewarm.go +++ b/cmd/atelet/sandbox_prewarm.go @@ -28,9 +28,9 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" - "github.com/agent-substrate/substrate/internal/imagecache" - "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" + "go.ate.dev/substrate/internal/imagecache" + "go.ate.dev/substrate/pkg/api/v1alpha1" + listersv1alpha1 "go.ate.dev/substrate/pkg/client/listers/api/v1alpha1" ) // prewarmMaxJitter spreads the fleet's asset downloads after a SandboxConfig diff --git a/cmd/atelet/sandbox_prewarm_test.go b/cmd/atelet/sandbox_prewarm_test.go index 2a0731c415..2ddab7b718 100644 --- a/cmd/atelet/sandbox_prewarm_test.go +++ b/cmd/atelet/sandbox_prewarm_test.go @@ -32,19 +32,19 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/utils/ptr" - "github.com/agent-substrate/substrate/cmd/atelet/internal/ateletpath" - "github.com/agent-substrate/substrate/internal/imagecache" - "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/client/clientset/versioned/fake" - "github.com/agent-substrate/substrate/pkg/client/informers/externalversions" - listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/registry" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/remote" + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/internal/imagecache" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/pkg/client/clientset/versioned/fake" + "go.ate.dev/substrate/pkg/client/informers/externalversions" + listersv1alpha1 "go.ate.dev/substrate/pkg/client/listers/api/v1alpha1" ) // pushPauseImage pushes a tiny image to ref and returns its manifest digest, diff --git a/cmd/atelet/snapshot_transfer.go b/cmd/atelet/snapshot_transfer.go index e8c54a4648..83fdfa5b35 100644 --- a/cmd/atelet/snapshot_transfer.go +++ b/cmd/atelet/snapshot_transfer.go @@ -20,8 +20,8 @@ import ( "os" "path/filepath" - "github.com/agent-substrate/substrate/internal/pluginsocket" - objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "go.ate.dev/substrate/internal/pluginsocket" + objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" diff --git a/cmd/atelet/snapshot_transfer_test.go b/cmd/atelet/snapshot_transfer_test.go index fb34b56e26..e77712ebb5 100644 --- a/cmd/atelet/snapshot_transfer_test.go +++ b/cmd/atelet/snapshot_transfer_test.go @@ -27,16 +27,16 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/atelet/internal/snapshottracker" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/ateinterceptors" - "github.com/agent-substrate/substrate/internal/objectstoreplugin" - "github.com/agent-substrate/substrate/internal/objectstoreplugin/objectstoreplugintest" - "github.com/agent-substrate/substrate/internal/pluginsocket" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/objectstorage" - objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "go.ate.dev/substrate/cmd/atelet/internal/snapshottracker" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/ateinterceptors" + "go.ate.dev/substrate/internal/objectstoreplugin" + "go.ate.dev/substrate/internal/objectstoreplugin/objectstoreplugintest" + "go.ate.dev/substrate/internal/pluginsocket" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/objectstorage" + objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/connectivity" @@ -51,7 +51,7 @@ import ( func newPluginHerder(t *testing.T, store objectstorage.ObjectStorage) *Atelet { t.Helper() // Test directories come from t.TempDir, so admit any absolute path. - plugin, err := objectstoreplugin.NewNodePlugin(store, "/") + plugin, err := objectstoreplugin.NewNodePlugin(store, "/", objectstorage.DefaultUploadMemory) if err != nil { t.Fatal(err) } @@ -320,7 +320,7 @@ type pluginSidecar struct { func newPluginSidecar(t *testing.T, store objectstorage.ObjectStorage) *pluginSidecar { t.Helper() - plugin, err := objectstoreplugin.NewNodePlugin(store, "/") + plugin, err := objectstoreplugin.NewNodePlugin(store, "/", objectstorage.DefaultUploadMemory) if err != nil { t.Fatal(err) } diff --git a/cmd/atelet/snapshotlog.go b/cmd/atelet/snapshotlog.go index b4062adf09..2d201306b4 100644 --- a/cmd/atelet/snapshotlog.go +++ b/cmd/atelet/snapshotlog.go @@ -17,9 +17,9 @@ package main import ( "log/slog" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/resources" "google.golang.org/grpc/status" ) diff --git a/cmd/atelet/snapshotlog_test.go b/cmd/atelet/snapshotlog_test.go index cc51812e00..7afce5ed15 100644 --- a/cmd/atelet/snapshotlog_test.go +++ b/cmd/atelet/snapshotlog_test.go @@ -24,8 +24,8 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/resources" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) diff --git a/cmd/atelet/statspoller.go b/cmd/atelet/statspoller.go index 37de64876a..2a0530f101 100644 --- a/cmd/atelet/statspoller.go +++ b/cmd/atelet/statspoller.go @@ -36,10 +36,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/workerstats" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/workerstats" ) // workerPoolLabel is the label the pool controller stamps on every worker pod diff --git a/cmd/atelet/statspoller_test.go b/cmd/atelet/statspoller_test.go index c17067b58e..8d9a685cdd 100644 --- a/cmd/atelet/statspoller_test.go +++ b/cmd/atelet/statspoller_test.go @@ -34,7 +34,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8sfake "k8s.io/client-go/kubernetes/fake" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/proto/ateworkerpb" ) // fakeStatsWorker answers GetActiveWorkloadStats with a canned response or diff --git a/cmd/atelet/systeminfovolume.go b/cmd/atelet/systeminfovolume.go index 9f30782148..76d6964220 100644 --- a/cmd/atelet/systeminfovolume.go +++ b/cmd/atelet/systeminfovolume.go @@ -26,11 +26,11 @@ import ( "sync" "time" - "github.com/agent-substrate/substrate/cmd/atelet/internal/ateletpath" - "github.com/agent-substrate/substrate/cmd/atelet/internal/trustbundle" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/volumepath" + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/cmd/atelet/internal/trustbundle" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/volumepath" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/cache" diff --git a/cmd/atelet/systeminfovolume_test.go b/cmd/atelet/systeminfovolume_test.go index c5286c3b33..2a56ab9368 100644 --- a/cmd/atelet/systeminfovolume_test.go +++ b/cmd/atelet/systeminfovolume_test.go @@ -31,10 +31,10 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/atelet/internal/ateletpath" - "github.com/agent-substrate/substrate/cmd/atelet/internal/trustbundle" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/cmd/atelet/internal/trustbundle" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" certsv1 "k8s.io/api/certificates/v1" certsv1beta1 "k8s.io/api/certificates/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/cmd/atelet/volumes.go b/cmd/atelet/volumes.go index 149d89ac97..b9c573c939 100644 --- a/cmd/atelet/volumes.go +++ b/cmd/atelet/volumes.go @@ -21,10 +21,10 @@ import ( "log/slog" "os" - "github.com/agent-substrate/substrate/cmd/atelet/internal/ateletpath" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/volume" - "github.com/agent-substrate/substrate/internal/volume/csi" + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/volume" + "go.ate.dev/substrate/internal/volume/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) diff --git a/cmd/atelet/volumes_test.go b/cmd/atelet/volumes_test.go index 7e57e36a2b..46dc30e80d 100644 --- a/cmd/atelet/volumes_test.go +++ b/cmd/atelet/volumes_test.go @@ -23,13 +23,13 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/cmd/atelet/internal/ateletpath" - "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/volume" - "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/client/clientset/versioned/fake" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/volume" + "go.ate.dev/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/pkg/client/clientset/versioned/fake" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) diff --git a/cmd/atelet/worker_sandbox_syncer.go b/cmd/atelet/worker_sandbox_syncer.go index 1378d08034..0d25863ff9 100644 --- a/cmd/atelet/worker_sandbox_syncer.go +++ b/cmd/atelet/worker_sandbox_syncer.go @@ -21,10 +21,10 @@ package main // random delay when a pool or config change reaches every node at once // - process: for each UID, sync the pod, or remove its worker directory // once the pod is gone -// - finish: forget the UID, or retry it with backoff up to -// syncMaxRetries times +// - finish: forget the UID, or retry it with backoff, capped at +// syncMaxRetries while the pool's SandboxConfig is unresolved // -// worker_sandbox_versions.go builds the sandbox config version file for each +// worker_sandbox_versions.go builds the sandbox runtimes file for each // worker pod. import ( @@ -35,6 +35,7 @@ import ( "math/rand/v2" "os" "path/filepath" + "sync" "time" corev1 "k8s.io/api/core/v1" @@ -44,9 +45,10 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - informersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/informers/externalversions/api/v1alpha1" + "go.ate.dev/substrate/internal/hardware" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/api/v1alpha1" + informersv1alpha1 "go.ate.dev/substrate/pkg/client/informers/externalversions/api/v1alpha1" ) // podUIDIndex indexes this node's worker pods by UID, the syncer's queue key. @@ -70,40 +72,70 @@ const orphanMinAge = time.Minute // for the file. const syncMaxJitter = 30 * time.Second -// syncMaxRetries bounds how often one pod's failed sync is retried before it -// is dropped. The pod is synced again only when its pool switches to another -// SandboxConfig, the config's spec changes, or atelet restarts. The backoff -// doubles from one second, so this covers a few minutes of informer lag or an -// unresolved pool without retrying forever. +// syncMaxRetries bounds how often one pod's errUnresolved sync is retried +// before it is dropped. The backoff doubles from one second, so this covers a +// few minutes of informer lag or an unresolved pool without retrying forever; +// the informer events that end those also enqueue the pod again. Every other +// failure is exempt and keeps retrying at the rate limiter's longest backoff: +// an asset fetch or probe that fails for the default version, one that leaves +// a restorable version out (errIncomplete), or a write to the worker +// directory, since nothing enqueues the pod when those recover. const syncMaxRetries = 8 -// workerSandboxSyncer keeps a sandbox versions file in each local worker's +// syncWorkers is how many goroutines drain the sync queue in run. More than +// one keeps a slow asset fetch on one pod from blocking directory cleanup and +// cached syncs for other pods on the node. +const syncWorkers = 2 + +// workerSandboxSyncer keeps a sandbox runtimes file in each local worker's // directory under workersDir: the runtimes of the enabled versions of the // SandboxConfig the worker's pool uses. It creates a worker's directory once // it writes the pod's file and removes it when the pod is gone. type workerSandboxSyncer struct { resolver *workerSandboxResolver // pods holds this node's worker pods, indexed by podUIDIndex. - pods cache.Indexer - // queue carries pod UIDs. One worker drains it, so a pod's file is never - // written and its directory removed at the same time. + pods cache.Indexer + assets sandboxAssetFetcher + // queue carries pod UIDs. A workqueue never hands the same key to two + // workers at once, so a pod's file is never written and its directory + // removed at the same time. queue workqueue.TypedRateLimitingInterface[string] workersDir string // nodepath.WorkersDir(); a temp dir in tests minAge time.Duration maxJitter time.Duration + + // probeCPUIdentity is hardware.ProbeCPUIdentity. Overridable in tests. + probeCPUIdentity func() (vendor, model string, err error) + // runCPUFeatures is runRunscCPUFeatures. Overridable in tests. + runCPUFeatures func(ctx context.Context, runscPath string) ([]string, error) + // mu guards hostCPU and cpuFeatures across the syncWorkers in run: + // hostCPU is the probed host CPU identity, nil until a probe succeeds, + // and cpuFeatures the encoded cpu_features of each runsc binary by its + // gVisor asset's SHA256 digest. A binary is fetched and run once per + // digest, except that two workers meeting a digest before either has + // memoized it both probe it; the duplicate costs a fetch from the asset + // cache and a run of runsc, and both store the same mask. + mu sync.Mutex + hostCPU *cpuIdentity + cpuFeatures map[string]string } -func newWorkerSandboxSyncer(resolver *workerSandboxResolver, pods cache.Indexer, workersDir string) *workerSandboxSyncer { +func newWorkerSandboxSyncer(resolver *workerSandboxResolver, pods cache.Indexer, assets sandboxAssetFetcher, workersDir string) *workerSandboxSyncer { return &workerSandboxSyncer{ resolver: resolver, pods: pods, - // Failures are informer lag and unresolved pools, both on the scale - // of seconds to minutes. + assets: assets, + // Backoff doubles from 1s for informer lag and unresolved pools, and + // caps at 5m where uncapped retries for transient asset fetch or probe + // failures stay until they recover. queue: workqueue.NewTypedRateLimitingQueue( workqueue.NewTypedItemExponentialFailureRateLimiter[string](time.Second, 5*time.Minute)), - workersDir: workersDir, - minAge: orphanMinAge, - maxJitter: syncMaxJitter, + workersDir: workersDir, + minAge: orphanMinAge, + maxJitter: syncMaxJitter, + probeCPUIdentity: hardware.ProbeCPUIdentity, + runCPUFeatures: runRunscCPUFeatures, + cpuFeatures: make(map[string]string), } } @@ -115,7 +147,7 @@ func newWorkerSandboxSyncer(resolver *workerSandboxResolver, pods cache.Indexer, // The syncer never holds up atelet startup: until the informers sync, or // while a list/watch fails (e.g. Forbidden while an RBAC rollout lags the // binary), files wait and existing ones stay as they are. -func startWorkerSandboxSyncer(ctx context.Context, pods coreinformers.PodInformer, workerPools informersv1alpha1.WorkerPoolInformer, configs informersv1alpha1.SandboxConfigInformer, workersDir string) (*workerSandboxSyncer, error) { +func startWorkerSandboxSyncer(ctx context.Context, pods coreinformers.PodInformer, workerPools informersv1alpha1.WorkerPoolInformer, configs informersv1alpha1.SandboxConfigInformer, assets sandboxAssetFetcher, workersDir string) (*workerSandboxSyncer, error) { if err := pods.Informer().AddIndexers(cache.Indexers{podUIDIndex: func(obj any) ([]string, error) { pod, ok := obj.(*corev1.Pod) if !ok { @@ -129,7 +161,7 @@ func startWorkerSandboxSyncer(ctx context.Context, pods coreinformers.PodInforme workerPools: workerPools.Lister(), configs: configs.Lister(), } - s := newWorkerSandboxSyncer(resolver, pods.Informer().GetIndexer(), workersDir) + s := newWorkerSandboxSyncer(resolver, pods.Informer().GetIndexer(), assets, workersDir) if err := watch(ctx, "worker pod", pods.Informer(), s.podHandler()); err != nil { return nil, err } @@ -140,14 +172,14 @@ func startWorkerSandboxSyncer(ctx context.Context, pods coreinformers.PodInforme return nil, err } go s.run(ctx, pods.Informer().HasSynced, workerPools.Informer().HasSynced, configs.Informer().HasSynced) - slog.InfoContext(ctx, "Worker sandbox versions syncer started") + slog.InfoContext(ctx, "Worker sandbox runtimes syncer started") return s, nil } // watch registers handler on inf and logs inf's list/watch failures. func watch(ctx context.Context, kind string, inf cache.SharedIndexInformer, handler cache.ResourceEventHandler) error { if err := inf.SetWatchErrorHandler(func(_ *cache.Reflector, err error) { - slog.WarnContext(ctx, kind+" list/watch failed; workers on this node cannot update their sandbox versions until it recovers", slog.Any("err", err)) + slog.WarnContext(ctx, kind+" list/watch failed; workers on this node cannot update their sandbox runtimes until it recovers", slog.Any("err", err)) }); err != nil { // Fails only on an already-started informer, where the default // reflector logging applies. @@ -282,13 +314,21 @@ func (s *workerSandboxSyncer) run(ctx context.Context, synced ...cache.InformerS return } s.enqueueExistingDirs(ctx) - for { - uid, shutdown := s.queue.Get() - if shutdown { - return - } - s.process(ctx, uid) + var wg sync.WaitGroup + for range syncWorkers { + wg.Add(1) + go func() { + defer wg.Done() + for { + uid, shutdown := s.queue.Get() + if shutdown { + return + } + s.process(ctx, uid) + } + }() } + wg.Wait() } // process brings the directory of the pod with the given UID up to date: @@ -326,24 +366,28 @@ func (s *workerSandboxSyncer) podByUID(uid string) (*corev1.Pod, error) { return pod, nil } -// finish forgets uid on success or errPermanent, and otherwise retries it with -// backoff up to syncMaxRetries times. A sync that fails before it writes -// leaves the existing file in place. +// finish forgets uid on success or errPermanent, caps errUnresolved retries at +// syncMaxRetries, and otherwise requeues with backoff without a retry cap. A +// sync that fails before it writes leaves the existing file in place. func (s *workerSandboxSyncer) finish(ctx context.Context, uid string, err error) { switch { case err == nil: s.queue.Forget(uid) case errors.Is(err, errPermanent): - slog.WarnContext(ctx, "Cannot sync worker sandbox versions; not retrying until the pool's SandboxConfig changes or atelet restarts", slog.String("pod_uid", uid), slog.Any("err", err)) + slog.WarnContext(ctx, "Cannot sync worker sandbox runtimes; not retrying until the pool's SandboxConfig changes or atelet restarts", slog.String("pod_uid", uid), slog.Any("err", err)) s.queue.Forget(uid) case ctx.Err() != nil: // Shutting down, not a sync failure. - case s.queue.NumRequeues(uid) < syncMaxRetries: - slog.WarnContext(ctx, "Could not sync worker sandbox versions; will retry", + case errors.Is(err, errIncomplete): + slog.WarnContext(ctx, "Worker sandbox runtimes file is missing a restorable version; will retry", + slog.String("pod_uid", uid), slog.Int("retries", s.queue.NumRequeues(uid)), slog.Any("err", err)) + s.queue.AddRateLimited(uid) + case s.queue.NumRequeues(uid) < syncMaxRetries || !errors.Is(err, errUnresolved): + slog.WarnContext(ctx, "Could not sync worker sandbox runtimes; will retry", slog.String("pod_uid", uid), slog.Int("retries", s.queue.NumRequeues(uid)), slog.Any("err", err)) s.queue.AddRateLimited(uid) default: - slog.WarnContext(ctx, "Could not sync worker sandbox versions; giving up until the pool's SandboxConfig changes or atelet restarts", + slog.WarnContext(ctx, "Could not sync worker sandbox runtimes; giving up until the pool's SandboxConfig changes or atelet restarts", slog.String("pod_uid", uid), slog.Int("retries", syncMaxRetries), slog.Any("err", err)) s.queue.Forget(uid) } diff --git a/cmd/atelet/worker_sandbox_syncer_test.go b/cmd/atelet/worker_sandbox_syncer_test.go index 10557ca6f6..d335d2e473 100644 --- a/cmd/atelet/worker_sandbox_syncer_test.go +++ b/cmd/atelet/worker_sandbox_syncer_test.go @@ -19,7 +19,9 @@ import ( "errors" "os" "path/filepath" + "runtime" "slices" + "strings" "testing" "time" @@ -33,13 +35,73 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" - "github.com/agent-substrate/substrate/internal/sandboxversions" - "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - atefake "github.com/agent-substrate/substrate/pkg/client/clientset/versioned/fake" - "github.com/agent-substrate/substrate/pkg/client/informers/externalversions" - listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" + "go.ate.dev/substrate/internal/sandboxruntimes" + "go.ate.dev/substrate/pkg/api/v1alpha1" + atefake "go.ate.dev/substrate/pkg/client/clientset/versioned/fake" + "go.ate.dev/substrate/pkg/client/informers/externalversions" + listersv1alpha1 "go.ate.dev/substrate/pkg/client/listers/api/v1alpha1" ) +const ( + testCPUVendor = "GenuineIntel" + testCPUModel = "6:143" +) + +// testCPUFeatures are the first two CPU feature names for runtime.GOARCH, +// which encode to bit 0 | bit 1 = "3" on both amd64 and arm64. +func testCPUFeatures() []string { + switch runtime.GOARCH { + case "arm64": + return []string{"fp", "asimd"} + default: + return []string{"pni", "pclmulqdq"} + } +} + +type fakeAssetFetcher struct { + fn func(ctx context.Context, rec *sandboxAssetsRecord) (map[string]string, error) +} + +func (f *fakeAssetFetcher) ensureSandboxAssets(ctx context.Context, rec *sandboxAssetsRecord) (map[string]string, error) { + return f.fn(ctx, rec) +} + +// newTestAssetFetcher returns a sandboxAssetFetcher that maps every asset of +// a record to a path under its digest without fetching anything, for syncers +// whose runsc probe is faked. +func newTestAssetFetcher() sandboxAssetFetcher { + return &fakeAssetFetcher{ + fn: func(_ context.Context, rec *sandboxAssetsRecord) (map[string]string, error) { + paths := make(map[string]string, len(rec.Assets)) + for name, a := range rec.Assets { + paths[name] = "/cache/" + a.SHA256 + "/" + name + } + return paths, nil + }, + } +} + +// newScriptAssetFetcher returns a sandboxAssetFetcher that backs every asset +// with an executable script printing testCPUFeatures(), for a syncer that +// runs the real runsc probe. +func newScriptAssetFetcher(t *testing.T) sandboxAssetFetcher { + t.Helper() + script := filepath.Join(t.TempDir(), "runsc") + body := "#!/bin/sh\necho " + strings.Join(testCPUFeatures(), ",") + "\n" + if err := os.WriteFile(script, []byte(body), 0o755); err != nil { + t.Fatal(err) + } + return &fakeAssetFetcher{ + fn: func(_ context.Context, rec *sandboxAssetsRecord) (map[string]string, error) { + paths := make(map[string]string, len(rec.Assets)) + for name := range rec.Assets { + paths[name] = script + } + return paths, nil + }, + } +} + // workerPod is a worker pod on this node with the given UID, in pool when // pool is not "". func workerPod(uid, pool string) *corev1.Pod { @@ -53,18 +115,26 @@ func workerPod(uid, pool string) *corev1.Pod { // resolvedPool is a gVisor WorkerPool whose class resolved to config, or has // not resolved when config is "". func resolvedPool(name, config string) *v1alpha1.WorkerPool { + return resolvedPoolOf(v1alpha1.SandboxClassGvisor, name, config) +} + +// resolvedPoolOf is a WorkerPool of the given sandbox class whose class +// resolved to config, or has not resolved when config is "". +func resolvedPoolOf(class v1alpha1.SandboxClass, name, config string) *v1alpha1.WorkerPool { pool := &v1alpha1.WorkerPool{ObjectMeta: metav1.ObjectMeta{Namespace: "workers", Name: name}} - pool.Spec.SandboxClasses = []v1alpha1.WorkerPoolSandboxClass{{Name: v1alpha1.SandboxClassGvisor}} - class := v1alpha1.WorkerPoolSandboxClassStatus{Name: v1alpha1.SandboxClassGvisor} + pool.Spec.SandboxClasses = []v1alpha1.WorkerPoolSandboxClass{{Name: class}} + status := v1alpha1.WorkerPoolSandboxClassStatus{Name: class} if config != "" { - class.ConfigRef = &v1alpha1.SandboxConfigReference{Name: config} + status.ConfigRef = &v1alpha1.SandboxConfigReference{Name: config} } - pool.Status.SandboxClasses = []v1alpha1.WorkerPoolSandboxClassStatus{class} + pool.Status.SandboxClasses = []v1alpha1.WorkerPoolSandboxClassStatus{status} return pool } // newTestSyncer serves objs from synced caches and keeps worker directories -// under a temp dir. +// under a temp dir. Its probes are faked: the host CPU is +// testCPUVendor/testCPUModel and every runsc reports testCPUFeatures(); +// TestRunRunscCPUFeatures covers the real probe. func newTestSyncer(t *testing.T, objs ...any) *workerSandboxSyncer { t.Helper() newIndexer := func() cache.Indexer { @@ -94,7 +164,9 @@ func newTestSyncer(t *testing.T, objs ...any) *workerSandboxSyncer { s := newWorkerSandboxSyncer(&workerSandboxResolver{ workerPools: listersv1alpha1.NewWorkerPoolLister(pools), configs: listersv1alpha1.NewSandboxConfigLister(configs), - }, pods, t.TempDir()) + }, pods, newTestAssetFetcher(), t.TempDir()) + s.probeCPUIdentity = func() (string, string, error) { return testCPUVendor, testCPUModel, nil } + s.runCPUFeatures = func(context.Context, string) ([]string, error) { return testCPUFeatures(), nil } // Tests that check the jitter set it. s.maxJitter = 0 return s @@ -297,11 +369,13 @@ func TestWorkerSandboxSyncerProcess(t *testing.T) { name string objs []any // existing is written to pod-a's file before processing, when set. - existing *sandboxversions.File + existing *sandboxruntimes.File + // assets overrides s.assets when set. + assets sandboxAssetFetcher // priorRetries is how often pod-a has been retried before. priorRetries int // want is the file afterwards; nil means no file. - want *sandboxversions.File + want *sandboxruntimes.File wantDir bool // wantUntouched is whether the existing file must not be rewritten. wantUntouched bool @@ -354,6 +428,40 @@ func TestWorkerSandboxSyncerProcess(t *testing.T) { name: "unresolved pool gives up after syncMaxRetries", objs: []any{workerPod("pod-a", "pool"), multiVersionConfig()}, priorRetries: syncMaxRetries, + }, { + // A transient failure on the default version leaves the existing file + // in place and requeues without the syncMaxRetries cap, since no + // informer event enqueues the pod when an asset fetch recovers. + name: "transient failure on default version keeps existing file and requeues past syncMaxRetries", + objs: []any{workerPod("pod-a", "pool"), resolvedPool("pool", "gvisor-release"), multiVersionConfig()}, + existing: stale, + assets: &fakeAssetFetcher{ + fn: func(context.Context, *sandboxAssetsRecord) (map[string]string, error) { + return nil, errors.New("temporary download failure on default version") + }, + }, + priorRetries: syncMaxRetries, + want: stale, + wantDir: true, + wantUntouched: true, + wantRetries: syncMaxRetries + 1, + }, { + // A transient failure on a restorable version writes the default + // runtime right away and requeues without the syncMaxRetries cap. + name: "incomplete restorable version writes partial file and requeues past syncMaxRetries", + objs: []any{workerPod("pod-a", "pool"), resolvedPool("pool", "gvisor-release"), multiVersionConfig()}, + assets: &fakeAssetFetcher{ + fn: func(ctx context.Context, rec *sandboxAssetsRecord) (map[string]string, error) { + if rec.Assets[gvisorAssetName].SHA256 != "sha-v2" { + return nil, errors.New("temporary download failure on restorable version") + } + return newTestAssetFetcher().ensureSandboxAssets(ctx, rec) + }, + }, + priorRetries: syncMaxRetries, + want: &sandboxruntimes.File{DefaultRuntime: wantRuntime("v2")}, + wantDir: true, + wantRetries: syncMaxRetries + 1, }, { name: "pod of no pool is not retried", objs: []any{workerPod("pod-a", ""), resolvedPool("pool", "gvisor-release"), multiVersionConfig()}, @@ -361,6 +469,9 @@ func TestWorkerSandboxSyncerProcess(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s := newTestSyncer(t, tt.objs...) + if tt.assets != nil { + s.assets = tt.assets + } limiter := workqueue.NewTypedItemExponentialFailureRateLimiter[string](time.Second, 5*time.Minute) for range tt.priorRetries { limiter.When("pod-a") @@ -371,13 +482,13 @@ func TestWorkerSandboxSyncerProcess(t *testing.T) { if err := os.MkdirAll(dir, 0o700); err != nil { t.Fatal(err) } - if _, err := sandboxversions.Write(s.versionsFile("pod-a"), tt.existing); err != nil { + if _, err := sandboxruntimes.Write(s.runtimesFile("pod-a"), tt.existing); err != nil { t.Fatal(err) } } var fileBefore time.Time if _, err := os.Stat(dir); err == nil { - file, err := os.Stat(s.versionsFile("pod-a")) + file, err := os.Stat(s.runtimesFile("pod-a")) if err != nil { t.Fatal(err) } @@ -393,7 +504,7 @@ func TestWorkerSandboxSyncerProcess(t *testing.T) { if _, err := os.Stat(dir); (err == nil) != tt.wantDir { t.Errorf("directory exists = %v (%v), want %v", err == nil, err, tt.wantDir) } - got, err := sandboxversions.Read(s.versionsFile("pod-a")) + got, err := sandboxruntimes.Read(s.runtimesFile("pod-a")) if err != nil && !errors.Is(err, os.ErrNotExist) { t.Fatal(err) } @@ -401,7 +512,7 @@ func TestWorkerSandboxSyncerProcess(t *testing.T) { t.Errorf("file mismatch (-want +got):\n%s", diff) } if tt.wantUntouched { - if after, err := os.Stat(s.versionsFile("pod-a")); err != nil || !after.ModTime().Equal(fileBefore) { + if after, err := os.Stat(s.runtimesFile("pod-a")); err != nil || !after.ModTime().Equal(fileBefore) { t.Errorf("an equal file was rewritten (mtime %v -> %v, %v)", fileBefore, after.ModTime(), err) } } @@ -559,7 +670,7 @@ func TestWorkerSandboxSyncerRun(t *testing.T) { // A pod Add event queued before the caches sync. s.queue.Add("pod-live") matches := func() bool { - return exists(s.podDir("pod-gone")) == tt.wantOrphan && exists(s.versionsFile("pod-live")) == tt.wantFile + return exists(s.podDir("pod-gone")) == tt.wantOrphan && exists(s.runtimesFile("pod-live")) == tt.wantFile } ctx, cancel := context.WithCancel(context.Background()) @@ -576,7 +687,7 @@ func TestWorkerSandboxSyncerRun(t *testing.T) { time.Sleep(300 * time.Millisecond) if !matches() { t.Errorf("before every cache synced: pod-gone directory exists = %v, pod-live file exists = %v; want %v, %v", - exists(s.podDir("pod-gone")), exists(s.versionsFile("pod-live")), tt.wantOrphan, tt.wantFile) + exists(s.podDir("pod-gone")), exists(s.runtimesFile("pod-live")), tt.wantOrphan, tt.wantFile) } } @@ -654,6 +765,7 @@ func TestStartWorkerSandboxSyncer(t *testing.T) { podFactory.Core().V1().Pods(), ateFactory.Api().V1alpha1().WorkerPools(), ateFactory.Api().V1alpha1().SandboxConfigs(), + newScriptAssetFetcher(t), dir) if err != nil { t.Fatalf("startWorkerSandboxSyncer() error = %v", err) @@ -662,7 +774,7 @@ func TestStartWorkerSandboxSyncer(t *testing.T) { podFactory.Start(ctx.Done()) for _, pod := range tt.pods { - waitFor(t, "the file of "+string(pod.UID), func() bool { return exists(s.versionsFile(string(pod.UID))) }) + waitFor(t, "the file of "+string(pod.UID), func() bool { return exists(s.runtimesFile(string(pod.UID))) }) } if tt.change != nil { if err := tt.change(ctx, kube); err != nil { @@ -670,7 +782,7 @@ func TestStartWorkerSandboxSyncer(t *testing.T) { } } for _, uid := range tt.wantFiles { - waitFor(t, "the file of "+uid, func() bool { return exists(s.versionsFile(uid)) }) + waitFor(t, "the file of "+uid, func() bool { return exists(s.runtimesFile(uid)) }) } for _, uid := range tt.wantNoDirs { waitFor(t, "the directory of "+uid+" to be removed", func() bool { return !exists(s.podDir(uid)) }) diff --git a/cmd/atelet/worker_sandbox_versions.go b/cmd/atelet/worker_sandbox_versions.go index f6037a7966..76c4c2d7fe 100644 --- a/cmd/atelet/worker_sandbox_versions.go +++ b/cmd/atelet/worker_sandbox_versions.go @@ -14,12 +14,14 @@ package main -// This file decides how one worker pod's sandbox versions file is built, +// This file decides how one worker pod's sandbox runtimes file is built, // in the steps syncPod runs: // - resolve: find the SandboxConfig of the pod's WorkerPool // - plan: pick the config's versions this node can run // - build: hash each version's assets and pause image digests into its -// sandbox_runtime_hash +// sandbox_runtime_hash, and probe what a snapshot taken on the runtime +// must find again: the host CPU identity for microVM, the runsc +// binary's CPU features for gVisor // - write: replace the file in the worker directory if its content changed // // worker_sandbox_syncer.go decides when each pod is synced, and creates and @@ -33,19 +35,21 @@ import ( "fmt" "log/slog" "os" + "os/exec" "path/filepath" "runtime" "slices" "strings" + "time" "github.com/google/go-containerregistry/pkg/name" corev1 "k8s.io/api/core/v1" - "github.com/agent-substrate/substrate/internal/hardware" - "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/sandboxversions" - "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" + "go.ate.dev/substrate/internal/hardware" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/sandboxruntimes" + "go.ate.dev/substrate/pkg/api/v1alpha1" + listersv1alpha1 "go.ate.dev/substrate/pkg/client/listers/api/v1alpha1" ) // errPermanent marks a failure that retrying cannot fix, so the syncer drops @@ -53,12 +57,31 @@ import ( // - the pod has no worker pool label // - the config's default version is missing, has no assets for this // node's architecture, or has a pause image not pinned by digest +// - the default version's runsc runs but fails, prints no features, or +// prints one this atelet does not know; the binary is content-addressed, +// so running it again gives the same answer +// +// The same failures on a restorable version leave it out of the file instead. var errPermanent = errors.New("permanent failure") -// sandboxPlan is what one worker's sandbox versions file is built from. +// errIncomplete marks a sync that wrote the worker's file with the default +// version and the restorable versions that resolved, while at least one +// restorable version had a transient failure. finish requeues the pod +// without the syncMaxRetries cap so the remaining restorable versions are +// added once the transient failure clears. +var errIncomplete = errors.New("incomplete sandbox runtimes") + +// errUnresolved marks a resolve failure caused by informer lag or a WorkerPool +// that has not resolved a SandboxConfig yet. Because the informer events that +// end those states also enqueue the pod again, finish caps errUnresolved +// retries at syncMaxRetries. +var errUnresolved = errors.New("unresolved sandbox config") + +// sandboxPlan is what one worker's sandbox runtimes file is built from. type sandboxPlan struct { - config string - versions []versionPlan // the default first + config string + sandboxClass v1alpha1.SandboxClass // the config's; decides which attributes its runtimes report + versions []versionPlan // the default first } // versionPlan is one SandboxConfig version projected onto this node's @@ -90,32 +113,32 @@ func poolConfigName(pool *v1alpha1.WorkerPool) string { return "" } -// resolve returns the sandbox class and SandboxConfig of a worker pod on this -// node. Apart from a pod in no pool, every failure can succeed once an -// informer catches up or the atecontroller resolves the pool. -func (r *workerSandboxResolver) resolve(pod *corev1.Pod) (v1alpha1.SandboxClass, *v1alpha1.SandboxConfig, error) { +// resolve returns the SandboxConfig of a worker pod on this node. Apart from +// a pod in no pool, every failure can succeed once an informer catches up or +// the atecontroller resolves the pool. +func (r *workerSandboxResolver) resolve(pod *corev1.Pod) (*v1alpha1.SandboxConfig, error) { poolName := pod.Labels[workerPoolLabel] if poolName == "" { - return "", nil, fmt.Errorf("%w: worker pod %s/%s has no %s label", errPermanent, pod.Namespace, pod.Name, workerPoolLabel) + return nil, fmt.Errorf("%w: worker pod %s/%s has no %s label", errPermanent, pod.Namespace, pod.Name, workerPoolLabel) } pool, err := r.workerPools.WorkerPools(pod.Namespace).Get(poolName) if err != nil { - return "", nil, fmt.Errorf("while getting WorkerPool %s/%s: %w", pod.Namespace, poolName, err) + return nil, fmt.Errorf("%w: while getting WorkerPool %s/%s: %w", errUnresolved, pod.Namespace, poolName, err) } configName := poolConfigName(pool) if configName == "" { - return "", nil, fmt.Errorf("WorkerPool %s/%s has not resolved a SandboxConfig for sandbox class %q", pod.Namespace, poolName, pool.Spec.DefaultSandboxClass()) + return nil, fmt.Errorf("%w: WorkerPool %s/%s has not resolved a SandboxConfig for sandbox class %q", errUnresolved, pod.Namespace, poolName, pool.Spec.DefaultSandboxClass()) } cfg, err := r.configs.Get(configName) if err != nil { - return "", nil, fmt.Errorf("while getting SandboxConfig %q: %w", configName, err) + return nil, fmt.Errorf("%w: while getting SandboxConfig %q: %w", errUnresolved, configName, err) } - return pool.Spec.DefaultSandboxClass(), cfg, nil + return cfg, nil } // plan resolves pod's SandboxConfig and the versions of it this node can run. func (r *workerSandboxResolver) plan(pod *corev1.Pod) (*sandboxPlan, error) { - _, cfg, err := r.resolve(pod) + cfg, err := r.resolve(pod) if err != nil { return nil, err } @@ -123,7 +146,7 @@ func (r *workerSandboxResolver) plan(pod *corev1.Pod) (*sandboxPlan, error) { if err != nil { return nil, err } - return &sandboxPlan{config: cfg.Name, versions: vers}, nil + return &sandboxPlan{config: cfg.Name, sandboxClass: cfg.Spec.SandboxClass, versions: vers}, nil } // planVersions returns the versions of cfg a worker on this node may use: the @@ -173,54 +196,207 @@ func recordForArch(cfg *v1alpha1.SandboxConfig, ver *v1alpha1.SandboxVersionConf } // buildFile returns the file content naming the runtimes of the versions in -// plan. The hashes come from the SandboxConfig spec alone, so it does no IO. -// A version whose runtime hash equals an earlier one's is left out, so the -// file lists each runtime once. +// plan. A version whose runtime hash equals an earlier one's is left out, so +// the file lists each runtime once. The default version's failure fails the +// sync, which leaves the existing file in place until a retry succeeds. A +// restorable version whose attributes cannot be built is left out; when the +// failure is transient the file is returned with an error wrapping +// errIncomplete, so the caller writes the runtimes that resolved right away +// and retries for the rest. // -// The file does not promise a runtime's assets are on the node. The -// prewarmer fetches each config's default version, and Run/Restore fetch any -// other version on demand. +// TODO: write the file as soon as the default version's attributes are known +// and add restorable runtimes on a later pass. On a fresh node the first +// worker cannot register until every restorable version's runsc has been +// fetched and probed, serially. // -// TODO: fetch the default version's assets and pause image through -// cmd/atelet/internal/filecache here, and remove sandbox_prewarm.go. Other -// versions stay fetched on demand at Restore. -func buildFile(ctx context.Context, plan *sandboxPlan) (*sandboxversions.File, error) { - out := &sandboxversions.File{} +// Listing a runtime does not promise its assets are on the node: probing +// fetches a gVisor version's runsc the first time this atelet sees it, but +// not its pause image, and nothing of a microVM version. The prewarmer +// fetches each config's default version, and Run/Restore fetch any other +// version on demand. +func (s *workerSandboxSyncer) buildFile(ctx context.Context, plan *sandboxPlan) (*sandboxruntimes.File, error) { + out := &sandboxruntimes.File{} seen := make(map[string]bool, len(plan.versions)) + var incomplete error for _, v := range plan.versions { h, err := versionRuntimeHash(v.rec) + if err == nil && seen[h] { + continue + } + var attrs sandboxruntimes.Attributes + if err == nil { + attrs, err = s.versionAttrs(ctx, plan.sandboxClass, v.rec, h) + } if err != nil { err = fmt.Errorf("SandboxConfig %q, version %q: %w", plan.config, v.name, err) if v.isDefault { - // No worker of the pool could start actors on this node. - return nil, fmt.Errorf("%w: %w", errPermanent, err) + // Without the default no worker of the pool could start + // actors on this node. + return nil, err + } + if errors.Is(err, errPermanent) { + slog.WarnContext(ctx, "Leaving a sandbox version out of the worker sandbox runtimes file", slog.Any("err", err)) + } else { + // Logged once by finish, with every version this names. + incomplete = errors.Join(incomplete, err) } - slog.WarnContext(ctx, "Leaving a sandbox version out of the worker sandbox versions file", slog.Any("err", err)) - continue - } - if seen[h] { continue } seen[h] = true - attrs := sandboxversions.Attributes{hardware.AttrSandboxRuntimeHash: h} if v.isDefault { out.DefaultRuntime = attrs } else { out.RestorableRuntimes = append(out.RestorableRuntimes, attrs) } } + if incomplete != nil { + return out, fmt.Errorf("%w: %w", errIncomplete, incomplete) + } return out, nil } +// versionAttrs returns the attributes of one version's runtime, whose +// sandbox_runtime_hash is hash: architecture and the hash for every class, +// the host CPU identity for a microVM runtime, and the runsc binary's +// cpu_features for a gVisor one. +func (s *workerSandboxSyncer) versionAttrs(ctx context.Context, class v1alpha1.SandboxClass, rec *sandboxAssetsRecord, hash string) (sandboxruntimes.Attributes, error) { + attrs := sandboxruntimes.Attributes{ + hardware.AttrArchitecture: runtime.GOARCH, + hardware.AttrSandboxRuntimeHash: hash, + } + switch class { + case v1alpha1.SandboxClassMicroVM: + vendor, model, err := s.hostCPUIdentity() + if err != nil { + return nil, fmt.Errorf("while probing host CPU identity: %w", err) + } + attrs[hardware.AttrCPUVendor], attrs[hardware.AttrCPUModel] = vendor, model + case v1alpha1.SandboxClassGvisor: + mask, err := s.gvisorCPUFeatures(ctx, rec) + if err != nil { + return nil, err + } + attrs[hardware.AttrCPUFeatures] = mask + default: + return nil, fmt.Errorf("%w: unsupported sandbox class %q", errPermanent, class) + } + return attrs, nil +} + +// cpuIdentity is the host CPU identity microVM runtimes report. +type cpuIdentity struct { + vendor, model string +} + +// hostCPUIdentity returns the host CPU's vendor and model, probing them the +// first time. A failed probe is tried again on the next sync. +func (s *workerSandboxSyncer) hostCPUIdentity() (vendor, model string, err error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.hostCPU == nil { + if vendor, model, err = s.probeCPUIdentity(); err != nil { + return "", "", err + } + s.hostCPU = &cpuIdentity{vendor: vendor, model: model} + } + return s.hostCPU.vendor, s.hostCPU.model, nil +} + +// gvisorCPUFeatures returns the encoded cpu_features of rec's runsc binary. +// The first time this atelet sees the binary's digest it fetches the binary +// into the local asset cache and runs ` cpu-features`. +func (s *workerSandboxSyncer) gvisorCPUFeatures(ctx context.Context, rec *sandboxAssetsRecord) (string, error) { + asset, ok := runscAssetOf(rec) + if !ok { + return "", fmt.Errorf("%w: gvisor version has no %q or %q asset", errPermanent, gvisorAssetName, runscAssetName) + } + digest := rec.Assets[asset].SHA256 + s.mu.Lock() + mask, ok := s.cpuFeatures[digest] + s.mu.Unlock() + if ok { + return mask, nil + } + // A download that hangs would hold a sync worker; prewarmTimeout bounds + // the fetch for the same reason it bounds prewarming. + ctx, cancel := context.WithTimeout(ctx, prewarmTimeout) + defer cancel() + paths, err := s.assets.ensureSandboxAssets(ctx, rec) + if err != nil { + return "", err + } + names, err := s.runCPUFeatures(ctx, paths[asset]) + if err != nil { + return "", err + } + mask, err = hardware.EncodeCPUFeatures(runtime.GOARCH, names) + if err != nil { + return "", fmt.Errorf("%w: %w", errPermanent, err) + } + s.mu.Lock() + s.cpuFeatures[digest] = mask + s.mu.Unlock() + return mask, nil +} + +// runscAssetOf returns the name of the asset in rec that carries runsc, the +// most preferred of runscAssetNames that is present with a digest. +func runscAssetOf(rec *sandboxAssetsRecord) (string, bool) { + for _, name := range runscAssetNames { + if a, ok := rec.Assets[name]; ok && a.SHA256 != "" { + return name, true + } + } + return "", false +} + +// cpuFeaturesTimeout bounds one `runsc cpu-features` run, which takes +// milliseconds. A runsc that hangs would hold a sync worker. A var so tests +// can shorten it. +var cpuFeaturesTimeout = 30 * time.Second + +// runRunscCPUFeatures executes ` cpu-features` and returns the +// comma-separated feature names it prints. The binary is content-addressed, +// so one that runs and fails, or prints nothing, would do so again and fails +// permanently; one that cannot be started, or is cut off by ctx, is retried. +func runRunscCPUFeatures(ctx context.Context, runscPath string) ([]string, error) { + ctx, cancel := context.WithTimeout(ctx, cpuFeaturesTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, runscPath, "cpu-features") + cmd.Env = []string{} // the probe needs nothing from atelet's environment + out, err := cmd.Output() + var exitErr *exec.ExitError + switch { + case err == nil: + case ctx.Err() != nil: + // Before the ExitError case: a process killed at the deadline exits + // with one too. + return nil, fmt.Errorf("while running %s cpu-features: %w", runscPath, ctx.Err()) + case errors.As(err, &exitErr): + if stderr := strings.TrimSpace(string(exitErr.Stderr)); stderr != "" { + return nil, fmt.Errorf("%w: %s cpu-features: %w (%s)", errPermanent, runscPath, err, stderr) + } + return nil, fmt.Errorf("%w: %s cpu-features: %w", errPermanent, runscPath, err) + default: + return nil, fmt.Errorf("while starting %s cpu-features: %w", runscPath, err) + } + printed := strings.TrimSpace(string(out)) + if printed == "" { + return nil, fmt.Errorf("%w: %s cpu-features printed no features", errPermanent, runscPath) + } + return strings.Split(printed, ","), nil +} + // versionRuntimeHash returns the runtime hash of one version. The CRD // requires every pause image to be pinned by digest, which is the digest the -// image cache resolves the ref to, so the hash needs no pull. +// image cache resolves the ref to, so the hash needs no pull. An unpinned one +// is errPermanent: only a spec change fixes it. func versionRuntimeHash(rec *sandboxAssetsRecord) (string, error) { var pauseDigest string if rec.PauseImage != "" { ref, err := name.NewDigest(rec.PauseImage) if err != nil { - return "", fmt.Errorf("pause image %q is not pinned by digest: %w", rec.PauseImage, err) + return "", fmt.Errorf("%w: pause image %q is not pinned by digest: %w", errPermanent, rec.PauseImage, err) } pauseDigest = ref.DigestStr() } @@ -243,32 +419,38 @@ func runtimeHash(rec *sandboxAssetsRecord, pauseDigest string) string { return "sha256:" + hex.EncodeToString(sum[:]) } -// syncPod writes pod's sandbox versions file: +// syncPod writes pod's sandbox runtimes file: // - plan: resolve the pool's config and its versions for this node -// - build: hash each version's runtime +// - build: hash and probe each version's runtime // - write: replace the file if its content changed +// +// A file missing restorable runtimes for a transient reason is written all +// the same, and the errIncomplete from build is returned so finish retries. func (s *workerSandboxSyncer) syncPod(ctx context.Context, pod *corev1.Pod) error { plan, err := s.resolver.plan(pod) if err != nil { return err } - want, err := buildFile(ctx, plan) - if err != nil { + want, buildErr := s.buildFile(ctx, plan) + if buildErr != nil && !errors.Is(buildErr, errIncomplete) { + return buildErr + } + if err := s.write(ctx, string(pod.UID), plan.config, want); err != nil { return err } - return s.write(ctx, string(pod.UID), plan.config, want) + return buildErr } -func (s *workerSandboxSyncer) write(ctx context.Context, uid, config string, want *sandboxversions.File) error { +func (s *workerSandboxSyncer) write(ctx context.Context, uid, config string, want *sandboxruntimes.File) error { if err := os.MkdirAll(s.podDir(uid), 0o700); err != nil { return fmt.Errorf("while creating worker directory: %w", err) } - wrote, err := sandboxversions.Write(s.versionsFile(uid), want) + wrote, err := sandboxruntimes.Write(s.runtimesFile(uid), want) if err != nil { return err } if wrote { - slog.InfoContext(ctx, "Wrote worker sandbox versions", + slog.InfoContext(ctx, "Wrote worker sandbox runtimes", slog.String("pod_uid", uid), slog.String("sandbox_config", config), slog.String("default_runtime_hash", want.DefaultRuntime[hardware.AttrSandboxRuntimeHash]), @@ -277,6 +459,6 @@ func (s *workerSandboxSyncer) write(ctx context.Context, uid, config string, wan return nil } -func (s *workerSandboxSyncer) versionsFile(uid string) string { - return filepath.Join(s.podDir(uid), nodepath.SandboxVersionsFileName) +func (s *workerSandboxSyncer) runtimesFile(uid string) string { + return filepath.Join(s.podDir(uid), nodepath.SandboxRuntimesFileName) } diff --git a/cmd/atelet/worker_sandbox_versions_test.go b/cmd/atelet/worker_sandbox_versions_test.go index e64666dab6..ad593f8858 100644 --- a/cmd/atelet/worker_sandbox_versions_test.go +++ b/cmd/atelet/worker_sandbox_versions_test.go @@ -19,17 +19,21 @@ import ( "crypto/sha256" "errors" "fmt" + "os" + "path/filepath" "runtime" + "strings" "testing" + "time" "github.com/google/go-cmp/cmp" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" - "github.com/agent-substrate/substrate/internal/hardware" - "github.com/agent-substrate/substrate/internal/sandboxversions" - "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/internal/hardware" + "go.ate.dev/substrate/internal/sandboxruntimes" + "go.ate.dev/substrate/pkg/api/v1alpha1" ) // pauseDigest is the digest of the pause image of multiVersionConfig's @@ -190,12 +194,16 @@ func TestWorkerSandboxResolverResolve(t *testing.T) { cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", "abc") tests := []struct { - name string - objs []any - pod *corev1.Pod - wantConfig string - wantErr bool - wantPermanent bool + name string + objs []any + pod *corev1.Pod + wantConfig string + wantErr bool + // wantPermanent and wantUnresolved are the error's class, which + // decides whether finish drops it, caps its retries, or retries it + // without a cap. + wantPermanent bool + wantUnresolved bool }{{ name: "the config the pool resolved for the pool's class", objs: []any{resolvedPool("pool", "gvisor-default"), cfg}, @@ -208,15 +216,17 @@ func TestWorkerSandboxResolverResolve(t *testing.T) { wantErr: true, wantPermanent: true, }, { - name: "pool not found", - objs: []any{cfg}, - pod: workerPod("pod-a", "pool"), - wantErr: true, + name: "pool not found", + objs: []any{cfg}, + pod: workerPod("pod-a", "pool"), + wantErr: true, + wantUnresolved: true, }, { - name: "pool has not resolved a config", - objs: []any{resolvedPool("pool", ""), cfg}, - pod: workerPod("pod-a", "pool"), - wantErr: true, + name: "pool has not resolved a config", + objs: []any{resolvedPool("pool", ""), cfg}, + pod: workerPod("pod-a", "pool"), + wantErr: true, + wantUnresolved: true, }, { // A pending config is one the atecontroller has claimed but not yet // confirmed exists and is not being deleted. @@ -226,8 +236,9 @@ func TestWorkerSandboxResolverResolve(t *testing.T) { pool.Status.SandboxClasses[0].PendingConfigRef = &v1alpha1.SandboxConfigReference{Name: "gvisor-default"} return pool }(), cfg}, - pod: workerPod("pod-a", "pool"), - wantErr: true, + pod: workerPod("pod-a", "pool"), + wantErr: true, + wantUnresolved: true, }, { // The class comes from the pool's spec, not from the status entry. name: "status resolved another class than the pool's", @@ -236,29 +247,34 @@ func TestWorkerSandboxResolverResolve(t *testing.T) { pool.Spec.SandboxClasses[0].Name = v1alpha1.SandboxClassMicroVM return pool }(), cfg}, - pod: workerPod("pod-a", "pool"), - wantErr: true, + pod: workerPod("pod-a", "pool"), + wantErr: true, + wantUnresolved: true, }, { - name: "config not found", - objs: []any{resolvedPool("pool", "gvisor-missing"), cfg}, - pod: workerPod("pod-a", "pool"), - wantErr: true, + name: "config not found", + objs: []any{resolvedPool("pool", "gvisor-missing"), cfg}, + pod: workerPod("pod-a", "pool"), + wantErr: true, + wantUnresolved: true, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s := newTestSyncer(t, tt.objs...) - class, got, err := s.resolver.resolve(tt.pod) + got, err := s.resolver.resolve(tt.pod) if (err != nil) != tt.wantErr { t.Fatalf("resolve() error = %v, want error %v", err, tt.wantErr) } if errors.Is(err, errPermanent) != tt.wantPermanent { t.Errorf("resolve() error = %v, want permanent %v", err, tt.wantPermanent) } + if errors.Is(err, errUnresolved) != tt.wantUnresolved { + t.Errorf("resolve() error = %v, want unresolved %v", err, tt.wantUnresolved) + } if err != nil { return } - if got.Name != tt.wantConfig || class != v1alpha1.SandboxClassGvisor { - t.Errorf("resolve() = %q, %q, want %q, %q", class, got.Name, v1alpha1.SandboxClassGvisor, tt.wantConfig) + if got.Name != tt.wantConfig { + t.Errorf("resolve() = %q, want %q", got.Name, tt.wantConfig) } }) } @@ -296,6 +312,9 @@ func TestVersionRuntimeHash(t *testing.T) { if (err != nil) != tt.wantErr { t.Fatalf("versionRuntimeHash() error = %v, want error %v", err, tt.wantErr) } + if err != nil && !errors.Is(err, errPermanent) { + t.Errorf("versionRuntimeHash() error = %v, want errPermanent", err) + } if got != tt.want { t.Errorf("versionRuntimeHash() = %q, want %q", got, tt.want) } @@ -304,34 +323,42 @@ func TestVersionRuntimeHash(t *testing.T) { } // wantRuntime is the runtime buildFile builds from multiVersionConfig's -// version with the given name. -func wantRuntime(name string) sandboxversions.Attributes { +// version with the given name when runsc reports testCPUFeatures() ("3"). +func wantRuntime(name string) sandboxruntimes.Attributes { + return wantRuntimeWithFeatures(name, "3") +} + +func wantRuntimeWithFeatures(name, cpuFeatures string) sandboxruntimes.Attributes { rec := &sandboxAssetsRecord{Assets: map[string]assetEntry{gvisorAssetName: {SHA256: "sha-" + name}}} - return sandboxversions.Attributes{hardware.AttrSandboxRuntimeHash: runtimeHash(rec, pauseDigest(name))} + return sandboxruntimes.Attributes{ + hardware.AttrArchitecture: runtime.GOARCH, + hardware.AttrCPUFeatures: cpuFeatures, + hardware.AttrSandboxRuntimeHash: runtimeHash(rec, pauseDigest(name)), + } } // wantVersions is the file content for multiVersionConfig. -func wantVersions() *sandboxversions.File { - return &sandboxversions.File{ +func wantVersions() *sandboxruntimes.File { + return &sandboxruntimes.File{ DefaultRuntime: wantRuntime("v2"), - RestorableRuntimes: []sandboxversions.Attributes{wantRuntime("v1")}, + RestorableRuntimes: []sandboxruntimes.Attributes{wantRuntime("v1")}, } } // planAndBuild runs the plan and build steps of syncPod for pod. -func planAndBuild(s *workerSandboxSyncer, pod *corev1.Pod) (*sandboxversions.File, error) { +func planAndBuild(s *workerSandboxSyncer, pod *corev1.Pod) (*sandboxruntimes.File, error) { plan, err := s.resolver.plan(pod) if err != nil { return nil, err } - return buildFile(context.Background(), plan) + return s.buildFile(context.Background(), plan) } func TestPlanAndBuild(t *testing.T) { defaultNoArch := multiVersionConfig() defaultNoArch.Spec.Versions[3].Assets = map[string]map[string]v1alpha1.AssetFile{"other-arch": defaultNoArch.Spec.Versions[3].Assets[runtime.GOARCH]} - onlyDefault := &sandboxversions.File{DefaultRuntime: wantRuntime("v2")} + onlyDefault := &sandboxruntimes.File{DefaultRuntime: wantRuntime("v2")} // sameAsDefault has v1 run the default version's binaries. sameAsDefault := multiVersionConfig() @@ -351,10 +378,18 @@ func TestPlanAndBuild(t *testing.T) { return cfg } + unsupportedClass := multiVersionConfig() + unsupportedClass.Spec.SandboxClass = "unknown" + + noRunscAsset := multiVersionConfig() + noRunscAsset.Spec.Versions[3].Assets = map[string]map[string]v1alpha1.AssetFile{ + runtime.GOARCH: {"other": {URL: "gs://bucket/other", SHA256: "sha-other"}}, + } + tests := []struct { name string cfg *v1alpha1.SandboxConfig - want *sandboxversions.File + want *sandboxruntimes.File wantErr bool wantPermanent bool }{{ @@ -387,6 +422,16 @@ func TestPlanAndBuild(t *testing.T) { name: "non-default version pause image not pinned by digest is left out", cfg: tagged(1), want: onlyDefault, + }, { + name: "unsupported sandbox class", + cfg: unsupportedClass, + wantErr: true, + wantPermanent: true, + }, { + name: "gvisor version has no gvisor or runsc asset", + cfg: noRunscAsset, + wantErr: true, + wantPermanent: true, }, { name: "surfaces an unresolved config", wantErr: true, @@ -412,3 +457,288 @@ func TestPlanAndBuild(t *testing.T) { }) } } + +func TestPlanAndBuildMicroVM(t *testing.T) { + pool := resolvedPoolOf(v1alpha1.SandboxClassMicroVM, "microvm-pool", "microvm-release") + version := func(name, sha string) v1alpha1.SandboxVersionConfig { + return v1alpha1.SandboxVersionConfig{ + Name: name, + Assets: map[string]map[string]v1alpha1.AssetFile{ + runtime.GOARCH: {"kernel": {URL: "gs://bucket/kernel-" + name, SHA256: sha}}, + }, + } + } + cfg := &v1alpha1.SandboxConfig{ + ObjectMeta: metav1.ObjectMeta{Name: "microvm-release"}, + Spec: v1alpha1.SandboxConfigSpec{ + SandboxClass: v1alpha1.SandboxClassMicroVM, + DefaultVersion: "v2", + Versions: []v1alpha1.SandboxVersionConfig{ + version("v1", "sha-v1"), + version("v2", "sha-v2"), + }, + }, + } + wantMicroVM := func(sha string) sandboxruntimes.Attributes { + rec := &sandboxAssetsRecord{Assets: map[string]assetEntry{"kernel": {SHA256: sha}}} + return sandboxruntimes.Attributes{ + hardware.AttrArchitecture: runtime.GOARCH, + hardware.AttrCPUVendor: testCPUVendor, + hardware.AttrCPUModel: testCPUModel, + hardware.AttrSandboxRuntimeHash: runtimeHash(rec, ""), + } + } + + s := newTestSyncer(t, pool, cfg) + s.assets = &fakeAssetFetcher{ + fn: func(context.Context, *sandboxAssetsRecord) (map[string]string, error) { + t.Fatal("microVM buildFile must not fetch sandbox assets") + return nil, nil + }, + } + probeCalls := 0 + probeErr := errors.New("broken /proc/cpuinfo") + s.probeCPUIdentity = func() (string, string, error) { + probeCalls++ + if probeErr != nil { + return "", "", probeErr + } + return testCPUVendor, testCPUModel, nil + } + + // A failed probe is not cached: the sync fails with a retryable error and + // the next one probes again. + if _, err := planAndBuild(s, workerPod("pod-a", "microvm-pool")); !errors.Is(err, probeErr) || errors.Is(err, errPermanent) { + t.Fatalf("planAndBuild() on probe failure = %v, want %v and not permanent", err, probeErr) + } + + // Once a probe succeeds, every later sync reuses it. + probeErr = nil + want := &sandboxruntimes.File{ + DefaultRuntime: wantMicroVM("sha-v2"), + RestorableRuntimes: []sandboxruntimes.Attributes{wantMicroVM("sha-v1")}, + } + for _, uid := range []string{"pod-a", "pod-b"} { + got, err := planAndBuild(s, workerPod(uid, "microvm-pool")) + if err != nil { + t.Fatalf("planAndBuild(%s) error = %v", uid, err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("planAndBuild(%s) mismatch (-want +got):\n%s", uid, diff) + } + } + if probeCalls != 2 { + t.Errorf("probe calls = %d, want 2: one that failed, one that succeeded and was reused", probeCalls) + } +} + +func TestGVisorCPUFeaturesMemoizationAndErrors(t *testing.T) { + feats := testCPUFeatures() + + t.Run("a version with an earlier version's binaries in another bucket is neither fetched nor probed", func(t *testing.T) { + // runtimeHash leaves URLs out, so v3 is v1's runtime. The syncer must + // see that from the spec alone, before touching v3's bucket. + cfg := multiVersionConfig() + v3 := *cfg.Spec.Versions[1].DeepCopy() + v3.Name = "v3" + v3.Assets[runtime.GOARCH][gvisorAssetName] = v1alpha1.AssetFile{URL: "gs://other-bucket/v1", SHA256: "sha-v1"} + cfg.Spec.Versions = append(cfg.Spec.Versions, v3) + + s := newTestSyncer(t, resolvedPool("pool", "gvisor-release"), cfg) + s.assets = &fakeAssetFetcher{ + fn: func(ctx context.Context, rec *sandboxAssetsRecord) (map[string]string, error) { + if url := rec.Assets[gvisorAssetName].URL; strings.HasPrefix(url, "gs://other-bucket/") { + return nil, fmt.Errorf("fetched a duplicate runtime from %s", url) + } + return newTestAssetFetcher().ensureSandboxAssets(ctx, rec) + }, + } + got, err := planAndBuild(s, workerPod("pod-a", "pool")) + if err != nil { + t.Fatalf("planAndBuild() error = %v", err) + } + if diff := cmp.Diff(wantVersions(), got); diff != "" { + t.Errorf("planAndBuild() mismatch (-want +got):\n%s", diff) + } + }) + + t.Run("memoizes runsc cpu-features by asset digest across versions and syncs", func(t *testing.T) { + cfg := multiVersionConfig() + // Add a third version "v3" that shares v1's gvisor asset digest ("sha-v1") + // but has its own pause image digest, so it has a distinct runtime hash + // while reusing the same runsc binary. + v3 := *cfg.Spec.Versions[1].DeepCopy() + v3.Name = "v3" + v3.PauseImage = "registry.k8s.io/pause@" + pauseDigest("v3") + cfg.Spec.Versions = append(cfg.Spec.Versions, v3) + + s := newTestSyncer(t, resolvedPool("pool", "gvisor-release"), cfg) + s.assets = &fakeAssetFetcher{ + fn: func(_ context.Context, rec *sandboxAssetsRecord) (map[string]string, error) { + return map[string]string{gvisorAssetName: "/cache/" + rec.Assets[gvisorAssetName].SHA256 + "/runsc"}, nil + }, + } + execCalls := make(map[string]int) + s.runCPUFeatures = func(_ context.Context, runscPath string) ([]string, error) { + execCalls[runscPath]++ + if runscPath == "/cache/sha-v2/runsc" { + return feats, nil // bits 0 and 1 -> "3" + } + return feats[:1], nil // bit 0 only -> "1" + } + + got, err := planAndBuild(s, workerPod("pod-a", "pool")) + if err != nil { + t.Fatalf("first planAndBuild() error = %v", err) + } + v3Rec := &sandboxAssetsRecord{Assets: map[string]assetEntry{gvisorAssetName: {SHA256: "sha-v1"}}} + want := &sandboxruntimes.File{ + DefaultRuntime: wantRuntimeWithFeatures("v2", "3"), + RestorableRuntimes: []sandboxruntimes.Attributes{ + wantRuntimeWithFeatures("v1", "1"), + { + hardware.AttrArchitecture: runtime.GOARCH, + hardware.AttrCPUFeatures: "1", + hardware.AttrSandboxRuntimeHash: runtimeHash(v3Rec, pauseDigest("v3")), + }, + }, + } + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("planAndBuild() mismatch (-want +got):\n%s", diff) + } + + // Syncing again must answer from the memo alone: no fetch, no runsc. + s.assets = &fakeAssetFetcher{ + fn: func(context.Context, *sandboxAssetsRecord) (map[string]string, error) { + return nil, errors.New("fetched a runsc whose cpu_features are memoized") + }, + } + if _, err := planAndBuild(s, workerPod("pod-b", "pool")); err != nil { + t.Fatalf("second planAndBuild() error = %v", err) + } + wantCalls := map[string]int{ + "/cache/sha-v2/runsc": 1, + "/cache/sha-v1/runsc": 1, + } + if diff := cmp.Diff(wantCalls, execCalls); diff != "" { + t.Errorf("runCPUFeatures call counts mismatch (-want +got):\n%s", diff) + } + }) + + t.Run("transient asset fetch or runsc execution error is retriable", func(t *testing.T) { + s := newTestSyncer(t, resolvedPool("pool", "gvisor-release"), multiVersionConfig()) + s.assets = &fakeAssetFetcher{ + fn: func(context.Context, *sandboxAssetsRecord) (map[string]string, error) { + return nil, errors.New("temporary download failure") + }, + } + if _, err := planAndBuild(s, workerPod("pod-a", "pool")); err == nil || errors.Is(err, errPermanent) { + t.Errorf("planAndBuild() on asset fetch error = %v, want non-permanent error", err) + } + + s = newTestSyncer(t, resolvedPool("pool", "gvisor-release"), multiVersionConfig()) + s.runCPUFeatures = func(context.Context, string) ([]string, error) { + return nil, errors.New("runsc failed") + } + if _, err := planAndBuild(s, workerPod("pod-a", "pool")); err == nil || errors.Is(err, errPermanent) { + t.Errorf("planAndBuild() on runCPUFeatures error = %v, want non-permanent error", err) + } + + // On a restorable version alone, buildFile returns the default runtime + // alongside errIncomplete so syncPod writes the partial file right away + // and requeues for the rest. + s = newTestSyncer(t, resolvedPool("pool", "gvisor-release"), multiVersionConfig()) + s.assets = &fakeAssetFetcher{ + fn: func(_ context.Context, rec *sandboxAssetsRecord) (map[string]string, error) { + if sha := rec.Assets[gvisorAssetName].SHA256; sha != "sha-v2" { + return nil, fmt.Errorf("temporary download failure for %s", sha) + } + return map[string]string{gvisorAssetName: "sha-v2"}, nil + }, + } + s.runCPUFeatures = func(context.Context, string) ([]string, error) { return feats, nil } + got, err := planAndBuild(s, workerPod("pod-a", "pool")) + if !errors.Is(err, errIncomplete) || errors.Is(err, errPermanent) { + t.Errorf("planAndBuild() on a restorable version's fetch error = %v, want errIncomplete and not errPermanent", err) + } + want := &sandboxruntimes.File{DefaultRuntime: wantRuntime("v2")} + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("planAndBuild() partial file mismatch (-want +got):\n%s", diff) + } + }) + + t.Run("unknown CPU feature on default version is permanent, on non-default is skipped", func(t *testing.T) { + s := newTestSyncer(t, resolvedPool("pool", "gvisor-release"), multiVersionConfig()) + s.runCPUFeatures = func(context.Context, string) ([]string, error) { + return []string{"not_a_known_feature"}, nil + } + if _, err := planAndBuild(s, workerPod("pod-a", "pool")); !errors.Is(err, errPermanent) { + t.Errorf("planAndBuild() with unknown feature on default = %v, want errPermanent", err) + } + + s = newTestSyncer(t, resolvedPool("pool", "gvisor-release"), multiVersionConfig()) + s.assets = &fakeAssetFetcher{ + fn: func(_ context.Context, rec *sandboxAssetsRecord) (map[string]string, error) { + return map[string]string{gvisorAssetName: rec.Assets[gvisorAssetName].SHA256}, nil + }, + } + s.runCPUFeatures = func(_ context.Context, runscPath string) ([]string, error) { + if runscPath == "sha-v2" { + return feats, nil + } + return []string{"future_unknown_feature"}, nil + } + got, err := planAndBuild(s, workerPod("pod-a", "pool")) + if err != nil { + t.Fatalf("planAndBuild() with unknown feature on non-default error = %v", err) + } + want := &sandboxruntimes.File{DefaultRuntime: wantRuntime("v2")} + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("planAndBuild() mismatch (-want +got):\n%s", diff) + } + }) +} + +func TestRunRunscCPUFeatures(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + script := func(name, body string) string { + t.Helper() + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte("#!/bin/sh\n"+body), 0o755); err != nil { + t.Fatal(err) + } + return p + } + + got, err := runRunscCPUFeatures(ctx, script("ok", "echo pni,avx2\n")) + if err != nil { + t.Fatalf("runRunscCPUFeatures() error = %v", err) + } + if diff := cmp.Diff([]string{"pni", "avx2"}, got); diff != "" { + t.Errorf("runRunscCPUFeatures() mismatch (-want +got):\n%s", diff) + } + + // A binary that runs and fails, or prints nothing, will do so again: + // permanent, with its stderr in the error. + _, err = runRunscCPUFeatures(ctx, script("fail", "echo boom >&2\nexit 1\n")) + if !errors.Is(err, errPermanent) || !strings.Contains(err.Error(), "boom") { + t.Errorf("runRunscCPUFeatures() on a failing binary = %v, want permanent error quoting boom", err) + } + if _, err := runRunscCPUFeatures(ctx, script("empty", "exit 0\n")); !errors.Is(err, errPermanent) { + t.Errorf("runRunscCPUFeatures() on empty output = %v, want permanent error", err) + } + + // A binary that cannot be started, or is cut off, is retried. + if _, err := runRunscCPUFeatures(ctx, filepath.Join(dir, "missing")); err == nil || errors.Is(err, errPermanent) { + t.Errorf("runRunscCPUFeatures() on a missing binary = %v, want non-permanent error", err) + } + origTimeout := cpuFeaturesTimeout + cpuFeaturesTimeout = 50 * time.Millisecond + t.Cleanup(func() { cpuFeaturesTimeout = origTimeout }) + // exec, so the kill reaches the process holding stdout. + _, err = runRunscCPUFeatures(ctx, script("hang", "exec sleep 10\n")) + if !errors.Is(err, context.DeadlineExceeded) || errors.Is(err, errPermanent) { + t.Errorf("runRunscCPUFeatures() on a hanging binary = %v, want context.DeadlineExceeded", err) + } +} diff --git a/cmd/atelet/workersupport.go b/cmd/atelet/workersupport.go index aa3bcad262..e899ca52e6 100644 --- a/cmd/atelet/workersupport.go +++ b/cmd/atelet/workersupport.go @@ -20,11 +20,11 @@ import ( "fmt" "log/slog" - "github.com/agent-substrate/substrate/cmd/atelet/internal/apivalidation" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/substratex509" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/atelet/internal/apivalidation" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/substratex509" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/peer" @@ -38,8 +38,10 @@ type workerSupportServer struct { } func (b *workerSupportServer) MintActorCertificate(ctx context.Context, req *ateletpb.MintActorCertificateRequest) (*ateletpb.MintActorCertificateResponse, error) { - // Check which worker is calling. - _, err := authenticatedWorkerIdentity(ctx) + // Identity comes only from the mTLS certificate, never from the request: + // the control plane mints only for the Worker that the actor is assigned + // to, so a worker gets certificates for the actor it hosts and no other. + workerIdentity, err := authenticatedWorkerIdentity(ctx) if err != nil { return nil, err } @@ -50,11 +52,9 @@ func (b *workerSupportServer) MintActorCertificate(ctx context.Context, req *ate return nil, resources.ToGRPCStatusError(errs) } - // TODO(identity): Check that we believe that this worker is running the - // requested actor? ate-api-server will further check that we (the atelet) - // are allowed to request a certificate for the actor. - resp, err := b.workers.MintWorkerActorCertificate(ctx, &ateapipb.MintWorkerActorCertificateRequest{ + // Workers are global-scoped and named by their pod UID. + Worker: &ateapipb.ObjectRef{Name: workerIdentity.PodUID}, Actor: &ateapipb.ObjectRef{ Atespace: req.GetActorAtespace(), Name: req.GetActorName(), @@ -106,10 +106,10 @@ func verifyClientOnSameNode(node *substratex509.PodIdentity) func(tls.Connection // worker says its capacity and sandbox runtimes are, in one // WorkerService.RegisterWorker call so capacity and runtimes land atomically. // -// It returns the control plane's error unwrapped so the caller retries: a -// worker reports once, so an accepted call is the only thing that puts -// capacity and runtimes on the Worker, and a Worker record the syncer has not -// created yet is the ordinary reason for a first attempt to fail. +// It returns the control plane's error unwrapped so the caller retries: an +// accepted call is the only thing that puts capacity and the current runtimes +// on the Worker, and a Worker record the syncer has not created yet is the +// ordinary reason for a first attempt to fail. func (s *workerSupportServer) RegisterWorker(ctx context.Context, req *ateletpb.RegisterWorkerRequest) (*ateletpb.RegisterWorkerResponse, error) { // Identity comes only from the mTLS certificate, never from the request: // a worker can report its own capacity and runtimes and no one else's. diff --git a/cmd/atelet/workersupport_test.go b/cmd/atelet/workersupport_test.go index 2215bbb79c..6b9c4d5301 100644 --- a/cmd/atelet/workersupport_test.go +++ b/cmd/atelet/workersupport_test.go @@ -25,11 +25,11 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/substratex509" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/substratex509" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" @@ -333,6 +333,8 @@ func TestMintActorCertificateForwardsToWorkerService(t *testing.T) { } want := []*ateapipb.MintWorkerActorCertificateRequest{{ + // The worker comes from the caller's certificate. + Worker: &ateapipb.ObjectRef{Name: "pod-a"}, Actor: &ateapipb.ObjectRef{ Atespace: "team-a", Name: "actor-1", diff --git a/cmd/atenet/README.md b/cmd/atenet/README.md index 8f17a3855c..d72b0f6a71 100644 --- a/cmd/atenet/README.md +++ b/cmd/atenet/README.md @@ -29,7 +29,7 @@ likely be split in the future for better scalability.) than a header. `atunnel` can't read that metadata directly, though -- it's a separate process on the worker pod, not part of Envoy -- so the port to reach on the actor itself (its default port, or an arbitrary one for - CONNECT) still travels as a real header, `atunnel.TargetPortHeader`. + CONNECT) still travels as a real header, `TargetPortHeader` in `internal/atunnel/ingress`. `ate-target-actor: /` identifies the Actor independently of `:authority`/`Host`. * Termination: the router drains gracefully on SIGTERM (readiness flip → diff --git a/cmd/atenet/internal/root.go b/cmd/atenet/internal/root.go index d7d8046de8..6c703ebfe2 100644 --- a/cmd/atenet/internal/root.go +++ b/cmd/atenet/internal/root.go @@ -18,11 +18,11 @@ import ( "fmt" "os" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router" - "github.com/agent-substrate/substrate/cmd/atenet/internal/sdsmint" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/version" "github.com/spf13/cobra" + "go.ate.dev/substrate/cmd/atenet/internal/router" + "go.ate.dev/substrate/cmd/atenet/internal/sdsmint" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/version" ) var previewFlags []string diff --git a/cmd/atenet/internal/router/cmd.go b/cmd/atenet/internal/router/cmd.go index 7bd65c2d3d..5a26645f6f 100644 --- a/cmd/atenet/internal/router/cmd.go +++ b/cmd/atenet/internal/router/cmd.go @@ -22,9 +22,9 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/egress" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" - "github.com/agent-substrate/substrate/internal/installdefaults" + "go.ate.dev/substrate/cmd/atenet/internal/router/egress" + "go.ate.dev/substrate/cmd/atenet/internal/router/ingress" + "go.ate.dev/substrate/internal/installdefaults" ) func NewRouterCmd() *cobra.Command { @@ -84,7 +84,6 @@ func bindRouterFlags(fs *pflag.FlagSet, cfg *routerConfig) { fs.StringVar(&cfg.UpstreamCredentialBundlePath, "upstream-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "PEM credential bundle (cert+key) the router presents as the client cert when dialing the actor's atunnel ingress server over mTLS. Empty disables upstream mTLS (legacy plaintext pod-IP:80) and needs --upstream-connect=false.") fs.StringVar(&cfg.UpstreamTrustBundlePath, "upstream-trust-bundle", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "PEM trust bundle used to validate the actor's atunnel ingress server certificate.") - fs.StringVar(&cfg.UpstreamSpiffePrefix, "upstream-spiffe-prefix", "spiffe://cluster.local/", "SPIFFE URI SAN prefix (trust domain) the actor's atunnel server cert must match. Empty falls back to default SAN check against the dialed pod IP (which SPIFFE-only certs never match).") fs.BoolVar(&cfg.UpstreamConnect, "upstream-connect", true, "Reach atunnel through mTLS CONNECT tunnels on port 8443; false reaches it over HTTPS on 443 instead (Envoy dataplane only). Requires --upstream-credential-bundle") fs.StringVar(&cfg.ActorIdentityCAFile, "actor-identity-ca-file", "", "PEM trust bundle for the actor-identity CA, used to verify the actor client certificates presented on egress CONNECTs. Required by the egress gateway's ext_proc sidecar; empty (the default) leaves egress authentication unconfigured and every egress CONNECT is denied.") diff --git a/cmd/atenet/internal/router/config.go b/cmd/atenet/internal/router/config.go index 55c9cb00da..84c43f4c6f 100644 --- a/cmd/atenet/internal/router/config.go +++ b/cmd/atenet/internal/router/config.go @@ -20,7 +20,7 @@ import ( "github.com/spiffe/go-spiffe/v2/spiffeid" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" + "go.ate.dev/substrate/cmd/atenet/internal/router/ingress" ) type atenetRouter string @@ -171,9 +171,6 @@ type routerConfig struct { // validate that server. Empty UpstreamCredentialBundlePath disables upstream mTLS. UpstreamCredentialBundlePath string UpstreamTrustBundlePath string - // UpstreamSpiffePrefix validates the actor's atunnel server cert by its - // SPIFFE URI SAN prefix (trust domain) instead of the dialed pod IP. - UpstreamSpiffePrefix string // UpstreamConnect tunnels actor traffic to atunnel's CONNECT listener. UpstreamConnect bool diff --git a/cmd/atenet/internal/router/config_test.go b/cmd/atenet/internal/router/config_test.go index 16ecb54bef..ece06c928b 100644 --- a/cmd/atenet/internal/router/config_test.go +++ b/cmd/atenet/internal/router/config_test.go @@ -21,7 +21,7 @@ import ( "github.com/spf13/pflag" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" + "go.ate.dev/substrate/cmd/atenet/internal/router/ingress" ) func TestRouterConfigValidate(t *testing.T) { diff --git a/cmd/atenet/internal/router/dataplane.go b/cmd/atenet/internal/router/dataplane.go index 7bae70d171..352e3eca35 100644 --- a/cmd/atenet/internal/router/dataplane.go +++ b/cmd/atenet/internal/router/dataplane.go @@ -24,8 +24,8 @@ import ( "golang.org/x/sync/errgroup" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" - "github.com/agent-substrate/substrate/internal/credbundle" + "go.ate.dev/substrate/cmd/atenet/internal/router/ingress" + "go.ate.dev/substrate/internal/credbundle" ) type dataplaneHealthCheck struct { @@ -122,7 +122,7 @@ func (s *RouterServer) newEnvoyXdsServer(ctx context.Context, parkCfg ingress.Pa if s.cfg.IngressAuth.mode() == IngressAuthDeprecatedInsecure { slog.WarnContext(ctx, "Ingress client authentication is optional: any client that presents no certificate, or uses a plaintext listener, can reach any actor. Set --ingress-auth-mode=static-mtls to require mTLS") } - xdsSrv.SetUpstreamTls(s.cfg.UpstreamCredentialBundlePath, s.cfg.UpstreamTrustBundlePath, s.cfg.UpstreamSpiffePrefix) + xdsSrv.SetUpstreamTls(s.cfg.UpstreamCredentialBundlePath, s.cfg.UpstreamTrustBundlePath) xdsSrv.SetUpstreamConnect(s.cfg.UpstreamConnect) // The snapshot is a pure function of the configuration applied above, so it diff --git a/cmd/atenet/internal/router/drain.go b/cmd/atenet/internal/router/drain.go index 1c4ef913b1..3e94d44397 100644 --- a/cmd/atenet/internal/router/drain.go +++ b/cmd/atenet/internal/router/drain.go @@ -23,7 +23,7 @@ import ( "path/filepath" "time" - "github.com/agent-substrate/substrate/internal/serverboot" + "go.ate.dev/substrate/internal/serverboot" ) // defaultDrainCompleteFile is where the drain sequence leaves its completion diff --git a/cmd/atenet/internal/router/drain_test.go b/cmd/atenet/internal/router/drain_test.go index b4b7083afa..17bcb77658 100644 --- a/cmd/atenet/internal/router/drain_test.go +++ b/cmd/atenet/internal/router/drain_test.go @@ -26,7 +26,7 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/serverboot" + "go.ate.dev/substrate/internal/serverboot" ) // drainFunc adapts a bare function to the dataplaneDrainer interface. diff --git a/cmd/atenet/internal/router/egress/actorjwt/actorjwt.go b/cmd/atenet/internal/router/egress/actorjwt/actorjwt.go index 4f79ca14a1..6ac39429ea 100644 --- a/cmd/atenet/internal/router/egress/actorjwt/actorjwt.go +++ b/cmd/atenet/internal/router/egress/actorjwt/actorjwt.go @@ -25,8 +25,8 @@ import ( "google.golang.org/protobuf/proto" "k8s.io/apimachinery/pkg/util/cache" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // Minter mints actor JWTs and reuses each token until a third of its lifetime diff --git a/cmd/atenet/internal/router/egress/actorjwt/actorjwt_test.go b/cmd/atenet/internal/router/egress/actorjwt/actorjwt_test.go index f7194887a4..07a8a49f08 100644 --- a/cmd/atenet/internal/router/egress/actorjwt/actorjwt_test.go +++ b/cmd/atenet/internal/router/egress/actorjwt/actorjwt_test.go @@ -33,8 +33,8 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) var testActor = resources.ActorRef{Atespace: "default", Name: "my-actor"} diff --git a/cmd/atenet/internal/router/egress/credentials.go b/cmd/atenet/internal/router/egress/credentials.go index 6ac51c3145..31f292df8f 100644 --- a/cmd/atenet/internal/router/egress/credentials.go +++ b/cmd/atenet/internal/router/egress/credentials.go @@ -25,11 +25,11 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" - "github.com/agent-substrate/substrate/internal/egresspolicy" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - "github.com/agent-substrate/substrate/pkg/proto/credproviderpb" + "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" + "go.ate.dev/substrate/internal/egresspolicy" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/credproviderpb" ) // mapCredentialProviderError converts a FetchSecret failure into a diff --git a/cmd/atenet/internal/router/egress/credentials_test.go b/cmd/atenet/internal/router/egress/credentials_test.go index 57fbcb09bd..e90d9aae93 100644 --- a/cmd/atenet/internal/router/egress/credentials_test.go +++ b/cmd/atenet/internal/router/egress/credentials_test.go @@ -27,9 +27,9 @@ import ( "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - "github.com/agent-substrate/substrate/pkg/proto/credproviderpb" + "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" + "go.ate.dev/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/credproviderpb" ) // credentialInjectionPolicySample injects "authorization: Bearer " from diff --git a/cmd/atenet/internal/router/egress/egress.go b/cmd/atenet/internal/router/egress/egress.go index 0fcd11fd7c..1fd82c42dc 100644 --- a/cmd/atenet/internal/router/egress/egress.go +++ b/cmd/atenet/internal/router/egress/egress.go @@ -41,12 +41,12 @@ import ( "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/structpb" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/egress/actorjwt" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" - "github.com/agent-substrate/substrate/internal/egresspolicy" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - "github.com/agent-substrate/substrate/pkg/proto/credproviderpb" + "go.ate.dev/substrate/cmd/atenet/internal/router/egress/actorjwt" + "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" + "go.ate.dev/substrate/internal/egresspolicy" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/credproviderpb" ) const ( diff --git a/cmd/atenet/internal/router/egress/egress_test.go b/cmd/atenet/internal/router/egress/egress_test.go index bc6df80dbc..8fe77bc535 100644 --- a/cmd/atenet/internal/router/egress/egress_test.go +++ b/cmd/atenet/internal/router/egress/egress_test.go @@ -42,10 +42,10 @@ import ( "google.golang.org/protobuf/types/known/structpb" "google.golang.org/protobuf/types/known/timestamppb" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" - "github.com/agent-substrate/substrate/internal/egresspolicy" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" + "go.ate.dev/substrate/internal/egresspolicy" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) const ( diff --git a/cmd/atenet/internal/router/egress/policycache.go b/cmd/atenet/internal/router/egress/policycache.go index 954d91a174..28f6f29e02 100644 --- a/cmd/atenet/internal/router/egress/policycache.go +++ b/cmd/atenet/internal/router/egress/policycache.go @@ -26,9 +26,9 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/agent-substrate/substrate/internal/egresspolicy" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/egresspolicy" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // DefaultPolicyCacheTTL is how long a fetched policy stays usable, and so how diff --git a/cmd/atenet/internal/router/egress/policycache_test.go b/cmd/atenet/internal/router/egress/policycache_test.go index 4a175d8ab8..b712a13ebf 100644 --- a/cmd/atenet/internal/router/egress/policycache_test.go +++ b/cmd/atenet/internal/router/egress/policycache_test.go @@ -26,8 +26,8 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/agent-substrate/substrate/internal/egresspolicy" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/egresspolicy" + "go.ate.dev/substrate/internal/resources" ) var testActorRef = resources.ActorRef{Atespace: testEgressAtespace, Name: testEgressActor} diff --git a/cmd/atenet/internal/router/egress/provider.go b/cmd/atenet/internal/router/egress/provider.go index fce57bb1be..252e3ec237 100644 --- a/cmd/atenet/internal/router/egress/provider.go +++ b/cmd/atenet/internal/router/egress/provider.go @@ -33,7 +33,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" - "github.com/agent-substrate/substrate/internal/ateapiauth" + "go.ate.dev/substrate/internal/ateapiauth" ) // credentialURIScheme is the only scheme a credential URI may carry. diff --git a/cmd/atenet/internal/router/egress/request.go b/cmd/atenet/internal/router/egress/request.go index 6313de4c58..e75ad8bd19 100644 --- a/cmd/atenet/internal/router/egress/request.go +++ b/cmd/atenet/internal/router/egress/request.go @@ -24,9 +24,9 @@ import ( extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" - "github.com/agent-substrate/substrate/internal/egresspolicy" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" + "go.ate.dev/substrate/internal/egresspolicy" + "go.ate.dev/substrate/internal/resources" ) const websocketDeniedBody = "WebSocket egress is not supported" diff --git a/cmd/atenet/internal/router/egress/request_test.go b/cmd/atenet/internal/router/egress/request_test.go index 67589c16b6..954d8dd32f 100644 --- a/cmd/atenet/internal/router/egress/request_test.go +++ b/cmd/atenet/internal/router/egress/request_test.go @@ -26,8 +26,8 @@ import ( "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/structpb" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // testActorSPIFFEID is the identity filter state the CONNECT chain shares. diff --git a/cmd/atenet/internal/router/egressmanifest_test.go b/cmd/atenet/internal/router/egressmanifest_test.go index 136b1bbe1c..b55796e5ae 100644 --- a/cmd/atenet/internal/router/egressmanifest_test.go +++ b/cmd/atenet/internal/router/egressmanifest_test.go @@ -16,10 +16,12 @@ package router import ( "os" + "slices" "strings" "testing" "time" + "github.com/google/go-cmp/cmp" "sigs.k8s.io/yaml" ) @@ -151,3 +153,81 @@ func envoyConfigFrom(t *testing.T, path, configMap string) string { t.Fatalf("%s has no ConfigMap named %s", path, configMap) return "" } + +// wantSampledLogFilter is the filter on the access logs of the per-request legs. +const wantSampledLogFilter = ` +or_filter: + filters: + - status_code_filter: + comparison: + op: GE + value: {default_value: 400, runtime_key: access_log.min_status} + - response_flag_filter: {} + - and_filter: + filters: + - header_filter: + header: + name: content-type + string_match: {prefix: application/grpc} + - grpc_status_filter: {exclude: true, statuses: [OK]} + - runtime_filter: + runtime_key: access_log.sample + percent_sampled: {numerator: 1, denominator: HUNDRED} + use_independent_randomness: true +` + +// Every actor request lands in the mitm or cleartext access log, so both keep +// failures and a sample of the rest behind runtime keys an operator can flip. +// No other filter chain's access log may filter, since each is the only record +// of its events. +func TestEgressManifestsSampleThePerRequestAccessLogs(t *testing.T) { + t.Parallel() + var want node + if err := yaml.Unmarshal([]byte(wantSampledLogFilter), &want); err != nil { + t.Fatalf("parsing wantSampledLogFilter: %v", err) + } + logs := chainAccessLogsByLeg(bootstrapTree(t, egressManifest)) + sampledLegs := []string{"mitm", "cleartext"} + for _, leg := range sampledLegs { + t.Run(leg, func(t *testing.T) { + t.Parallel() + if len(logs[leg]) == 0 { + t.Fatalf("no access log with leg %q; the manifest changed shape and this test is checking nothing", leg) + } + for _, al := range logs[leg] { + if diff := cmp.Diff(want, child(al, "filter")); diff != "" { + t.Errorf("access log %q filter mismatch (-want +got):\n%s", leg, diff) + } + } + }) + } + for leg, als := range logs { + if slices.Contains(sampledLegs, leg) { + continue + } + for _, al := range als { + if _, ok := al["filter"]; ok { + t.Errorf("access log %q has a filter; only %v may", leg, sampledLegs) + } + } + } +} + +// chainAccessLogsByLeg groups the access logs of every network filter in every +// filter chain by their json_format leg. The CONNECT log has a text format and +// no leg, so it is grouped as "connect". +func chainAccessLogsByLeg(tree node) map[string][]node { + out := map[string][]node{} + for _, lc := range allChains(tree) { + for _, f := range list(lc.chain, "filters") { + for _, al := range list(child(f, "typed_config"), "access_log") { + leg := str(child(child(child(al, "typed_config"), "log_format"), "json_format"), "leg") + if leg == "" { + leg = "connect" + } + out[leg] = append(out[leg], al) + } + } + } + return out +} diff --git a/cmd/atenet/internal/router/egresspolicymanifest_test.go b/cmd/atenet/internal/router/egresspolicymanifest_test.go index 5dc34d7c55..4aae9928fa 100644 --- a/cmd/atenet/internal/router/egresspolicymanifest_test.go +++ b/cmd/atenet/internal/router/egresspolicymanifest_test.go @@ -21,7 +21,7 @@ import ( "sigs.k8s.io/yaml" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" + "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" ) // These tests pin the egress manifests to the Go constants the handler diff --git a/cmd/atenet/internal/router/extproc/egress_websocket_test.go b/cmd/atenet/internal/router/extproc/egress_websocket_test.go index f82116830b..dc25406859 100644 --- a/cmd/atenet/internal/router/extproc/egress_websocket_test.go +++ b/cmd/atenet/internal/router/extproc/egress_websocket_test.go @@ -23,8 +23,8 @@ import ( extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" "google.golang.org/protobuf/types/known/structpb" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/egress" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" + "go.ate.dev/substrate/cmd/atenet/internal/router/egress" + "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" ) const websocketDenialBody = "WebSocket egress is not supported" diff --git a/cmd/atenet/internal/router/extproc/handler.go b/cmd/atenet/internal/router/extproc/handler.go index 7105493d16..0309bf1acb 100644 --- a/cmd/atenet/internal/router/extproc/handler.go +++ b/cmd/atenet/internal/router/extproc/handler.go @@ -20,7 +20,7 @@ import ( extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" "google.golang.org/protobuf/types/known/structpb" - "github.com/agent-substrate/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/ateattr" ) // Handler applies one direction's policy to a request. Implementations live in diff --git a/cmd/atenet/internal/router/extproc/metadata_test.go b/cmd/atenet/internal/router/extproc/metadata_test.go index 09a4b2548b..e33ee4f04b 100644 --- a/cmd/atenet/internal/router/extproc/metadata_test.go +++ b/cmd/atenet/internal/router/extproc/metadata_test.go @@ -19,8 +19,8 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/internal/atenet" corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + "go.ate.dev/substrate/internal/atenet" "google.golang.org/protobuf/types/known/structpb" ) diff --git a/cmd/atenet/internal/router/extproc/metrics.go b/cmd/atenet/internal/router/extproc/metrics.go index 0ff6972496..3f6a07ddb5 100644 --- a/cmd/atenet/internal/router/extproc/metrics.go +++ b/cmd/atenet/internal/router/extproc/metrics.go @@ -26,7 +26,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/agent-substrate/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/ateattr" ) // ServiceName is the OpenTelemetry service name and instrumentation scope diff --git a/cmd/atenet/internal/router/extproc/metrics_test.go b/cmd/atenet/internal/router/extproc/metrics_test.go index b02eadea01..2e081f9bfa 100644 --- a/cmd/atenet/internal/router/extproc/metrics_test.go +++ b/cmd/atenet/internal/router/extproc/metrics_test.go @@ -27,7 +27,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/agent-substrate/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/ateattr" ) func TestClassifyOutcome(t *testing.T) { diff --git a/cmd/atenet/internal/router/health.go b/cmd/atenet/internal/router/health.go index 19f7802196..335917945b 100644 --- a/cmd/atenet/internal/router/health.go +++ b/cmd/atenet/internal/router/health.go @@ -25,7 +25,7 @@ import ( "sync" "time" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "k8s.io/apimachinery/pkg/version" "k8s.io/client-go/kubernetes" diff --git a/cmd/atenet/internal/router/health_test.go b/cmd/atenet/internal/router/health_test.go index a04892514b..ccac1a4c72 100644 --- a/cmd/atenet/internal/router/health_test.go +++ b/cmd/atenet/internal/router/health_test.go @@ -25,7 +25,7 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "k8s.io/client-go/kubernetes" kubernetesfake "k8s.io/client-go/kubernetes/fake" diff --git a/cmd/atenet/internal/router/ingress/errors.go b/cmd/atenet/internal/router/ingress/errors.go index caaf6c7b32..6218d44dfa 100644 --- a/cmd/atenet/internal/router/ingress/errors.go +++ b/cmd/atenet/internal/router/ingress/errors.go @@ -23,9 +23,9 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/resources" ) // actorNotFoundErr returns a 404 denial identifying the missing actor. diff --git a/cmd/atenet/internal/router/ingress/errors_test.go b/cmd/atenet/internal/router/ingress/errors_test.go index d366fa72e2..a95bf1bb86 100644 --- a/cmd/atenet/internal/router/ingress/errors_test.go +++ b/cmd/atenet/internal/router/ingress/errors_test.go @@ -23,8 +23,8 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" + "go.ate.dev/substrate/internal/resources" ) func TestActorNotFoundErr(t *testing.T) { diff --git a/cmd/atenet/internal/router/ingress/flight.go b/cmd/atenet/internal/router/ingress/flight.go index c339d55e38..7afd444b01 100644 --- a/cmd/atenet/internal/router/ingress/flight.go +++ b/cmd/atenet/internal/router/ingress/flight.go @@ -17,7 +17,7 @@ package ingress import ( "sync/atomic" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) diff --git a/cmd/atenet/internal/router/ingress/ingress.go b/cmd/atenet/internal/router/ingress/ingress.go index 6b68011cea..58dd9f5dec 100644 --- a/cmd/atenet/internal/router/ingress/ingress.go +++ b/cmd/atenet/internal/router/ingress/ingress.go @@ -36,10 +36,10 @@ import ( "go.opentelemetry.io/otel" "google.golang.org/protobuf/types/known/structpb" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" - "github.com/agent-substrate/substrate/internal/atenet" - "github.com/agent-substrate/substrate/internal/atunnel" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" + "go.ate.dev/substrate/internal/atenet" + atunnelingress "go.ate.dev/substrate/internal/atunnel/ingress" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // defaultActorPort is the actor's port when a request names no other one. @@ -54,6 +54,8 @@ const ( OriginalDstAddressKey = "local" // OriginalDstPortKey is the actor's target port. OriginalDstPortKey = "port" + // WorkerSPIFFEIDKey is the SPIFFE ID atunnel's certificate must carry. + WorkerSPIFFEIDKey = "worker_spiffe_id" ) // Handler routes ingress requests to the worker hosting their actor. @@ -100,7 +102,7 @@ func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestM targetAuthority = md.Host } if _, portStr, err := net.SplitHostPort(targetAuthority); err == nil { - if p, ok := atunnel.ParsePort(portStr); ok { + if p, ok := atunnelingress.ParsePort(portStr); ok { targetPort = p } } @@ -121,16 +123,20 @@ func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestM // The first IP is in the cluster's primary IP family. // TODO: choose the IP family that matches the dataplane's own address. + assignment := actor.GetStatus().GetWorkerAssignment() var workerIP string - if ips := actor.GetStatus().GetWorkerAssignment().GetWorkerPodIps(); len(ips) > 0 { + if ips := assignment.GetWorkerPodIps(); len(ips) > 0 { workerIP = ips[0] } + workerSPIFFEID := assignment.GetWorkerSpiffeId() slog.InfoContext(ctx, "ResumeActor result", slog.Any("actor", actorRef), slog.String("state", actor.GetStatus().GetState().String()), - slog.String("workerIP", workerIP)) + slog.String("workerIP", workerIP), + slog.String("workerSPIFFEID", workerSPIFFEID)) - if ip := net.ParseIP(workerIP); ip == nil { + // Without the SPIFFE ID, atunnel's identity can't be checked. + if ip := net.ParseIP(workerIP); ip == nil || workerSPIFFEID == "" { return res, extproc.NewReqError(envoy_type.StatusCode_InternalServerError, "actor %s routing failed", actorRef) } @@ -147,6 +153,7 @@ func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestM OriginalDstMetadataKey: map[string]any{ OriginalDstAddressKey: targetAddr, OriginalDstPortKey: strconv.Itoa(targetPort), + WorkerSPIFFEIDKey: workerSPIFFEID, }, }) if err != nil { diff --git a/cmd/atenet/internal/router/ingress/ingress_test.go b/cmd/atenet/internal/router/ingress/ingress_test.go index 35c940dba1..760b55f0b8 100644 --- a/cmd/atenet/internal/router/ingress/ingress_test.go +++ b/cmd/atenet/internal/router/ingress/ingress_test.go @@ -33,11 +33,11 @@ import ( "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/structpb" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/atenet" - "github.com/agent-substrate/substrate/internal/atunnel" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/atenet" + atunnelingress "go.ate.dev/substrate/internal/atunnel/ingress" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) type mockClient struct { @@ -66,7 +66,7 @@ func TestHandleRequestHeadersAcceptsMixedCaseRoutingHeaders(t *testing.T) { t.Errorf("atespace = %q, want %q", got, want) } return &ateapipb.ResumeActorResponse{Actor: &ateapipb.Actor{ - Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIps: []string{"10.0.0.52"}}}, + Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerSpiffeId: testWorkerSPIFFEID, WorkerPodIps: []string{"10.0.0.52"}}}, }}, nil }, } @@ -158,7 +158,7 @@ func TestHandleRequestHeadersConnectAttributeOverridesInnerHeader(t *testing.T) resumeFn: func(_ context.Context, in *ateapipb.ResumeActorRequest, _ ...grpc.CallOption) (*ateapipb.ResumeActorResponse, error) { gotActorRef = in.GetActor().GetAtespace() + "/" + in.GetActor().GetName() return &ateapipb.ResumeActorResponse{Actor: &ateapipb.Actor{ - Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIps: []string{"10.0.0.52"}}}, + Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerSpiffeId: testWorkerSPIFFEID, WorkerPodIps: []string{"10.0.0.52"}}}, }}, nil }, } @@ -207,6 +207,15 @@ func dynamicMetadataPort(dynamicMetadata *structpb.Struct) string { return dynamicMetadata.GetFields()[OriginalDstMetadataKey].GetStructValue().GetFields()[OriginalDstPortKey].GetStringValue() } +// testWorkerSPIFFEID is the SPIFFE ID the test assignments give their worker. +const testWorkerSPIFFEID = "spiffe://cluster.local/ns/workers/sa/substrate-worker" + +// dynamicMetadataWorkerSPIFFEID extracts the SPIFFE ID HandleRequestHeaders +// requires of atunnel. +func dynamicMetadataWorkerSPIFFEID(dynamicMetadata *structpb.Struct) string { + return dynamicMetadata.GetFields()[OriginalDstMetadataKey].GetStructValue().GetFields()[WorkerSPIFFEIDKey].GetStringValue() +} + func TestHandleRequestHeadersDoesNotLogSensitiveData(t *testing.T) { const testUUID = "123e4567-e89b-12d3-a456-426614174000" const secret = "do-not-log-me" @@ -219,7 +228,7 @@ func TestHandleRequestHeadersDoesNotLogSensitiveData(t *testing.T) { h := New(&mockClient{ resumeFn: func(ctx context.Context, in *ateapipb.ResumeActorRequest, opts ...grpc.CallOption) (*ateapipb.ResumeActorResponse, error) { - return &ateapipb.ResumeActorResponse{Actor: &ateapipb.Actor{Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIps: []string{"10.0.0.52"}}}}}, nil + return &ateapipb.ResumeActorResponse{Actor: &ateapipb.Actor{Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerSpiffeId: testWorkerSPIFFEID, WorkerPodIps: []string{"10.0.0.52"}}}}}, nil }, }, ParkedRequestConfig{}, nil) @@ -325,7 +334,7 @@ func TestHandleRequestHeaders(t *testing.T) { authority: testUUID + ".team-a.actors.resources.substrate.ate.dev", resumeResp: &ateapipb.ResumeActorResponse{ Actor: &ateapipb.Actor{ - Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIps: []string{"invalid-ip"}}}, + Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerSpiffeId: testWorkerSPIFFEID, WorkerPodIps: []string{"invalid-ip"}}}, }, }, expectErr: true, @@ -344,12 +353,24 @@ func TestHandleRequestHeaders(t *testing.T) { expectedErrStr: `actor team-a/123e4567-e89b-12d3-a456-426614174000 routing failed`, expectedStatus: envoy_type.StatusCode_InternalServerError, }, + { + name: "No worker SPIFFE ID from resume returns 500", + authority: testUUID + ".team-a.actors.resources.substrate.ate.dev", + resumeResp: &ateapipb.ResumeActorResponse{ + Actor: &ateapipb.Actor{ + Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIps: []string{"10.0.0.52"}}}, + }, + }, + expectErr: true, + expectedErrStr: `actor team-a/123e4567-e89b-12d3-a456-426614174000 routing failed`, + expectedStatus: envoy_type.StatusCode_InternalServerError, + }, { name: "successful resume ignores host and port for actor routing", authority: "127.0.0.1:44681", resumeResp: &ateapipb.ResumeActorResponse{ Actor: &ateapipb.Actor{ - Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIps: []string{"10.0.0.52"}}}, + Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerSpiffeId: testWorkerSPIFFEID, WorkerPodIps: []string{"10.0.0.52"}}}, }, }, expectErr: false, @@ -361,7 +382,7 @@ func TestHandleRequestHeaders(t *testing.T) { authority: "127.0.0.1:44681", resumeResp: &ateapipb.ResumeActorResponse{ Actor: &ateapipb.Actor{ - Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIps: []string{"fd00::52", "10.0.0.52"}}}, + Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerSpiffeId: testWorkerSPIFFEID, WorkerPodIps: []string{"fd00::52", "10.0.0.52"}}}, }, }, expectErr: false, @@ -441,7 +462,7 @@ func TestHandleRequestHeaders(t *testing.T) { for _, headerOption := range mutation.GetSetHeaders() { gotMutations[strings.ToLower(headerOption.Header.Key)] = string(headerOption.Header.RawValue) } - if _, ok := gotMutations[strings.ToLower(atunnel.TargetPortHeader)]; ok { + if _, ok := gotMutations[strings.ToLower(atunnelingress.TargetPortHeader)]; ok { t.Errorf("target port must be emitted only as dynamic metadata") } if got, want := gotMutations[strings.ToLower(atenet.TargetActorHeader)], "team-a/"+testUUID; got != want { @@ -453,6 +474,9 @@ func TestHandleRequestHeaders(t *testing.T) { if got := dynamicMetadataPort(res.DynamicMetadata); got != tc.expectedTargetPort { t.Errorf("dynamic metadata port = %q, want %q", got, tc.expectedTargetPort) } + if got, want := dynamicMetadataWorkerSPIFFEID(res.DynamicMetadata), testWorkerSPIFFEID; got != want { + t.Errorf("dynamic metadata worker SPIFFE ID = %q, want %q", got, want) + } }) } } @@ -463,14 +487,14 @@ func TestHandleRequestHeaders(t *testing.T) { // produces the same ":443" original-dst mutation as an ordinary // request (the router only ever dials the worker's atunnel server), and // reports the arbitrary port itself in dynamic metadata for Envoy to write to -// atunnel.TargetPortHeader. +// atunnelingress.TargetPortHeader. func TestHandleRequestHeadersHandlesConnectMethod(t *testing.T) { const testUUID = "123e4567-e89b-12d3-a456-426614174000" authority := testUUID + ".team-a.actors.resources.substrate.ate.dev:9090" clientMock := &mockClient{ resumeFn: func(ctx context.Context, in *ateapipb.ResumeActorRequest, opts ...grpc.CallOption) (*ateapipb.ResumeActorResponse, error) { - return &ateapipb.ResumeActorResponse{Actor: &ateapipb.Actor{Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIps: []string{"10.0.0.52"}}}}}, nil + return &ateapipb.ResumeActorResponse{Actor: &ateapipb.Actor{Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerSpiffeId: testWorkerSPIFFEID, WorkerPodIps: []string{"10.0.0.52"}}}}}, nil }, } h := New(clientMock, ParkedRequestConfig{}, nil) @@ -503,7 +527,7 @@ func TestHandleRequestHeadersUsesRetainedConnectAuthorityForPort(t *testing.T) { clientMock := &mockClient{ resumeFn: func(context.Context, *ateapipb.ResumeActorRequest, ...grpc.CallOption) (*ateapipb.ResumeActorResponse, error) { return &ateapipb.ResumeActorResponse{Actor: &ateapipb.Actor{ - Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIps: []string{"10.0.0.52"}}}, + Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerSpiffeId: testWorkerSPIFFEID, WorkerPodIps: []string{"10.0.0.52"}}}, }}, nil }, } @@ -655,7 +679,7 @@ func TestHandleRequestHeadersConnectRequiresOuterTargetActor(t *testing.T) { atomic.AddInt32(&resumeCalls, 1) gotActorRef = in.GetActor().GetAtespace() + "/" + in.GetActor().GetName() return &ateapipb.ResumeActorResponse{Actor: &ateapipb.Actor{ - Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIps: []string{"10.0.0.52"}}}, + Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerSpiffeId: testWorkerSPIFFEID, WorkerPodIps: []string{"10.0.0.52"}}}, }}, nil }, } @@ -720,7 +744,7 @@ func TestHandleRequestHeaders_FullLotServesRunningActor(t *testing.T) { Actor: &ateapipb.Actor{ Status: &ateapipb.ActorStatus{ State: ateapipb.ActorState_ACTOR_STATE_RUNNING, - WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIps: []string{"10.0.0.1"}}, + WorkerAssignment: &ateapipb.WorkerAssignment{WorkerSpiffeId: testWorkerSPIFFEID, WorkerPodIps: []string{"10.0.0.1"}}, }, }, }, nil diff --git a/cmd/atenet/internal/router/ingress/metrics.go b/cmd/atenet/internal/router/ingress/metrics.go index ee79c33559..23e3c7718a 100644 --- a/cmd/atenet/internal/router/ingress/metrics.go +++ b/cmd/atenet/internal/router/ingress/metrics.go @@ -23,8 +23,8 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" - "github.com/agent-substrate/substrate/internal/ateattr" + "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" + "go.ate.dev/substrate/internal/ateattr" ) // Request-parking instruments. parking.active is the live count of parked diff --git a/cmd/atenet/internal/router/ingress/metrics_test.go b/cmd/atenet/internal/router/ingress/metrics_test.go index ad4db89556..f6c2acec0e 100644 --- a/cmd/atenet/internal/router/ingress/metrics_test.go +++ b/cmd/atenet/internal/router/ingress/metrics_test.go @@ -23,7 +23,7 @@ import ( sdkmetric "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/metricdata" - "github.com/agent-substrate/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/ateattr" ) // newTestParkingMetrics returns ParkingMetrics whose rejected counter reports diff --git a/cmd/atenet/internal/router/ingress/parking_test.go b/cmd/atenet/internal/router/ingress/parking_test.go index 332f4528a4..06c139225c 100644 --- a/cmd/atenet/internal/router/ingress/parking_test.go +++ b/cmd/atenet/internal/router/ingress/parking_test.go @@ -20,7 +20,7 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/ateattr" ) func TestParkingLot_CapacityAndRelease(t *testing.T) { diff --git a/cmd/atenet/internal/router/ingress/resumer.go b/cmd/atenet/internal/router/ingress/resumer.go index 58ac0dded6..d522d3df1a 100644 --- a/cmd/atenet/internal/router/ingress/resumer.go +++ b/cmd/atenet/internal/router/ingress/resumer.go @@ -21,10 +21,10 @@ import ( "sync/atomic" "time" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/trace" "google.golang.org/grpc/codes" diff --git a/cmd/atenet/internal/router/ingress/resumer_test.go b/cmd/atenet/internal/router/ingress/resumer_test.go index 965bfa941f..3579ddf4dd 100644 --- a/cmd/atenet/internal/router/ingress/resumer_test.go +++ b/cmd/atenet/internal/router/ingress/resumer_test.go @@ -24,11 +24,11 @@ import ( "testing/synctest" "time" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" "google.golang.org/grpc/codes" diff --git a/cmd/atenet/internal/router/router.go b/cmd/atenet/internal/router/router.go index 724ab979f9..5c84e1964a 100644 --- a/cmd/atenet/internal/router/router.go +++ b/cmd/atenet/internal/router/router.go @@ -36,15 +36,15 @@ import ( "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/controller-runtime/pkg/client/config" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/egress" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" - "github.com/agent-substrate/substrate/internal/ateapiauth" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/serverboot" - "github.com/agent-substrate/substrate/internal/version" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - "github.com/agent-substrate/substrate/pkg/proto/credproviderpb" + "go.ate.dev/substrate/cmd/atenet/internal/router/egress" + "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" + "go.ate.dev/substrate/cmd/atenet/internal/router/ingress" + "go.ate.dev/substrate/internal/ateapiauth" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/serverboot" + "go.ate.dev/substrate/internal/version" + "go.ate.dev/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/credproviderpb" ) // dataPlaneTraceRatio is the default root sampling fraction for parentless diff --git a/cmd/atenet/internal/router/status.go b/cmd/atenet/internal/router/status.go index 2afae673a1..1f9907dfdc 100644 --- a/cmd/atenet/internal/router/status.go +++ b/cmd/atenet/internal/router/status.go @@ -32,8 +32,8 @@ import ( "go.opentelemetry.io/otel" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" + "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" + "go.ate.dev/substrate/cmd/atenet/internal/router/ingress" ) var BuildTag = "dev" diff --git a/cmd/atenet/internal/router/status_test.go b/cmd/atenet/internal/router/status_test.go index 02f3055a5c..43b973f044 100644 --- a/cmd/atenet/internal/router/status_test.go +++ b/cmd/atenet/internal/router/status_test.go @@ -36,8 +36,8 @@ import ( corev1 "k8s.io/api/core/v1" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" + "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" + "go.ate.dev/substrate/cmd/atenet/internal/router/ingress" ) func TestStatuszEndpoint(t *testing.T) { diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index a3b3089426..bca5cb0f52 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -34,7 +34,8 @@ import ( "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/wrapperspb" - "github.com/agent-substrate/substrate/internal/atenet" + "go.ate.dev/substrate/internal/atenet" + "go.ate.dev/substrate/internal/installdefaults" accesslogv3 "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3" clusterv3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" @@ -71,9 +72,9 @@ import ( resourcev3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3" serverv3 "github.com/envoyproxy/go-control-plane/pkg/server/v3" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" - "github.com/agent-substrate/substrate/internal/atunnel" + "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" + "go.ate.dev/substrate/cmd/atenet/internal/router/ingress" + atunnelingress "go.ate.dev/substrate/internal/atunnel/ingress" ) const ( @@ -202,6 +203,14 @@ const ( tunnelPortFilterStateKey = "dev.ate.tunnel.port" ) +// upstreamSANsFilterStateKey is Envoy's list of SANs the upstream certificate +// must carry, on top of the cluster's matchers. It does not split the tunnel pool. +const upstreamSANsFilterStateKey = "envoy.network.upstream_subject_alt_names" + +// trustDomainSPIFFEPrefix starts every SPIFFE ID in the pod certificates' trust +// domain. +const trustDomainSPIFFEPrefix = "spiffe://" + installdefaults.AteletTrustDomain + "/" + // tunnelCircuitBreakerLimit lifts Envoy's default of 1024: there is a tunnel // per active actor and port, and idle ones count too. const tunnelCircuitBreakerLimit = 65536 @@ -233,12 +242,6 @@ type XdsServer struct { // client cert and validates the atunnel server against upstreamTrustBundlePath. upstreamCredentialBundlePath string upstreamTrustBundlePath string - // upstreamSpiffePrefix, when set, makes the upstream validator accept the - // atunnel server cert by matching its SPIFFE URI SAN against this prefix - // (trust-domain match) instead of the actor's ephemeral pod IP. The atunnel - // cert carries only a spiffe:// URI SAN, so without this Envoy's default - // SAN check against the dialed IP fails ("verify SAN list"). - upstreamSpiffePrefix string // upstreamConnect: see SetUpstreamConnect. upstreamConnect bool @@ -400,16 +403,20 @@ const otlpDefaultPort = "4317" // (cert+key concatenated) presented to the actor's atunnel ingress server; // trustBundlePath is the CA bundle used to validate that server. Empty // credentialBundlePath leaves the upstream as plaintext. -func (x *XdsServer) SetUpstreamTls(credentialBundlePath, trustBundlePath, spiffePrefix string) { +func (x *XdsServer) SetUpstreamTls(credentialBundlePath, trustBundlePath string) { x.mu.Lock() defer x.mu.Unlock() x.upstreamCredentialBundlePath = credentialBundlePath x.upstreamTrustBundlePath = trustBundlePath - x.upstreamSpiffePrefix = spiffePrefix +} + +// upstreamValidated reports whether the router verifies atunnel's certificate. +func (x *XdsServer) upstreamValidated() bool { + return x.upstreamCredentialBundlePath != "" && x.upstreamTrustBundlePath != "" } // SetUpstreamConnect makes actor traffic reach atunnel through CONNECT tunnels -// on atunnel.DefaultConnectPort, one per actor and port. It needs +// on atunnelingress.DefaultConnectPort, one per actor and port. It needs // SetUpstreamTls, since that listener only accepts mTLS. func (x *XdsServer) SetUpstreamConnect(enabled bool) { x.mu.Lock() @@ -743,8 +750,8 @@ func (x *XdsServer) buildOtlpCollectorCluster() *clusterv3.Cluster { // for the ORIGINAL_DST actor cluster, or nil when upstream mTLS is not // configured. The router presents its podidentity credential bundle as the // client cert and validates the atunnel ingress server against the trust -// bundle. Validation is by the SPIFFE URI SAN prefix (see upstreamSpiffePrefix) -// rather than the dialed pod IP. +// bundle. The cluster accepts any SPIFFE ID in the trust domain; +// workerIdentityFilterStateFilter narrows each connection to the routed worker. // // Both the client cert and the trust bundle are referenced as SDS secrets // served by this same control plane over ADS, never inlined as file @@ -769,20 +776,15 @@ func (x *XdsServer) buildUpstreamTransportSocket() *corev3.TransportSocket { // tracks ClusterTrustBundle rotation, while the SAN matcher stays // inline in the default context. Envoy merges the two, which is why // this is a combined rather than a plain SDS validation context. - defaultCtx := &tlsv3.CertificateValidationContext{} - // Validate the atunnel server by its SPIFFE URI SAN (trust-domain - // prefix) rather than the dialed pod IP. Without this, Envoy checks the - // cert SAN against the ephemeral pod IP, which the SPIFFE-only cert - // never matches. - if x.upstreamSpiffePrefix != "" { - defaultCtx.MatchTypedSubjectAltNames = []*tlsv3.SubjectAltNameMatcher{ + defaultCtx := &tlsv3.CertificateValidationContext{ + MatchTypedSubjectAltNames: []*tlsv3.SubjectAltNameMatcher{ { SanType: tlsv3.SubjectAltNameMatcher_URI, Matcher: &matcherv3.StringMatcher{ - MatchPattern: &matcherv3.StringMatcher_Prefix{Prefix: x.upstreamSpiffePrefix}, + MatchPattern: &matcherv3.StringMatcher_Prefix{Prefix: trustDomainSPIFFEPrefix}, }, }, - } + }, } commonTls.ValidationContextType = &tlsv3.CommonTlsContext_CombinedValidationContext{ CombinedValidationContext: &tlsv3.CommonTlsContext_CombinedCertificateValidationContext{ @@ -974,7 +976,7 @@ func (x *XdsServer) buildActorConnectCluster() *clusterv3.Cluster { LbPolicy: clusterv3.Cluster_CLUSTER_PROVIDED, LbConfig: &clusterv3.Cluster_OriginalDstLbConfig_{ OriginalDstLbConfig: &clusterv3.Cluster_OriginalDstLbConfig{ - UpstreamPortOverride: wrapperspb.UInt32(atunnel.DefaultConnectPort), + UpstreamPortOverride: wrapperspb.UInt32(atunnelingress.DefaultConnectPort), }, }, TransportSocket: x.buildUpstreamTransportSocket(), @@ -1022,7 +1024,7 @@ func (x *XdsServer) buildRoutes() *routev3.RouteConfiguration { RequestHeadersToAdd: []*corev3.HeaderValueOption{ { Header: &corev3.HeaderValue{ - Key: atunnel.TargetPortHeader, + Key: atunnelingress.TargetPortHeader, Value: fmt.Sprintf("%%DYNAMIC_METADATA(%s:%s)%%", ingress.OriginalDstMetadataKey, ingress.OriginalDstPortKey), }, @@ -1204,6 +1206,17 @@ func actorTunnelFilterStateFilter() *hcmv3.HttpFilter { ) } +// workerIdentityFilterStateFilter requires atunnel's certificate to carry the +// worker SPIFFE ID ext_proc resolved. Sharing it once with the upstream also +// covers the CONNECT tunnel's connection; missing metadata becomes "-", which +// never matches. +func workerIdentityFilterStateFilter() *hcmv3.HttpFilter { + workerID := "%DYNAMIC_METADATA(" + ingress.OriginalDstMetadataKey + ":" + ingress.WorkerSPIFFEIDKey + ")%" + return setFilterStateHTTPFilter( + filterStateValue(upstreamSANsFilterStateKey, "", workerID, setfilterstatecommonv3.FilterStateValue_ONCE), + ) +} + // buildActorConnectListener turns each connection from an actor_tunnel cluster // into a CONNECT to atunnel, which takes the port from the authority and // ignores the host. @@ -1399,6 +1412,9 @@ func (x *XdsServer) buildHcm(statPrefix string, captureActorRouting bool) *anypb if x.upstreamConnect { httpFilters = append(httpFilters, actorTunnelFilterStateFilter()) } + if x.upstreamValidated() { + httpFilters = append(httpFilters, workerIdentityFilterStateFilter()) + } httpFilters = append(httpFilters, &hcmv3.HttpFilter{ Name: "envoy.filters.http.router", ConfigType: &hcmv3.HttpFilter_TypedConfig{ @@ -1577,7 +1593,7 @@ func (x *XdsServer) buildHttpsListener() *listenerv3.Listener { // gRPC requires a negotiated "h2"; http/1.1 keeps plain HTTPS clients // working alongside it. HTTP/1.1-only actors are safe either way: atunnel // downgrades every non-gRPC request to HTTP/1.1 on the actor leg (see - // atunnel.protocolMirrorTransport), so offering h2 at the edge cannot + // atunnelingress.protocolMirrorTransport), so offering h2 at the edge cannot // change what an actor receives. alpn := []string{"h2", "http/1.1"} diff --git a/cmd/atenet/internal/router/xds_test.go b/cmd/atenet/internal/router/xds_test.go index f028070623..7862ff559d 100644 --- a/cmd/atenet/internal/router/xds_test.go +++ b/cmd/atenet/internal/router/xds_test.go @@ -39,6 +39,7 @@ import ( listenerv3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" routev3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" streamaccesslogv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/stream/v3" + setfilterstatecommonv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/common/set_filter_state/v3" setfilterstatev3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/set_filter_state/v3" hcmv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" tlsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" @@ -48,9 +49,9 @@ import ( cachev3 "github.com/envoyproxy/go-control-plane/pkg/cache/v3" resourcev3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" - "github.com/agent-substrate/substrate/internal/atunnel" + "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" + "go.ate.dev/substrate/cmd/atenet/internal/router/ingress" + atunnelingress "go.ate.dev/substrate/internal/atunnel/ingress" ) func TestActorRoutingFilterStateFilter(t *testing.T) { @@ -800,13 +801,12 @@ func rotateProjectedVolume(t *testing.T, dir, newTsDir, oldTsDir string, bundle // likewise never picked up. func TestUpstreamTransportSocket_DeliversCertsOverSds(t *testing.T) { const ( - credPath = "/run/podidentity.podcert.ate.dev/credential-bundle.pem" - trustPath = "/run/podidentity.podcert.ate.dev/trust-bundle.pem" - spiffePrefix = "spiffe://cluster.local/" + credPath = "/run/podidentity.podcert.ate.dev/credential-bundle.pem" + trustPath = "/run/podidentity.podcert.ate.dev/trust-bundle.pem" ) server := NewXdsServer(18000) - server.SetUpstreamTls(credPath, trustPath, spiffePrefix) + server.SetUpstreamTls(credPath, trustPath) ts := server.buildUpstreamTransportSocket() if ts == nil { @@ -849,10 +849,8 @@ func TestUpstreamTransportSocket_DeliversCertsOverSds(t *testing.T) { t.Error("Expected the trust bundle SDS config to use the ADS config source") } - // The SPIFFE SAN matcher has to survive the move into the combined - // context's default half. Losing it silently downgrades validation to a - // SAN check against the dialed pod IP, which the SPIFFE-only atunnel - // server cert never carries. + // The trust-domain matcher has to survive the move into the combined + // context: it still holds if the per-request filter is ever skipped. sans := combined.GetDefaultValidationContext().GetMatchTypedSubjectAltNames() if len(sans) != 1 { t.Fatalf("Expected 1 SAN matcher, got %d", len(sans)) @@ -860,8 +858,80 @@ func TestUpstreamTransportSocket_DeliversCertsOverSds(t *testing.T) { if got := sans[0].GetSanType(); got != tlsv3.SubjectAltNameMatcher_URI { t.Errorf("Expected a URI SAN matcher, got %v", got) } - if got := sans[0].GetMatcher().GetPrefix(); got != spiffePrefix { - t.Errorf("Expected SAN prefix '%s', got '%s'", spiffePrefix, got) + if got, want := sans[0].GetMatcher().GetPrefix(), "spiffe://cluster.local/"; got != want { + t.Errorf("Expected SAN prefix '%s', got '%s'", want, got) + } +} + +// TestXdsServer_WorkerIdentityFilter checks every ext_proc HCM requires the +// routed worker's identity from atunnel, on both the HTTPS and CONNECT paths. +func TestXdsServer_WorkerIdentityFilter(t *testing.T) { + hcmFilters := func(t *testing.T, l *listenerv3.Listener) []*hcmv3.HttpFilter { + t.Helper() + hcm := &hcmv3.HttpConnectionManager{} + if err := l.GetFilterChains()[0].GetFilters()[0].GetTypedConfig().UnmarshalTo(hcm); err != nil { + t.Fatalf("listener %s: unmarshal HCM: %v", l.GetName(), err) + } + return hcm.GetHttpFilters() + } + setsUpstreamSANs := func(t *testing.T, f *hcmv3.HttpFilter) *setfilterstatecommonv3.FilterStateValue { + t.Helper() + if f.GetName() != "envoy.filters.http.set_filter_state" { + return nil + } + config := &setfilterstatev3.Config{} + if err := f.GetTypedConfig().UnmarshalTo(config); err != nil { + t.Fatalf("unmarshal set_filter_state: %v", err) + } + for _, v := range config.GetOnRequestHeaders() { + if v.GetObjectKey() == "envoy.network.upstream_subject_alt_names" { + return v + } + } + return nil + } + + for _, upstreamConnect := range []bool{false, true} { + server := NewXdsServer(18000) + server.SetConfig(8085, 50053, "127.0.0.1") + server.SetConnectPorts(8081, 0) + server.SetUpstreamTls("/run/bundle.pem", "/run/trust.pem") + server.SetUpstreamConnect(upstreamConnect) + listeners := snapshotListeners(t, server) + for _, name := range []string{IngressHTTPListener, MainInternalName} { + filters := hcmFilters(t, listeners[name]) + n := len(filters) + if n < 2 || filters[n-1].GetName() != "envoy.filters.http.router" { + t.Fatalf("upstreamConnect=%v listener %s: filters do not end with the router", upstreamConnect, name) + } + // Last before the router, so ext_proc has written the metadata. + v := setsUpstreamSANs(t, filters[n-2]) + if v == nil { + t.Fatalf("upstreamConnect=%v listener %s: filter before the router does not set the upstream SANs", upstreamConnect, name) + } + if got, want := v.GetFormatString().GetTextFormatSource().GetInlineString(), "%DYNAMIC_METADATA(envoy.filters.listener.original_dst:worker_spiffe_id)%"; got != want { + t.Errorf("upstreamConnect=%v listener %s: upstream SANs = %q, want %q", upstreamConnect, name, got, want) + } + // ONCE carries it to the tunnel's own connection to atunnel. + if got := v.GetSharedWithUpstream(); got != setfilterstatecommonv3.FilterStateValue_ONCE { + t.Errorf("upstreamConnect=%v listener %s: shared with upstream = %v, want ONCE", upstreamConnect, name, got) + } + // A factory key would make set_filter_state build a plain string + // that Envoy never reads as the SAN list. + if got := v.GetFactoryKey(); got != "" { + t.Errorf("upstreamConnect=%v listener %s: factory key = %q, want none", upstreamConnect, name, got) + } + } + } + + // Without a trust bundle nothing is verified, so there is nothing to narrow. + server := NewXdsServer(18000) + server.SetConfig(8085, 50053, "127.0.0.1") + server.SetUpstreamTls("/run/bundle.pem", "") + for _, f := range hcmFilters(t, snapshotListeners(t, server)[IngressHTTPListener]) { + if setsUpstreamSANs(t, f) != nil { + t.Error("upstream SANs set although atunnel's certificate is not verified") + } } } @@ -878,7 +948,7 @@ func TestXdsServer_UpdateSnapshot_UpstreamSecrets(t *testing.T) { t.Run("PublishedWhenConfigured", func(t *testing.T) { server := NewXdsServer(18000) server.SetConfig(8085, 50053, "127.0.0.1") - server.SetUpstreamTls(credPath, trustPath, "spiffe://cluster.local/") + server.SetUpstreamTls(credPath, trustPath) secrets := snapshotSecrets(t, server) if len(secrets) != 2 { @@ -925,7 +995,7 @@ func TestXdsServer_UpdateSnapshot_UpstreamSecrets(t *testing.T) { t.Run("TrustSecretOmittedWithoutTrustBundle", func(t *testing.T) { server := NewXdsServer(18000) server.SetConfig(8085, 50053, "127.0.0.1") - server.SetUpstreamTls(credPath, "", "") + server.SetUpstreamTls(credPath, "") secrets := snapshotSecrets(t, server) if len(secrets) != 1 { @@ -953,7 +1023,7 @@ func TestUpstreamCertSecret_ProjectedVolumeRotation(t *testing.T) { writeProjectedVolume(t, dir, tsDirA, bundleA) server := NewXdsServer(18000) - server.SetUpstreamTls(credPath, "", "") + server.SetUpstreamTls(credPath, "") tlsCert := server.buildUpstreamCertSecret().GetTlsCertificate() chainPath := tlsCert.GetCertificateChain().GetFilename() @@ -1172,7 +1242,7 @@ func TestXdsServer_BuildOriginalDstCluster_UsesMetadataKey(t *testing.T) { // TestXdsServer_ActorClusterProtocolOptions pins where downstream-protocol // mirroring is allowed: only on the mTLS atunnel leg, where atunnel guards // HTTP/1.1-only actors by downgrading non-gRPC HTTP/2 (see -// atunnel.protocolMirrorTransport). The legacy plaintext cluster dials the +// atunnelingress.protocolMirrorTransport). The legacy plaintext cluster dials the // actor directly with no such guard, so it must carry no protocol options at // all — Envoy's implicit HTTP/1.1. func TestXdsServer_ActorClusterProtocolOptions(t *testing.T) { @@ -1182,7 +1252,7 @@ func TestXdsServer_ActorClusterProtocolOptions(t *testing.T) { t.Errorf("legacy plaintext actor cluster has protocol options %v, want none (implicit HTTP/1.1)", opts) } - x.SetUpstreamTls("/run/bundle.pem", "/run/trust.pem", "spiffe://ate.dev/") + x.SetUpstreamTls("/run/bundle.pem", "/run/trust.pem") cluster := x.buildOriginalDstCluster() ts := cluster.GetTransportSocket() if ts == nil { @@ -1229,7 +1299,7 @@ func TestXdsServer_BuildRoutesWritesTargetPortHeader(t *testing.T) { t.Fatalf("route adds request headers %v, want exactly one", headers) } header := headers[0] - if got, want := header.GetHeader().GetKey(), atunnel.TargetPortHeader; got != want { + if got, want := header.GetHeader().GetKey(), atunnelingress.TargetPortHeader; got != want { t.Errorf("header key = %q, want %q", got, want) } if got, want := header.GetHeader().GetValue(), "%DYNAMIC_METADATA(envoy.filters.listener.original_dst:port)%"; got != want { @@ -1455,7 +1525,7 @@ func downstreamTLS(t *testing.T, raw any) *tlsv3.DownstreamTlsContext { // listener always offers h2 before http/1.1: gRPC over TLS requires a // negotiated "h2", and the offer is unconditional because atunnel downgrades // every non-gRPC request to HTTP/1.1 on the actor leg (see -// atunnel.protocolMirrorTransport and TestProtocolMirrorTransport), so an +// atunnelingress.protocolMirrorTransport and TestProtocolMirrorTransport), so an // HTTP/1.1-only actor cannot tell what the client negotiated at the edge. The // CONNECT-TLS listener stays ALPN-free: its clients speak HTTP/1.1 CONNECT, // and an h2 offer there would move them onto extended CONNECT the tunnel path diff --git a/cmd/atenet/internal/router/xds_upstream_connect_test.go b/cmd/atenet/internal/router/xds_upstream_connect_test.go index c243c3777a..d72c79f921 100644 --- a/cmd/atenet/internal/router/xds_upstream_connect_test.go +++ b/cmd/atenet/internal/router/xds_upstream_connect_test.go @@ -34,9 +34,9 @@ import ( resourcev3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3" "github.com/spf13/pflag" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" - "github.com/agent-substrate/substrate/internal/atenet" - "github.com/agent-substrate/substrate/internal/atunnel" + "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" + "go.ate.dev/substrate/internal/atenet" + atunnelingress "go.ate.dev/substrate/internal/atunnel/ingress" ) // upstreamConnectSnapshot also enables the CONNECT listener, so main_internal @@ -46,7 +46,7 @@ func upstreamConnectSnapshot(t *testing.T) *cachev3.Snapshot { server := NewXdsServer(18000) server.SetConfig(8085, 50053, "127.0.0.1") server.SetConnectPorts(8081, 0) - server.SetUpstreamTls("/run/bundle.pem", "/run/trust.pem", "spiffe://cluster.local/") + server.SetUpstreamTls("/run/bundle.pem", "/run/trust.pem") server.SetUpstreamConnect(true) if err := server.UpdateSnapshot(); err != nil { t.Fatalf("UpdateSnapshot failed: %v", err) @@ -173,7 +173,8 @@ func TestXdsServer_UpstreamConnectResources(t *testing.T) { } // The tunnel filter copies ext_proc's decision, and the router picks a - // tunnel by it. + // tunnel by it. The worker identity filter follows it (see + // TestXdsServer_WorkerIdentityFilter). for _, name := range []string{IngressHTTPListener, MainInternalName} { hcm := &hcmv3.HttpConnectionManager{} if err := listeners[name].(*listenerv3.Listener).GetFilterChains()[0].GetFilters()[0].GetTypedConfig().UnmarshalTo(hcm); err != nil { @@ -185,11 +186,11 @@ func TestXdsServer_UpstreamConnectResources(t *testing.T) { names = append(names, f.GetName()) } n := len(filters) - if n < 3 || names[n-3] != httpExtProcFilterName || names[n-2] != "envoy.filters.http.set_filter_state" || names[n-1] != "envoy.filters.http.router" { - t.Fatalf("listener %s HTTP filters = %v, want them to end with ext_proc, set_filter_state, router", name, names) + if n < 4 || names[n-4] != httpExtProcFilterName || names[n-3] != "envoy.filters.http.set_filter_state" || names[n-1] != "envoy.filters.http.router" { + t.Fatalf("listener %s HTTP filters = %v, want them to end with ext_proc, the tunnel set_filter_state, the worker identity filter, router", name, names) } config := &setfilterstatev3.Config{} - if err := filters[n-2].GetTypedConfig().UnmarshalTo(config); err != nil { + if err := filters[n-3].GetTypedConfig().UnmarshalTo(config); err != nil { t.Fatalf("listener %s: unmarshal set_filter_state: %v", name, err) } if got := config.GetOnRequestHeaders()[0].GetObjectKey(); got != tunnelKeyFilterStateKey { @@ -387,7 +388,7 @@ func TestXdsServer_ActorTunnelClusters(t *testing.T) { } x := NewXdsServer(0) - x.SetUpstreamTls("/run/bundle.pem", "/run/trust.pem", "spiffe://cluster.local/") + x.SetUpstreamTls("/run/bundle.pem", "/run/trust.pem") c := x.buildActorConnectCluster() if c.GetType() != clusterv3.Cluster_ORIGINAL_DST { t.Errorf("actor_connect type = %v, want ORIGINAL_DST", c.GetType()) @@ -396,8 +397,8 @@ func TestXdsServer_ActorTunnelClusters(t *testing.T) { if lb.GetMetadataKey() != nil { t.Error("actor_connect reads a metadata key; the worker address comes from filter state") } - if got := lb.GetUpstreamPortOverride().GetValue(); got != atunnel.DefaultConnectPort { - t.Errorf("actor_connect port = %d, want atunnel's CONNECT port %d", got, atunnel.DefaultConnectPort) + if got := lb.GetUpstreamPortOverride().GetValue(); got != atunnelingress.DefaultConnectPort { + t.Errorf("actor_connect port = %d, want atunnel's CONNECT port %d", got, atunnelingress.DefaultConnectPort) } upstreamTLS := &tlsv3.UpstreamTlsContext{} if err := c.GetTransportSocket().GetTypedConfig().UnmarshalTo(upstreamTLS); err != nil { diff --git a/cmd/atenet/internal/sdsmint/minter.go b/cmd/atenet/internal/sdsmint/minter.go index 1ec60e30fe..c1d9bc2a4a 100644 --- a/cmd/atenet/internal/sdsmint/minter.go +++ b/cmd/atenet/internal/sdsmint/minter.go @@ -30,7 +30,7 @@ import ( "strings" "time" - "github.com/agent-substrate/substrate/internal/localca" + "go.ate.dev/substrate/internal/localca" ) // errHostNotAllowed is returned when a requested hostname will not be minted. diff --git a/cmd/atenet/internal/sdsmint/minter_test.go b/cmd/atenet/internal/sdsmint/minter_test.go index 4e84e5e55b..935c4a57cf 100644 --- a/cmd/atenet/internal/sdsmint/minter_test.go +++ b/cmd/atenet/internal/sdsmint/minter_test.go @@ -23,7 +23,7 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/localca" + "go.ate.dev/substrate/internal/localca" ) func testMinter(t *testing.T, opts minterOptions) *minter { diff --git a/cmd/atenet/internal/sdsmint/run.go b/cmd/atenet/internal/sdsmint/run.go index 15d144e54b..884508523c 100644 --- a/cmd/atenet/internal/sdsmint/run.go +++ b/cmd/atenet/internal/sdsmint/run.go @@ -27,9 +27,9 @@ import ( "syscall" "time" - "github.com/agent-substrate/substrate/internal/localca" - "github.com/agent-substrate/substrate/internal/version" secretservice "github.com/envoyproxy/go-control-plane/envoy/service/secret/v3" + "go.ate.dev/substrate/internal/localca" + "go.ate.dev/substrate/internal/version" "google.golang.org/grpc" ) diff --git a/cmd/atenet/main.go b/cmd/atenet/main.go index 097f9617f9..bc0fa57a80 100644 --- a/cmd/atenet/main.go +++ b/cmd/atenet/main.go @@ -14,7 +14,7 @@ package main -import "github.com/agent-substrate/substrate/cmd/atenet/internal" +import "go.ate.dev/substrate/cmd/atenet/internal" func main() { internal.Execute() diff --git a/cmd/ateworker-gvisor/actorpath.go b/cmd/ateworker-gvisor/actorpath.go index eef2bd318e..32ed676d7a 100644 --- a/cmd/ateworker-gvisor/actorpath.go +++ b/cmd/ateworker-gvisor/actorpath.go @@ -23,8 +23,8 @@ import ( "os" "path/filepath" - "github.com/agent-substrate/substrate/internal/imagecache" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/imagecache" + "go.ate.dev/substrate/internal/proto/ateworkerpb" ) // ociBundlePath is the container's OCI bundle. diff --git a/cmd/ateworker-gvisor/actorpath_test.go b/cmd/ateworker-gvisor/actorpath_test.go index dba4708c1d..54c40b965e 100644 --- a/cmd/ateworker-gvisor/actorpath_test.go +++ b/cmd/ateworker-gvisor/actorpath_test.go @@ -23,8 +23,8 @@ import ( "golang.org/x/sys/unix" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/roottest" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/roottest" ) func TestResetRunscStateAndPidFileDirs(t *testing.T) { diff --git a/cmd/ateworker-gvisor/ckptcgroup.go b/cmd/ateworker-gvisor/ckptcgroup.go new file mode 100644 index 0000000000..7660da6207 --- /dev/null +++ b/cmd/ateworker-gvisor/ckptcgroup.go @@ -0,0 +1,251 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + "fmt" + "io/fs" + "log/slog" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + + "go.ate.dev/substrate/internal/ocispec" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/snapshotcache" + "golang.org/x/sys/unix" +) + +// During checkpoint, moving the sentry from -_pause into a sibling +// -_ckpt cgroup (memory.max = "max", cpu.max copied from -_pause) +// prevents newly written pages.img cache from evicting clean pages or +// OOM-killing the sentry under the actor's memory.max. Removing -_ckpt +// after sandbox teardown reparents the cache to the worker scope for fast +// local Resume, and dropActorCheckpointCacheAsync unlinks non-preserved files +// in the restore dir or evicts preserved checkpoint page cache via +// POSIX_FADV_DONTNEED once background restore page loading completes. + +const ( + // checkpointCgroupSuffix uses a leading underscore like _pause so it cannot + // collide with a DNS-label application container leaf (-). + checkpointCgroupSuffix = "-_ckpt" +) + +type checkpointCgroup struct { + leaf string + dir string + pid int +} + +func checkpointCgroupDir(root, actorUID string) string { + return filepath.Join(root, actorUID+checkpointCgroupSuffix) +} + +// isolateCheckpointCache moves the actor's sentry from its pause leaf into a +// sibling -_ckpt cgroup, preserving the pause leaf's cpu.max so checkpoint +// serialization remains capped to the actor's CPU limit. +func isolateCheckpointCache(root string, actorDirs *ateworkerpb.ActorDirs, actorUID string) (*checkpointCgroup, error) { + pid, err := readSandboxPID(actorDirs) + if err != nil { + return nil, err + } + + leaf := filepath.Join(root, ocispec.GVisorCgroupLeaf(actorUID, ocispec.PauseContainer)) + if !cgroupHasPID(leaf, pid) { + return nil, fmt.Errorf("sandbox pid %d is not in cgroup %q", pid, leaf) + } + dir := checkpointCgroupDir(root, actorUID) + if err := os.Mkdir(dir, 0o755); err != nil && !errors.Is(err, fs.ErrExist) { + return nil, fmt.Errorf("while creating cgroup %q: %w", dir, err) + } + if cpuMax, err := os.ReadFile(filepath.Join(leaf, "cpu.max")); err == nil { + if err := os.WriteFile(filepath.Join(dir, "cpu.max"), cpuMax, 0o644); err != nil { + slog.Debug("Failed to copy cpu.max to checkpoint cgroup", slog.String("cgroup", dir), slog.Any("err", err)) + } + } + c := &checkpointCgroup{leaf: leaf, dir: dir, pid: pid} + if err := movePID(dir, pid); err != nil { + return nil, errors.Join(err, c.remove()) + } + return c, nil +} + +func readSandboxPID(actorDirs *ateworkerpb.ActorDirs) (int, error) { + pidFile := pidFilePath(actorDirs, ocispec.PauseContainer) + b, err := os.ReadFile(pidFile) + if err != nil { + return 0, fmt.Errorf("while reading sandbox pid file %q: %w", pidFile, err) + } + pid, err := strconv.Atoi(strings.TrimSpace(string(b))) + if err != nil || pid <= 0 { + return 0, fmt.Errorf("invalid sandbox pid %q in %q: %w", string(b), pidFile, err) + } + return pid, nil +} + +func cgroupHasPID(dir string, pid int) bool { + procs, err := os.ReadFile(filepath.Join(dir, "cgroup.procs")) + if err != nil { + return false + } + return slices.Contains(strings.Fields(string(procs)), strconv.Itoa(pid)) +} + +// movePID is a var so tests can model cgroup2 PID migration on a temp dir. +var movePID = writeCgroupPID + +func writeCgroupPID(dir string, pid int) error { + if err := os.WriteFile(filepath.Join(dir, "cgroup.procs"), []byte(strconv.Itoa(pid)), 0o644); err != nil { + return fmt.Errorf("while moving PID %d into cgroup %q: %w", pid, dir, err) + } + return nil +} + +// restore moves the sentry back into its pause leaf on checkpoint failure. It +// returns nil once the sentry has exited; any other failure leaves the sentry +// alive in the checkpoint cgroup, so the caller must terminate it. +func (c *checkpointCgroup) restore() error { + if c == nil { + return nil + } + if !cgroupHasPID(c.dir, c.pid) { + return nil + } + err := movePID(c.leaf, c.pid) + if errors.Is(err, unix.ESRCH) { + return nil + } + return err +} + +// remove deletes the empty sibling cgroup, reparenting its page cache. +func (c *checkpointCgroup) remove() error { + if c == nil { + return nil + } + if err := os.Remove(c.dir); err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("while removing cgroup %q: %w", c.dir, err) + } + return nil +} + +// checkpointIsolated isolates the sentry into a sibling cgroup (when the +// worker's cgroup is delegated), runs checkpoint, and tears down the workload +// before removing the sibling cgroup so its page cache reparents cleanly. If +// checkpoint fails, it moves the sentry back into its pause leaf (or terminates +// the workload if that rollback fails). +func checkpointIsolated( + ctx context.Context, + cgroupDelegated bool, + cgroupRoot, actorUID string, + actorDirs *ateworkerpb.ActorDirs, + checkpoint func(context.Context) error, + terminate func() error, +) error { + var ckptCgroup *checkpointCgroup + if cgroupDelegated { + var err error + ckptCgroup, err = isolateCheckpointCache(cgroupRoot, actorDirs, actorUID) + if err != nil { + slog.WarnContext(ctx, "Checkpointing without page cache isolation", + slog.String("actorUID", actorUID), slog.Any("err", err)) + } + } + terminateOnExit := true + defer func() { + if terminateOnExit { + _ = terminate() + } + err := ckptCgroup.remove() + if err != nil { + // If terminate left the sentry alive, move it back to its pause leaf. + if rerr := ckptCgroup.restore(); rerr != nil { + slog.WarnContext(ctx, "Failed to move the sentry back into its pause leaf", + slog.String("actorUID", actorUID), slog.Any("err", rerr)) + } else { + err = ckptCgroup.remove() + } + } + if err != nil { + slog.WarnContext(ctx, "Failed to remove the checkpoint cgroup", + slog.String("actorUID", actorUID), slog.Any("err", err)) + } + }() + + if err := checkpoint(ctx); err != nil { + if rerr := ckptCgroup.restore(); rerr != nil { + slog.WarnContext(ctx, "Failed to move the sentry back into its pause leaf; terminating workload", + slog.String("actorUID", actorUID), slog.Any("err", rerr)) + } else { + terminateOnExit = false + } + return err + } + return nil +} + +// removeStaleCheckpointCgroups removes empty *-_ckpt cgroups left behind on restart. +func removeStaleCheckpointCgroups(ctx context.Context, root string) { + dirs, err := filepath.Glob(filepath.Join(root, "*"+checkpointCgroupSuffix)) + if err != nil { + return + } + for _, dir := range dirs { + if err := os.Remove(dir); err != nil && !errors.Is(err, fs.ErrNotExist) { + slog.WarnContext(ctx, "Failed to remove a stale checkpoint cgroup", slog.String("cgroup", dir), slog.Any("err", err)) + } + } +} + +func isCheckpointDataFile(name string) bool { + ext := filepath.Ext(name) + return ext == ".img" || ext == ".tar" +} + +// evictCheckpointCache is a var so tests can observe eviction. +var evictCheckpointCache = snapshotcache.EvictWithRetries + +// dropActorCheckpointCacheAsync pins the snapshot data files in restoreDir +// while the caller holds the actor lock. When the restore dir is not preserved, +// it unlinks them immediately (the kernel frees pages and blocks once the +// sentry closes its open FDs). When the restore dir is preserved, it waits in +// the background (bounded by actorCtx) for async restore page loading to finish +// and evicts clean pages from the host page cache. +func dropActorCheckpointCacheAsync(actorCtx context.Context, actorUID, restoreDir string, preserveRestoreDir bool, waitRestore func(context.Context) error) { + targets := snapshotcache.FindFiles(restoreDir, isCheckpointDataFile) + if len(targets) == 0 { + return + } + if !preserveRestoreDir { + snapshotcache.DropFiles(targets) + return + } + go func() { + if waitRestore != nil { + if err := waitRestore(actorCtx); err != nil { + slog.Warn("Skipping checkpoint page cache eviction after restore wait error", + slog.String("actorUID", actorUID), slog.Any("err", err)) + return + } + } + evictCheckpointCache(targets) + }() +} diff --git a/cmd/ateworker-gvisor/ckptcgroup_test.go b/cmd/ateworker-gvisor/ckptcgroup_test.go new file mode 100644 index 0000000000..adaa737838 --- /dev/null +++ b/cmd/ateworker-gvisor/ckptcgroup_test.go @@ -0,0 +1,552 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "testing" + "time" + + "go.ate.dev/substrate/internal/ocispec" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/snapshotcache" + "golang.org/x/sys/unix" +) + +const testActorUID = "0b5f3c1e-7a2d-4e8f-9c61-2d4a8b7e5f10" + +type ckptFixture struct { + root, leaf string + actorDirs *ateworkerpb.ActorDirs +} + +func newCkptFixture(t *testing.T, sandboxPID string) ckptFixture { + t.Helper() + actorRoot := t.TempDir() + f := ckptFixture{ + root: t.TempDir(), + actorDirs: &ateworkerpb.ActorDirs{ + RootDir: actorRoot, + CheckpointDir: t.TempDir(), + }, + } + f.leaf = filepath.Join(f.root, ocispec.GVisorCgroupLeaf(testActorUID, ocispec.PauseContainer)) + mustMkdir(t, f.leaf) + mustWrite(t, filepath.Join(f.leaf, "cgroup.procs"), "100\n"+sandboxPID+"\n") + mustWrite(t, filepath.Join(f.leaf, "memory.max"), "805306368\n") + mustWrite(t, filepath.Join(f.leaf, "cpu.max"), "200000 100000\n") + + if sandboxPID != "" { + pidFile := pidFilePath(f.actorDirs, ocispec.PauseContainer) + mustMkdir(t, filepath.Dir(pidFile)) + mustWrite(t, pidFile, sandboxPID+"\n") + } + + origMovePID := movePID + t.Cleanup(func() { movePID = origMovePID }) + movePID = func(dir string, pid int) error { + if err := writeCgroupPID(dir, pid); err != nil { + return err + } + // Emulate cgroup2 migration out of the sibling checkpoint cgroup so rmdir + // succeeds once the sentry moves back into its pause leaf. + ckptDir := checkpointCgroupDir(f.root, testActorUID) + if dir != ckptDir { + _ = os.Remove(filepath.Join(ckptDir, "cgroup.procs")) + _ = os.Remove(filepath.Join(ckptDir, "cpu.max")) + } + return nil + } + return f +} + +func mustMkdir(t *testing.T, dir string) { + t.Helper() + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } +} + +func mustWrite(t *testing.T, path, data string) { + t.Helper() + if err := os.WriteFile(path, []byte(data), 0o644); err != nil { + t.Fatal(err) + } +} + +func mustRead(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func TestIsolateCheckpointCacheMovesOnlyTheSentry(t *testing.T) { + f := newCkptFixture(t, "101") + + c, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID) + if err != nil { + t.Fatalf("isolateCheckpointCache: %v", err) + } + dir := checkpointCgroupDir(f.root, testActorUID) + if got := mustRead(t, filepath.Join(dir, "cgroup.procs")); got != "101" { + t.Errorf("checkpoint cgroup.procs = %q, want the sentry's PID 101", got) + } + if got := mustRead(t, filepath.Join(dir, "cpu.max")); got != "200000 100000\n" { + t.Errorf("checkpoint cpu.max = %q, want \"200000 100000\\n\"", got) + } + if _, err := os.Stat(filepath.Join(dir, "memory.max")); !os.IsNotExist(err) { + t.Errorf("checkpoint memory.max should not be set, stat err = %v", err) + } + + if err := c.restore(); err != nil { + t.Fatalf("restore: %v", err) + } + if got := mustRead(t, filepath.Join(f.leaf, "cgroup.procs")); got != "101" { + t.Errorf("pause leaf cgroup.procs after restore = %q, want 101", got) + } + if err := c.remove(); err != nil { + t.Fatalf("remove: %v", err) + } +} + +func TestIsolateCheckpointCacheWithoutPIDFile(t *testing.T) { + f := newCkptFixture(t, "") + + if _, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID); err == nil { + t.Fatal("isolateCheckpointCache succeeded with no pause pid file") + } + if _, err := os.Stat(checkpointCgroupDir(f.root, testActorUID)); !os.IsNotExist(err) { + t.Errorf("checkpoint cgroup left behind: stat err = %v", err) + } +} + +func TestIsolateCheckpointCacheRejectsPIDOutsidePauseLeaf(t *testing.T) { + f := newCkptFixture(t, "101") + mustWrite(t, filepath.Join(f.leaf, "cgroup.procs"), "100\n") + + if _, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID); err == nil { + t.Fatal("isolateCheckpointCache succeeded with the pid outside the pause leaf") + } + if _, err := os.Stat(checkpointCgroupDir(f.root, testActorUID)); !os.IsNotExist(err) { + t.Errorf("checkpoint cgroup created for a reused pid: stat err = %v", err) + } + if got := mustRead(t, filepath.Join(f.leaf, "cgroup.procs")); got != "100\n" { + t.Errorf("pause leaf cgroup.procs = %q, want unchanged \"100\\n\"", got) + } +} + +func TestCheckpointCgroupRestoreAfterLeafGone(t *testing.T) { + f := newCkptFixture(t, "101") + c, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID) + if err != nil { + t.Fatalf("isolateCheckpointCache: %v", err) + } + if err := os.RemoveAll(f.leaf); err != nil { + t.Fatal(err) + } + // The sentry is still alive in the checkpoint cgroup with nowhere to go + // back to; the caller must terminate it rather than leave it unlimited. + if err := c.restore(); !errors.Is(err, os.ErrNotExist) { + t.Errorf("restore with the leaf gone = %v, want ErrNotExist", err) + } + if got := mustRead(t, filepath.Join(c.dir, "cgroup.procs")); got != "101" { + t.Errorf("checkpoint cgroup.procs = %q, want the live sentry 101", got) + } +} + +func TestCheckpointCgroupRestoreSkipsExitedSentry(t *testing.T) { + f := newCkptFixture(t, "101") + c, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID) + if err != nil { + t.Fatalf("isolateCheckpointCache: %v", err) + } + + // If the sentry exited (no longer in ckpt cgroup.procs) and its PID was + // recycled elsewhere, restore must not move it into the pause leaf. + mustWrite(t, filepath.Join(c.dir, "cgroup.procs"), "") + mustWrite(t, filepath.Join(f.leaf, "cgroup.procs"), "100\n") + if err := c.restore(); err != nil { + t.Fatalf("restore with exited sentry: %v", err) + } + if got := mustRead(t, filepath.Join(f.leaf, "cgroup.procs")); got != "100\n" { + t.Errorf("pause leaf cgroup.procs = %q, want unchanged \"100\\n\"", got) + } + + // If the sentry exits between cgroupHasPID and movePID (ESRCH), restore returns nil. + mustWrite(t, filepath.Join(c.dir, "cgroup.procs"), "101\n") + movePID = func(string, int) error { return unix.ESRCH } + if err := c.restore(); err != nil { + t.Errorf("restore on ESRCH = %v, want nil", err) + } +} + +func TestCheckpointCgroupNilIsNoop(t *testing.T) { + var c *checkpointCgroup + if err := c.restore(); err != nil { + t.Errorf("nil restore = %v", err) + } + if err := c.remove(); err != nil { + t.Errorf("nil remove = %v", err) + } +} + +func TestCheckpointIsolated(t *testing.T) { + t.Run("terminates before removing checkpoint cgroup on success", func(t *testing.T) { + f := newCkptFixture(t, "101") + ckptDir := checkpointCgroupDir(f.root, testActorUID) + var terminated bool + + err := checkpointIsolated( + context.Background(), + true, f.root, testActorUID, f.actorDirs, + func(context.Context) error { + if got := mustRead(t, filepath.Join(ckptDir, "cgroup.procs")); got != "101" { + t.Errorf("checkpoint cgroup.procs during checkpoint = %q, want 101", got) + } + return nil + }, + func() error { + terminated = true + // Emulate container teardown emptying the checkpoint cgroup so rmdir succeeds. + _ = os.Remove(filepath.Join(ckptDir, "cpu.max")) + return os.Remove(filepath.Join(ckptDir, "cgroup.procs")) + }, + ) + if err != nil { + t.Fatalf("checkpointIsolated: %v", err) + } + if !terminated { + t.Error("terminate was not called") + } + if _, err := os.Stat(ckptDir); !os.IsNotExist(err) { + t.Errorf("checkpoint cgroup still exists after checkpointIsolated: stat err = %v", err) + } + }) + + t.Run("skips cgroup isolation when cgroup is not delegated", func(t *testing.T) { + f := newCkptFixture(t, "") + ckptDir := checkpointCgroupDir(f.root, testActorUID) + var terminated bool + + err := checkpointIsolated( + context.Background(), + false, f.root, testActorUID, f.actorDirs, + func(context.Context) error { + if _, err := os.Stat(ckptDir); !os.IsNotExist(err) { + t.Errorf("checkpoint cgroup should not be created when !cgroupDelegated: %v", err) + } + return nil + }, + func() error { + terminated = true + return nil + }, + ) + if err != nil { + t.Fatalf("checkpointIsolated: %v", err) + } + if !terminated { + t.Error("terminate was not called") + } + }) + + t.Run("restores sentry and keeps workload alive when checkpoint fails", func(t *testing.T) { + f := newCkptFixture(t, "101") + ckptDir := checkpointCgroupDir(f.root, testActorUID) + var terminated bool + + err := checkpointIsolated( + context.Background(), + true, f.root, testActorUID, f.actorDirs, + func(context.Context) error { + return errors.New("checkpoint boom") + }, + func() error { + terminated = true + return nil + }, + ) + if err == nil || !strings.Contains(err.Error(), "checkpoint boom") { + t.Fatalf("checkpointIsolated err = %v, want checkpoint boom", err) + } + if terminated { + t.Error("terminate called after successful cgroup rollback") + } + if got := mustRead(t, filepath.Join(f.leaf, "cgroup.procs")); got != "101" { + t.Errorf("pause leaf cgroup.procs = %q, want 101", got) + } + if _, err := os.Stat(ckptDir); !os.IsNotExist(err) { + t.Errorf("checkpoint cgroup left behind: stat err = %v", err) + } + }) + + t.Run("terminates workload if both checkpoint and cgroup restore fail", func(t *testing.T) { + f := newCkptFixture(t, "101") + ckptDir := checkpointCgroupDir(f.root, testActorUID) + var terminated bool + + err := checkpointIsolated( + context.Background(), + true, f.root, testActorUID, f.actorDirs, + func(context.Context) error { + movePID = func(string, int) error { return unix.EACCES } + return errors.New("checkpoint boom") + }, + func() error { + terminated = true + _ = os.Remove(filepath.Join(ckptDir, "cpu.max")) + return os.Remove(filepath.Join(ckptDir, "cgroup.procs")) + }, + ) + if err == nil { + t.Fatal("checkpointIsolated succeeded on checkpoint error") + } + if !terminated { + t.Error("terminate was not called when restore failed; sentry would stay in unlimited cgroup") + } + if _, err := os.Stat(ckptDir); !os.IsNotExist(err) { + t.Errorf("checkpoint cgroup left behind: stat err = %v", err) + } + }) + + t.Run("terminates workload if checkpoint fails and the pause leaf is gone", func(t *testing.T) { + f := newCkptFixture(t, "101") + ckptDir := checkpointCgroupDir(f.root, testActorUID) + var terminated bool + + err := checkpointIsolated( + context.Background(), + true, f.root, testActorUID, f.actorDirs, + func(context.Context) error { + if err := os.RemoveAll(f.leaf); err != nil { + t.Fatal(err) + } + return errors.New("checkpoint boom") + }, + func() error { + terminated = true + _ = os.Remove(filepath.Join(ckptDir, "cpu.max")) + return os.Remove(filepath.Join(ckptDir, "cgroup.procs")) + }, + ) + if err == nil { + t.Fatal("checkpointIsolated succeeded on checkpoint error") + } + if !terminated { + t.Error("terminate was not called with the pause leaf gone; sentry would stay in unlimited cgroup") + } + if _, err := os.Stat(ckptDir); !os.IsNotExist(err) { + t.Errorf("checkpoint cgroup left behind: stat err = %v", err) + } + }) + + t.Run("moves the sentry back into its pause leaf when terminate leaves it running", func(t *testing.T) { + f := newCkptFixture(t, "101") + ckptDir := checkpointCgroupDir(f.root, testActorUID) + + err := checkpointIsolated( + context.Background(), + true, f.root, testActorUID, f.actorDirs, + func(context.Context) error { return nil }, + func() error { + // Non-empty ckptDir emulates a sentry still alive in the cgroup. + return errors.New("terminate boom") + }, + ) + if err != nil { + t.Fatalf("checkpointIsolated: %v", err) + } + if got := mustRead(t, filepath.Join(f.leaf, "cgroup.procs")); got != "101" { + t.Errorf("pause leaf cgroup.procs = %q, want the live sentry 101 back under its memory.max", got) + } + if _, err := os.Stat(ckptDir); !os.IsNotExist(err) { + t.Errorf("checkpoint cgroup left behind: stat err = %v", err) + } + }) +} + +func TestRemoveStaleCheckpointCgroups(t *testing.T) { + root := t.TempDir() + stale := checkpointCgroupDir(root, testActorUID) + mustMkdir(t, stale) + busy := checkpointCgroupDir(root, "busy") + mustMkdir(t, busy) + // A non-empty directory stands in for a cgroup that still has processes: + // rmdir fails on both. + mustWrite(t, filepath.Join(busy, "cgroup.procs"), strconv.Itoa(42)) + leaf := filepath.Join(root, ocispec.GVisorCgroupLeaf(testActorUID, ocispec.PauseContainer)) + mustMkdir(t, leaf) + + removeStaleCheckpointCgroups(context.Background(), root) + + if _, err := os.Stat(stale); !os.IsNotExist(err) { + t.Errorf("stale checkpoint cgroup not removed: stat err = %v", err) + } + if _, err := os.Stat(busy); err != nil { + t.Errorf("busy checkpoint cgroup removed: %v", err) + } + if _, err := os.Stat(leaf); err != nil { + t.Errorf("pause leaf removed: %v", err) + } +} + +func newRestoreDir(t *testing.T) string { + t.Helper() + restoreDir := filepath.Join(t.TempDir(), "restore-dir") + mustMkdir(t, filepath.Join(restoreDir, "fs")) + for _, name := range []string{"checkpoint.img", "pages.img", "fs/pages.img", "fs/fscheckpoint.pb", "manifest.json"} { + mustWrite(t, filepath.Join(restoreDir, name), "staged data") + } + return restoreDir +} + +func captureEviction(t *testing.T) <-chan []snapshotcache.File { + t.Helper() + evicted := make(chan []snapshotcache.File, 1) + orig := evictCheckpointCache + t.Cleanup(func() { evictCheckpointCache = orig }) + evictCheckpointCache = func(targets []snapshotcache.File) { evicted <- targets } + return evicted +} + +func TestDropStagedRestoreFiles(t *testing.T) { + restoreDir := newRestoreDir(t) + evicted := captureEviction(t) + + waitCalled := false + dropActorCheckpointCacheAsync(context.Background(), testActorUID, restoreDir, false, func(context.Context) error { + waitCalled = true + return errors.New("waitRestore should not be called when !preserveRestoreDir") + }) + if waitCalled { + t.Fatal("waitRestore was called when !preserveRestoreDir") + } + + for _, name := range []string{"checkpoint.img", "pages.img", "fs/pages.img"} { + if _, err := os.Stat(filepath.Join(restoreDir, name)); !os.IsNotExist(err) { + t.Fatalf("staged %s still exists after dropActorCheckpointCacheAsync: err=%v", name, err) + } + } + for _, name := range []string{"fs/fscheckpoint.pb", "manifest.json"} { + if _, err := os.Stat(filepath.Join(restoreDir, name)); err != nil { + t.Fatalf("%s should be preserved: %v", name, err) + } + } + select { + case <-evicted: + t.Fatal("dropped files were also evicted") + default: + } +} + +func TestEvictPreservedRestoreDir(t *testing.T) { + t.Run("evicts once the restore wait completes", func(t *testing.T) { + restoreDir := newRestoreDir(t) + evicted := captureEviction(t) + restoreDone := make(chan struct{}) + + dropActorCheckpointCacheAsync(context.Background(), testActorUID, restoreDir, true, func(ctx context.Context) error { + select { + case <-restoreDone: + return nil + case <-ctx.Done(): + return ctx.Err() + } + }) + + select { + case <-evicted: + t.Fatal("evicted before the restore wait completed") + case <-time.After(50 * time.Millisecond): + } + close(restoreDone) + select { + case targets := <-evicted: + var names []string + for _, f := range targets { + rel, _ := filepath.Rel(restoreDir, f.Path) + names = append(names, rel) + } + slices.Sort(names) + if want := []string{"checkpoint.img", "fs/pages.img", "pages.img"}; !slices.Equal(names, want) { + t.Errorf("evicted %v, want %v", names, want) + } + case <-time.After(5 * time.Second): + t.Fatal("eviction did not run after the restore wait completed") + } + for _, name := range []string{"checkpoint.img", "pages.img", "fs/pages.img"} { + if _, err := os.Stat(filepath.Join(restoreDir, name)); err != nil { + t.Errorf("preserved %s was dropped: %v", name, err) + } + } + }) + + t.Run("skips eviction when the restore wait fails", func(t *testing.T) { + restoreDir := newRestoreDir(t) + evicted := captureEviction(t) + waited := make(chan struct{}) + + dropActorCheckpointCacheAsync(context.Background(), testActorUID, restoreDir, true, func(context.Context) error { + defer close(waited) + return errors.New("runsc wait boom") + }) + + <-waited + select { + case <-evicted: + t.Fatal("evicted after the restore wait failed") + case <-time.After(50 * time.Millisecond): + } + }) + + t.Run("waits no longer than the actor is hosted", func(t *testing.T) { + restoreDir := newRestoreDir(t) + evicted := captureEviction(t) + actorCtx, unhost := context.WithCancel(context.Background()) + waited := make(chan error, 1) + + dropActorCheckpointCacheAsync(actorCtx, testActorUID, restoreDir, true, func(ctx context.Context) error { + <-ctx.Done() + waited <- ctx.Err() + return ctx.Err() + }) + + unhost() + select { + case err := <-waited: + if !errors.Is(err, context.Canceled) { + t.Errorf("restore wait ended with %v, want context.Canceled", err) + } + case <-time.After(5 * time.Second): + t.Fatal("restore wait did not end when the actor was unhosted") + } + select { + case <-evicted: + t.Fatal("evicted after the actor was unhosted") + case <-time.After(50 * time.Millisecond): + } + }) +} diff --git a/cmd/ateworker-gvisor/cleanup_test.go b/cmd/ateworker-gvisor/cleanup_test.go index abbb4667f1..655f765872 100644 --- a/cmd/ateworker-gvisor/cleanup_test.go +++ b/cmd/ateworker-gvisor/cleanup_test.go @@ -23,7 +23,7 @@ import ( "slices" "testing" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/proto/ateworkerpb" ) // fakeRunsc stands in for *runsc. containers is the set runsc has a record of; diff --git a/cmd/ateworker-gvisor/durable.go b/cmd/ateworker-gvisor/durable.go index 9ae6ce0338..0dcb8b4e93 100644 --- a/cmd/ateworker-gvisor/durable.go +++ b/cmd/ateworker-gvisor/durable.go @@ -23,8 +23,8 @@ import ( "os" "path/filepath" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/resources" ) const fsCheckpointManifest = "fscheckpoint.pb" @@ -70,3 +70,25 @@ func fsRestoreArgs(checkpointDir string) ([]string, error) { } return []string{"--fs-restore-image-path", fsDir}, nil } + +// fsRestoreContainers returns the container names that own the master mount of +// at least one durable-dir volume (the first container to mount each volume), +// in order. Only those containers register an entry in runsc's fsRestore +// waitMap for `runsc wait -fsrestore `. +func fsRestoreContainers(containers []*ateworkerpb.Container) []string { + seen := make(map[string]bool) + var out []string + for _, c := range containers { + ownsMount := false + for _, m := range c.GetDurableDirVolumeMounts() { + if !seen[m.GetVolumeName()] { + seen[m.GetVolumeName()] = true + ownsMount = true + } + } + if ownsMount { + out = append(out, c.GetName()) + } + } + return out +} diff --git a/cmd/ateworker-gvisor/durable_test.go b/cmd/ateworker-gvisor/durable_test.go index 1806bc7245..ee92c52184 100644 --- a/cmd/ateworker-gvisor/durable_test.go +++ b/cmd/ateworker-gvisor/durable_test.go @@ -22,7 +22,7 @@ import ( "slices" "testing" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/proto/ateworkerpb" ) func TestDurableFSCheckpointPaths(t *testing.T) { @@ -148,3 +148,36 @@ func TestFSRestoreArgs(t *testing.T) { } }) } + +func TestFSRestoreContainers(t *testing.T) { + containers := []*ateworkerpb.Container{ + { + Name: "init", + }, + { + Name: "app", + DurableDirVolumeMounts: []*ateworkerpb.DurableDirVolumeMount{ + {VolumeName: "data", MountPath: "/var/data"}, + }, + }, + { + Name: "sidecar-shared-only", + DurableDirVolumeMounts: []*ateworkerpb.DurableDirVolumeMount{ + {VolumeName: "data", MountPath: "/mnt/data"}, + }, + }, + { + Name: "sidecar-with-own-vol", + DurableDirVolumeMounts: []*ateworkerpb.DurableDirVolumeMount{ + {VolumeName: "data", MountPath: "/mnt/data"}, + {VolumeName: "logs", MountPath: "/var/log"}, + }, + }, + } + + got := fsRestoreContainers(containers) + want := []string{"app", "sidecar-with-own-vol"} + if !slices.Equal(got, want) { + t.Errorf("fsRestoreContainers() = %v, want %v", got, want) + } +} diff --git a/cmd/ateworker-gvisor/fidelity_test.go b/cmd/ateworker-gvisor/fidelity_test.go index e058e45966..3a68ec4a26 100644 --- a/cmd/ateworker-gvisor/fidelity_test.go +++ b/cmd/ateworker-gvisor/fidelity_test.go @@ -23,8 +23,8 @@ import ( "google.golang.org/grpc/codes" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/proto/ateworkerpb" ) // TestValidateFidelity pins what this runtime serves. gVisor keeps rootfs diff --git a/cmd/ateworker-gvisor/flags.go b/cmd/ateworker-gvisor/flags.go index 58073f577e..3c673e9e89 100644 --- a/cmd/ateworker-gvisor/flags.go +++ b/cmd/ateworker-gvisor/flags.go @@ -26,9 +26,10 @@ import ( "log/slog" "os" "path/filepath" + "reflect" "sync" - "github.com/fsnotify/fsnotify" + "go.ate.dev/substrate/internal/ateworker/register" ) // commandFlags holds per-command flags for a single command. @@ -126,24 +127,12 @@ func newFlagStore(ctx context.Context, path string) (*flagStore, error) { return nil, err } - watcher, err := fsnotify.NewWatcher() - if err != nil { - return nil, fmt.Errorf("while creating fsnotify watcher for %q: %w", cleanPath, err) - } - - // Watch the parent directory so both in-place writes and atomic file - // replacements (renames) are detected. - dir := filepath.Dir(cleanPath) - if err := watcher.Add(dir); err != nil { - _ = watcher.Close() - return nil, fmt.Errorf("while watching directory %q for %q: %w", dir, cleanPath, err) - } - + changes := register.WatchFilePath(ctx, cleanPath) store := &flagStore{ path: cleanPath, cfg: cfg, } - go store.watchLoop(ctx, watcher) + go store.watchLoop(ctx, changes) return store, nil } @@ -155,41 +144,20 @@ func (s *flagStore) get() *flagConfig { return cfg } -func (s *flagStore) watchLoop(ctx context.Context, watcher *fsnotify.Watcher) { - defer func() { - if err := watcher.Close(); err != nil { - slog.Warn("Failed to close flags config watcher", slog.String("path", s.path), slog.Any("err", err)) - } - }() +func (s *flagStore) watchLoop(ctx context.Context, changes <-chan struct{}) { for { select { case <-ctx.Done(): return - case event, ok := <-watcher.Events: + case _, ok := <-changes: if !ok { return } - if s.isConfigEvent(event) { - s.reload() - } - case err, ok := <-watcher.Errors: - if !ok { - return - } - slog.Error("Error watching flags configuration file", - slog.String("path", s.path), - slog.Any("err", err)) + s.reload() } } } -func (s *flagStore) isConfigEvent(event fsnotify.Event) bool { - if filepath.Clean(event.Name) != s.path { - return false - } - return event.Has(fsnotify.Write) || event.Has(fsnotify.Create) -} - func (s *flagStore) reload() { defer func() { if s.onReload != nil { @@ -206,7 +174,12 @@ func (s *flagStore) reload() { } s.mu.Lock() - s.cfg = cfg + unchanged := reflect.DeepEqual(s.cfg, cfg) + if !unchanged { + s.cfg = cfg + } s.mu.Unlock() - slog.Info("Reloaded flags configuration", slog.String("path", s.path)) + if !unchanged { + slog.Info("Reloaded flags configuration", slog.String("path", s.path)) + } } diff --git a/cmd/ateworker-gvisor/flags_test.go b/cmd/ateworker-gvisor/flags_test.go index 0d291d90eb..a794ba2c57 100644 --- a/cmd/ateworker-gvisor/flags_test.go +++ b/cmd/ateworker-gvisor/flags_test.go @@ -214,4 +214,17 @@ func TestFlagStoreWatchAndReload(t *testing.T) { if !reflect.DeepEqual(gotAfterBad, wantUpdated) { t.Fatalf("after invalid reload buildArgs(create) = %v, want previous valid config %v", gotAfterBad, wantUpdated) } + + // Reloading when the file content is unchanged (e.g. on a polling tick) + // must be a no-op and keep the existing *flagConfig pointer. + if err := os.WriteFile(cfgPath, []byte(updatedJSON), 0o600); err != nil { + t.Fatalf("failed to rewrite config: %v", err) + } + waitForReload(t, reloadCh) + beforePtr := store.get() + store.reload() + waitForReload(t, reloadCh) + if afterPtr := store.get(); afterPtr != beforePtr { + t.Errorf("reload() with unchanged content replaced *flagConfig (%p -> %p), want no-op", beforePtr, afterPtr) + } } diff --git a/cmd/ateworker-gvisor/hosted.go b/cmd/ateworker-gvisor/hosted.go index e236f5a1c9..0761de4ed8 100644 --- a/cmd/ateworker-gvisor/hosted.go +++ b/cmd/ateworker-gvisor/hosted.go @@ -24,12 +24,12 @@ import ( "sync/atomic" "time" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateworker/actornet" - "github.com/agent-substrate/substrate/internal/atunnel" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/workerstats" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateworker/actornet" + "go.ate.dev/substrate/internal/atunnel/ingress" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/workerstats" ) // hostedActor holds one actor's attribution, network, and runtime state. @@ -45,6 +45,9 @@ type hostedActor struct { usage *workerstats.Activation // deadReported keeps the usage sweep to one warning per activation. deadReported atomic.Bool + // ctx is canceled when the actor is replaced or unhosted. + ctx context.Context + cancel context.CancelFunc } // admitActor reserves capacity before network setup. An actor that is already @@ -55,12 +58,14 @@ func (s *AteWorkerService) admitActor(attribution resources.ActorAttribution) (* var stale *actornet.SandboxSession if old, ok := s.actors[attribution.UID]; ok { stale = old.network + old.cancel() } else if len(s.actors)+s.draining >= s.maxActors { return nil, nil, apierror.ResourceExhausted("worker is full: %d actors", s.maxActors) } // The sandbox cgroup is new with every activation, so its counters start // at zero. - hosted := &hostedActor{attribution: attribution, usage: workerstats.NewActivation(time.Now(), false)} + ctx, cancel := context.WithCancel(context.Background()) + hosted := &hostedActor{attribution: attribution, usage: workerstats.NewActivation(time.Now(), false), ctx: ctx, cancel: cancel} s.actors[attribution.UID] = hosted return hosted, stale, nil } @@ -76,11 +81,16 @@ func (s *AteWorkerService) hostActor(ctx context.Context, attribution resources. if err != nil { return nil, err } + // evict drops the reservation when network setup fails. + evict := func() { + hosted.cancel() + s.actorsMu.Lock() + delete(s.actors, uid) + s.actorsMu.Unlock() + } if stale != nil { if err := stale.Close(ctx); err != nil { - s.actorsMu.Lock() - delete(s.actors, uid) - s.actorsMu.Unlock() + evict() return nil, fmt.Errorf("while clearing stale state for actor %s: %w", uid, err) } } @@ -92,9 +102,7 @@ func (s *AteWorkerService) hostActor(ctx context.Context, attribution resources. DNSPolicy: s.tunnel.ActorEgressPolicy(attribution), }, s.tunnel.Egress, s.tunnel.DNSRelay) if err != nil { - s.actorsMu.Lock() - delete(s.actors, uid) - s.actorsMu.Unlock() + evict() return nil, fmt.Errorf("while setting up the sandbox network: %w", err) } @@ -102,9 +110,7 @@ func (s *AteWorkerService) hostActor(ctx context.Context, attribution resources. resolvConf, err := actorResolvConf(actorDirs) if err != nil { _ = session.Close(ctx) - s.actorsMu.Lock() - delete(s.actors, uid) - s.actorsMu.Unlock() + evict() return nil, err } @@ -128,6 +134,7 @@ func (s *AteWorkerService) unhostActor(ctx context.Context, actorUID string) err if !ok { return nil } + hosted.cancel() defer func() { s.actorsMu.Lock() s.draining-- @@ -186,7 +193,7 @@ func (s *AteWorkerService) setSession(actorUID string, session *workloadSession) } // sandboxDialer looks up the actor on each dial. -func (s *AteWorkerService) sandboxDialer(actorUID string) atunnel.DialFunc { +func (s *AteWorkerService) sandboxDialer(actorUID string) ingress.DialFunc { return func(ctx context.Context, network, address string) (net.Conn, error) { s.actorsMu.RLock() var session *actornet.SandboxSession diff --git a/cmd/ateworker-gvisor/hosted_test.go b/cmd/ateworker-gvisor/hosted_test.go index b70a76861f..e3aebb6594 100644 --- a/cmd/ateworker-gvisor/hosted_test.go +++ b/cmd/ateworker-gvisor/hosted_test.go @@ -25,9 +25,9 @@ import ( "google.golang.org/grpc/codes" "time" - "github.com/agent-substrate/substrate/internal/actorlock" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/actorlock" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/resources" ) func TestAdmitActorEnforcesTheCeilingConcurrently(t *testing.T) { diff --git a/cmd/ateworker-gvisor/internal/cdiinject/inject.go b/cmd/ateworker-gvisor/internal/cdiinject/inject.go index a1306d02de..d21e741067 100644 --- a/cmd/ateworker-gvisor/internal/cdiinject/inject.go +++ b/cmd/ateworker-gvisor/internal/cdiinject/inject.go @@ -35,7 +35,7 @@ import ( specs "github.com/opencontainers/runtime-spec/specs-go" "golang.org/x/sys/unix" - "github.com/agent-substrate/substrate/internal/cdi" + "go.ate.dev/substrate/internal/cdi" ) // Options tune what a merge is allowed to do. Everything vendor-specific lives diff --git a/cmd/ateworker-gvisor/internal/cdiinject/inject_test.go b/cmd/ateworker-gvisor/internal/cdiinject/inject_test.go index d5ca98f302..15457365d1 100644 --- a/cmd/ateworker-gvisor/internal/cdiinject/inject_test.go +++ b/cmd/ateworker-gvisor/internal/cdiinject/inject_test.go @@ -25,7 +25,7 @@ import ( specs "github.com/opencontainers/runtime-spec/specs-go" "golang.org/x/sys/unix" - "github.com/agent-substrate/substrate/internal/cdi" + "go.ate.dev/substrate/internal/cdi" ) // statDev returns a device node's numbers, so the expectation matches whatever diff --git a/cmd/ateworker-gvisor/main.go b/cmd/ateworker-gvisor/main.go index be216acb61..31d0ab15c8 100644 --- a/cmd/ateworker-gvisor/main.go +++ b/cmd/ateworker-gvisor/main.go @@ -33,31 +33,31 @@ import ( "time" "cloud.google.com/go/compute/metadata" - "github.com/agent-substrate/substrate/cmd/ateworker-gvisor/internal/cgroupstats" - "github.com/agent-substrate/substrate/internal/actorlock" - "github.com/agent-substrate/substrate/internal/actorlog" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateinterceptors" - "github.com/agent-substrate/substrate/internal/ateworker/actornet" - "github.com/agent-substrate/substrate/internal/ateworker/cgroup" - "github.com/agent-substrate/substrate/internal/ateworker/phaselog" - "github.com/agent-substrate/substrate/internal/ateworker/pid1" - "github.com/agent-substrate/substrate/internal/ateworker/register" - "github.com/agent-substrate/substrate/internal/ateworker/tunnel" - "github.com/agent-substrate/substrate/internal/imagecache" - "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/ocispec" - "github.com/agent-substrate/substrate/internal/otlprelay" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/serverboot" - "github.com/agent-substrate/substrate/internal/sizing" - "github.com/agent-substrate/substrate/internal/version" - "github.com/agent-substrate/substrate/internal/wakeupprobe" - "github.com/agent-substrate/substrate/internal/workerstats" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/spf13/pflag" + "go.ate.dev/substrate/cmd/ateworker-gvisor/internal/cgroupstats" + "go.ate.dev/substrate/internal/actorlock" + "go.ate.dev/substrate/internal/actorlog" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateinterceptors" + "go.ate.dev/substrate/internal/ateworker/actornet" + "go.ate.dev/substrate/internal/ateworker/cgroup" + "go.ate.dev/substrate/internal/ateworker/phaselog" + "go.ate.dev/substrate/internal/ateworker/pid1" + "go.ate.dev/substrate/internal/ateworker/register" + "go.ate.dev/substrate/internal/ateworker/tunnel" + "go.ate.dev/substrate/internal/imagecache" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/ocispec" + "go.ate.dev/substrate/internal/otlprelay" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/serverboot" + "go.ate.dev/substrate/internal/sizing" + "go.ate.dev/substrate/internal/version" + "go.ate.dev/substrate/internal/wakeupprobe" + "go.ate.dev/substrate/internal/workerstats" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "google.golang.org/grpc/reflection" @@ -194,7 +194,7 @@ func do(ctx context.Context) error { if err := resources.ValidateWorkerPodUID(*podUID); err != nil { return fmt.Errorf("in resources.ValidateWorkerPodUID: %w", err) } - // atelet owns the directory and the sandbox versions file in it; only the + // atelet owns the directory and the sandbox runtimes file in it; only the // socket below is this process's. Both outlive a container restart. if err := os.MkdirAll(ateworkerDir, 0o700); err != nil { return fmt.Errorf("in os.MkdirAll(%q): %w", ateworkerDir, err) @@ -202,9 +202,13 @@ func do(ctx context.Context) error { // Prepare the pod cgroup so runsc can create per-actor-container leaves under // it with real accounting. - if _, err := cgroup.Delegate(ctx); err != nil { + cgroupDelegated, err := cgroup.Delegate(ctx) + if err != nil { return fmt.Errorf("while setting up cgroup delegation: %w", err) } + if cgroupDelegated { + removeStaleCheckpointCgroups(ctx, defaultCgroupRoot) + } // Clean up any old socket. sockPath := nodepath.WorkerSocketPath(*podUID) @@ -233,6 +237,7 @@ func do(ctx context.Context) error { return err } ateworkerService := NewService(tunnel, actorLogger, *maxActors, flags) + ateworkerService.cgroupDelegated = cgroupDelegated // The controller sets both from the downward API. pool := workerstats.Pool{Namespace: os.Getenv("POD_NAMESPACE"), Name: os.Getenv("WORKER_POOL_NAME")} if pool.Namespace == "" || pool.Name == "" { @@ -267,11 +272,12 @@ func do(ctx context.Context) error { svr.GracefulStop() }() - // Report what this worker can supply. Nothing else tells the control plane, - // which places no Actor here until it lands, so a worker that cannot report - // is one that will sit idle forever. Report retries every failure it can - // outlast, including the window before the Worker record exists; anything - // that reaches here is a misconfiguration no restart-in-place will fix. + // Report what this worker can supply and the sandbox runtimes it can run. + // Nothing else tells the control plane, which places no Actor here until it + // lands, so a worker that cannot report is one that will sit idle forever. + // Report retries every failure it can outlast, including the window before + // the Worker record exists; anything that reaches here is a misconfiguration + // no restart-in-place will fix. go func() { err := register.Report(ctx, register.ReportConfig{ SocketPath: nodepath.WorkerSupportSocket, @@ -280,9 +286,10 @@ func do(ctx context.Context) error { AteletSPIFFEID: tunnelConfig.BrokerIdentity, Actors: *maxActors, SandboxClass: string(atev1alpha1.SandboxClassGvisor), + RuntimesFile: nodepath.WorkerSandboxRuntimesPath(*podUID), }) if err != nil && ctx.Err() == nil { - serverboot.Fatal(ctx, "Failed to report worker capacity", err) + serverboot.Fatal(ctx, "Failed to register worker", err) } }() @@ -340,6 +347,10 @@ type AteWorkerService struct { // than a constant so tests can point GetWorkloadStats at a fixture tree. cgroupRoot string + // cgroupDelegated is set when the worker's cgroup is delegated (false on + // privileged workers that inherit the host cgroup namespace). + cgroupDelegated bool + // readSandboxCgroup overrides cgroupstats.Read when set. Only tests set it: // it is the seam that lets them interleave a lifecycle transition with the // stats handlers' lock-free read, the way containerStatsReader does for the @@ -781,52 +792,69 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke containers := req.GetSpec().GetContainers() durablePaths := durableFSCheckpointPaths(containers) + + // Cleanup the containers after checkpointing. This also unhosts the actor, + // before the snapshot listing below can fail. + // This is best-effort cleanup for actor containers that may have been left behind after checkpointing. + terminate := func() error { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + err := s.terminateWorkload(cleanupCtx, attribution.Ref, attribution.UID, req.GetRunscPath(), req.GetActorDirs(), containers) + if err != nil { + slog.WarnContext(cleanupCtx, "failed to terminate workload after checkpoint", + slog.String("actor", attribution.Ref.String()), + slog.String("actorUID", attribution.UID), + slog.Any("err", err)) + } + timing.teardown = lap(&tLast) + s.recordFinalIfEnded(cleanupCtx, hosted) + return err + } + + var checkpointFn func(context.Context) error switch fidelity { case ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: if len(durablePaths) == 0 { return nil, fmt.Errorf("no durable-dir volumes found for VOLUMES snapshot") } - err := rcmd.cmdFsCheckpoint(ctx, ocispec.PauseContainer, checkpointPath, durablePaths) - timing.durableDir = lap(&tLast) - if err != nil { - return nil, fmt.Errorf("while checkpointing durable-dir volumes: %w", err) + checkpointFn = func(ctx context.Context) error { + err := rcmd.cmdFsCheckpoint(ctx, ocispec.PauseContainer, checkpointPath, durablePaths) + timing.durableDir = lap(&tLast) + if err != nil { + return fmt.Errorf("while checkpointing durable-dir volumes: %w", err) + } + return nil } case ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: // Checkpoint pause container (root of the sandbox), splitting durable-dir // volumes into /fs when present. - sandbox, sbErr := openSandboxProcess(req.GetActorDirs()) - if sbErr != nil { - slog.WarnContext(ctx, "Cannot watch the sandbox exit after checkpoint", slog.Any("err", sbErr)) - } else { - defer sandbox.Close() - } - err := rcmd.cmdCheckpoint(ctx, ocispec.PauseContainer, checkpointPath, durablePaths) - if err != nil { - timing.checkpoint = lap(&tLast) - return nil, fmt.Errorf("while checkpointing pause: %w", err) - } - // the checkpoint stops the sandbox, runsc delete fails until it is reaped - if sandbox != nil { - if err := sandbox.waitGone(sandboxExitTimeout); err != nil { - slog.WarnContext(ctx, "Sandbox did not go away after checkpoint", slog.Any("err", err)) + checkpointFn = func(ctx context.Context) error { + sandbox, sbErr := openSandboxProcess(req.GetActorDirs()) + if sbErr != nil { + slog.WarnContext(ctx, "Cannot watch the sandbox exit after checkpoint", slog.Any("err", sbErr)) + } else { + defer sandbox.Close() } + err := rcmd.cmdCheckpoint(ctx, ocispec.PauseContainer, checkpointPath, durablePaths) + if err != nil { + timing.checkpoint = lap(&tLast) + return fmt.Errorf("while checkpointing pause: %w", err) + } + // the checkpoint stops the sandbox, runsc delete fails until it is reaped + if sandbox != nil { + if err := sandbox.waitGone(sandboxExitTimeout); err != nil { + slog.WarnContext(ctx, "Sandbox did not go away after checkpoint", slog.Any("err", err)) + } + } + timing.checkpoint = lap(&tLast) + return nil } - timing.checkpoint = lap(&tLast) default: return nil, fmt.Errorf("unsupported snapshot fidelity: %v", fidelity) } - - // Cleanup the containers after checkpointing. This also unhosts the actor, - // before the snapshot listing below can fail. - // This is best-effort cleanup for actor containers that may have been left behind after checkpointing. - if err := s.terminateWorkload(ctx, attribution.Ref, attribution.UID, req.GetRunscPath(), req.GetActorDirs(), req.GetSpec().GetContainers()); err != nil { - slog.WarnContext(ctx, "failed to terminate workload after checkpoint", - slog.String("actor", attribution.Ref.String()), - slog.String("actorUID", attribution.UID), - slog.Any("err", err)) + if err := checkpointIsolated(ctx, s.cgroupDelegated, s.cgroupRoot, req.GetActorUid(), req.GetActorDirs(), checkpointFn, terminate); err != nil { + return nil, err } - timing.teardown = lap(&tLast) - s.recordFinalIfEnded(ctx, hosted) // Report exactly the files runsc wrote so atelet ships precisely this set // (checkpoint.img plus any pages images and fs/ durable-dir files), rather @@ -1119,6 +1147,21 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb if err != nil { return nil, err } + dropActorCheckpointCacheAsync(hosted.ctx, req.GetActorUid(), checkpointDir, req.GetPreserveRestoreDir(), func(waitCtx context.Context) error { + if fidelity == ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { + if err := rcmd.cmdWaitRestore(waitCtx, ocispec.PauseContainer); err != nil { + return err + } + } + if len(restoreArgs) > 0 { + for _, ctr := range fsRestoreContainers(containers) { + if err := rcmd.cmdWaitFSRestore(waitCtx, ctr); err != nil { + return err + } + } + } + return nil + }) s.actorLogger.EmitLifecycleLog(ctx, "Actor restored", attribution) s.setSession(req.GetActorUid(), &workloadSession{rcmd: rcmd, containers: containerNames(containers)}) diff --git a/cmd/ateworker-gvisor/main_test.go b/cmd/ateworker-gvisor/main_test.go index 8a937d7c40..5061629ca2 100644 --- a/cmd/ateworker-gvisor/main_test.go +++ b/cmd/ateworker-gvisor/main_test.go @@ -30,13 +30,13 @@ import ( "google.golang.org/grpc/codes" - "github.com/agent-substrate/substrate/internal/actorlog" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/ocispec" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/actorlog" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/ocispec" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/resources" ) // Every RPC rejects a request without ActorDirs before touching any state. diff --git a/cmd/ateworker-gvisor/phaselog.go b/cmd/ateworker-gvisor/phaselog.go index c7190d23dd..7d390bc1c3 100644 --- a/cmd/ateworker-gvisor/phaselog.go +++ b/cmd/ateworker-gvisor/phaselog.go @@ -19,8 +19,8 @@ package main import ( "time" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/ateworker/phaselog" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/ateworker/phaselog" ) // The phase names, suffixed onto the phaselog duration keys. Kept out of diff --git a/cmd/ateworker-gvisor/phaselog_test.go b/cmd/ateworker-gvisor/phaselog_test.go index 55d750159c..e647c4c26a 100644 --- a/cmd/ateworker-gvisor/phaselog_test.go +++ b/cmd/ateworker-gvisor/phaselog_test.go @@ -24,9 +24,9 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/ateworker/phaselog" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/ateworker/phaselog" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/resources" ) // renderPhaseRecord logs attrs through a JSON handler, so the test checks the diff --git a/cmd/ateworker-gvisor/runsc.go b/cmd/ateworker-gvisor/runsc.go index 6b5da7101e..7d53c37082 100644 --- a/cmd/ateworker-gvisor/runsc.go +++ b/cmd/ateworker-gvisor/runsc.go @@ -28,10 +28,10 @@ import ( "slices" "strings" - "github.com/agent-substrate/substrate/internal/ocispec" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/sizing" + "go.ate.dev/substrate/internal/ocispec" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/sizing" ) type runsc struct { @@ -296,12 +296,10 @@ func (r *runsc) cmdKill(ctx context.Context, containerName, signal string) error return nil } -// cmdWait blocks until the given container's process exits. Used during -// graceful shutdown to confirm the actor has stopped before ateworker exits. -func (r *runsc) cmdWait(ctx context.Context, containerName string) error { - slog.InfoContext(ctx, "About to run runsc wait", slog.String("container", containerName)) - - cmd := exec.CommandContext(ctx, r.path, r.buildArgs("wait", containerName)...) +// runWait runs `runsc wait [extraArgs...] ` and blocks until the +// waited-on condition completes. +func (r *runsc) runWait(ctx context.Context, containerName string, extraArgs ...string) error { + cmd := exec.CommandContext(ctx, r.path, r.buildArgs("wait", containerName, extraArgs...)...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { @@ -309,3 +307,21 @@ func (r *runsc) cmdWait(ctx context.Context, containerName string) error { } return nil } + +// cmdWait blocks until the given container's process exits. Used during +// graceful shutdown to confirm the actor has stopped before ateworker exits. +func (r *runsc) cmdWait(ctx context.Context, containerName string) error { + slog.InfoContext(ctx, "About to run runsc wait", slog.String("container", containerName)) + return r.runWait(ctx, containerName) +} + +// cmdWaitRestore blocks until background page loading from `runsc restore -background` completes. +func (r *runsc) cmdWaitRestore(ctx context.Context, containerName string) error { + return r.runWait(ctx, containerName, "-restore") +} + +// cmdWaitFSRestore blocks until background filesystem page loading from +// `--fs-restore-image-path` completes for the given container. +func (r *runsc) cmdWaitFSRestore(ctx context.Context, containerName string) error { + return r.runWait(ctx, containerName, "-fsrestore") +} diff --git a/cmd/ateworker-gvisor/runsc_test.go b/cmd/ateworker-gvisor/runsc_test.go index 94deffdbd7..553c7de701 100644 --- a/cmd/ateworker-gvisor/runsc_test.go +++ b/cmd/ateworker-gvisor/runsc_test.go @@ -28,10 +28,10 @@ import ( "github.com/opencontainers/runtime-spec/specs-go" "google.golang.org/grpc/codes" - "github.com/agent-substrate/substrate/internal/actorlock" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ocispec" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/actorlock" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ocispec" + "go.ate.dev/substrate/internal/proto/ateworkerpb" ) var testActorDirs = &ateworkerpb.ActorDirs{ @@ -155,6 +155,34 @@ func TestBuildArgs(t *testing.T) { "my-container", }, }, + { + name: "wait -restore", + command: "wait", + containerName: "my-container", + extraArgs: []string{"-restore"}, + want: []string{ + "-global1", + "-global2", + "-root", "/node/actors/test-actor-123/runsc-state", + "wait", + "-restore", + "my-container", + }, + }, + { + name: "wait -fsrestore", + command: "wait", + containerName: "my-container", + extraArgs: []string{"-fsrestore"}, + want: []string{ + "-global1", + "-global2", + "-root", "/node/actors/test-actor-123/runsc-state", + "wait", + "-fsrestore", + "my-container", + }, + }, { name: "list", command: "list", diff --git a/cmd/ateworker-gvisor/sandboxexit.go b/cmd/ateworker-gvisor/sandboxexit.go index 1694a06b87..109e0cf221 100644 --- a/cmd/ateworker-gvisor/sandboxexit.go +++ b/cmd/ateworker-gvisor/sandboxexit.go @@ -25,8 +25,8 @@ import ( "golang.org/x/sys/unix" - "github.com/agent-substrate/substrate/internal/ocispec" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/ocispec" + "go.ate.dev/substrate/internal/proto/ateworkerpb" ) // sandboxExitTimeout bounds the wait for a checkpointed sandbox to go away. diff --git a/cmd/ateworker-gvisor/sandboxexit_test.go b/cmd/ateworker-gvisor/sandboxexit_test.go index 2ae50e2aa6..c7b477d8f6 100644 --- a/cmd/ateworker-gvisor/sandboxexit_test.go +++ b/cmd/ateworker-gvisor/sandboxexit_test.go @@ -24,8 +24,8 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/ocispec" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/ocispec" + "go.ate.dev/substrate/internal/proto/ateworkerpb" ) // startFakeSandbox starts a process that runs until killed and records it as diff --git a/cmd/ateworker-gvisor/sandboxnet.go b/cmd/ateworker-gvisor/sandboxnet.go index 5da536dddc..d1b368de54 100644 --- a/cmd/ateworker-gvisor/sandboxnet.go +++ b/cmd/ateworker-gvisor/sandboxnet.go @@ -25,9 +25,8 @@ import ( "os" "path/filepath" - "github.com/agent-substrate/substrate/internal/ateworker/actornet" - "github.com/agent-substrate/substrate/internal/ateworker/actornet/dns" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/ateworker/actornet" + "go.ate.dev/substrate/internal/proto/ateworkerpb" ) // actorResolvConf writes the resolver bind source outside the actor's rootfs. @@ -40,7 +39,7 @@ func actorResolvConf(actorDirs *ateworkerpb.ActorDirs) (string, error) { if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { return "", fmt.Errorf("creating the actor directory: %w", err) } - if err := os.WriteFile(path, dns.SandboxResolvConf(actornet.ActorVethGateway, pod), 0o644); err != nil { + if err := os.WriteFile(path, actornet.SandboxResolvConf(actornet.ActorVethGateway, pod), 0o644); err != nil { return "", fmt.Errorf("writing the actor resolv.conf: %w", err) } return path, nil diff --git a/cmd/ateworker-gvisor/stats.go b/cmd/ateworker-gvisor/stats.go index 54ad824491..5a2a71b7ab 100644 --- a/cmd/ateworker-gvisor/stats.go +++ b/cmd/ateworker-gvisor/stats.go @@ -24,13 +24,13 @@ import ( "path/filepath" "time" - "github.com/agent-substrate/substrate/cmd/ateworker-gvisor/internal/cgroupstats" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/ocispec" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/resources" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/cmd/ateworker-gvisor/internal/cgroupstats" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/ocispec" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/resources" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" ) // defaultCgroupRoot is the worker pod's own cgroup scope. The worker runs in a diff --git a/cmd/ateworker-gvisor/stats_test.go b/cmd/ateworker-gvisor/stats_test.go index e03ca0c564..2b75483651 100644 --- a/cmd/ateworker-gvisor/stats_test.go +++ b/cmd/ateworker-gvisor/stats_test.go @@ -19,7 +19,7 @@ package main import ( "bytes" "context" - "github.com/agent-substrate/substrate/internal/actorlock" + "go.ate.dev/substrate/internal/actorlock" "log/slog" "os" "path/filepath" @@ -32,12 +32,12 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/protobuf/testing/protocmp" - "github.com/agent-substrate/substrate/cmd/ateworker-gvisor/internal/cgroupstats" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ocispec" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/workerstats" + "go.ate.dev/substrate/cmd/ateworker-gvisor/internal/cgroupstats" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ocispec" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/workerstats" ) // The lifecycle RPCs that host and unhost actors need netlink, runsc, and diff --git a/cmd/ateworker-gvisor/usage_test.go b/cmd/ateworker-gvisor/usage_test.go index e7f8d997f8..f9b059086e 100644 --- a/cmd/ateworker-gvisor/usage_test.go +++ b/cmd/ateworker-gvisor/usage_test.go @@ -24,12 +24,12 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/actorlock" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/ocispec" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/workerstats" - "github.com/agent-substrate/substrate/internal/workerstats/workerstatstest" + "go.ate.dev/substrate/internal/actorlock" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/ocispec" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/workerstats" + "go.ate.dev/substrate/internal/workerstats/workerstatstest" ) func withUsageRecorder(s *AteWorkerService) *workerstatstest.Recorder { diff --git a/cmd/ateworker-microvm/actorpath.go b/cmd/ateworker-microvm/actorpath.go index 4d99fe30bb..a5f9c5c4b3 100644 --- a/cmd/ateworker-microvm/actorpath.go +++ b/cmd/ateworker-microvm/actorpath.go @@ -22,7 +22,7 @@ package main import ( "path/filepath" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/proto/ateworkerpb" ) // ociBundlePath is the container's OCI bundle. diff --git a/cmd/ateworker-microvm/checkpoint.go b/cmd/ateworker-microvm/checkpoint.go index 394a9dee5f..b4e80fa60f 100644 --- a/cmd/ateworker-microvm/checkpoint.go +++ b/cmd/ateworker-microvm/checkpoint.go @@ -25,15 +25,15 @@ import ( "path/filepath" "time" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/resources" - - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/ch" - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/kata" - "github.com/agent-substrate/substrate/internal/ateworker/phaselog" - "github.com/agent-substrate/substrate/internal/imagecache" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/workerstats" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/resources" + + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/ch" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/kata" + "go.ate.dev/substrate/internal/ateworker/phaselog" + "go.ate.dev/substrate/internal/imagecache" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/workerstats" "golang.org/x/sync/errgroup" ) diff --git a/cmd/ateworker-microvm/csi.go b/cmd/ateworker-microvm/csi.go index 154d4928f9..6417ce0ad2 100644 --- a/cmd/ateworker-microvm/csi.go +++ b/cmd/ateworker-microvm/csi.go @@ -21,9 +21,9 @@ import ( "fmt" "os" - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/kata" - "github.com/agent-substrate/substrate/internal/ocispec" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/kata" + "go.ate.dev/substrate/internal/ocispec" + "go.ate.dev/substrate/internal/proto/ateworkerpb" ) // hasCsiVolumes reports whether any container mounts a CSI volume. diff --git a/cmd/ateworker-microvm/csi_test.go b/cmd/ateworker-microvm/csi_test.go index aaa2c9bb02..f03cff8d5d 100644 --- a/cmd/ateworker-microvm/csi_test.go +++ b/cmd/ateworker-microvm/csi_test.go @@ -19,7 +19,7 @@ package main import ( "testing" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/proto/ateworkerpb" ) func TestHasCsiVolumes(t *testing.T) { diff --git a/cmd/ateworker-microvm/durable.go b/cmd/ateworker-microvm/durable.go index 830ac75a77..915ccc669d 100644 --- a/cmd/ateworker-microvm/durable.go +++ b/cmd/ateworker-microvm/durable.go @@ -45,10 +45,10 @@ import ( "slices" "strings" - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/kata" - "github.com/agent-substrate/substrate/internal/ocispec" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/tarutil" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/kata" + "go.ate.dev/substrate/internal/ocispec" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/tarutil" ) // durableTarFile is the snapshot file holding the tar of one durable-dir diff --git a/cmd/ateworker-microvm/durable_test.go b/cmd/ateworker-microvm/durable_test.go index 64d01ca1c5..ca848f9267 100644 --- a/cmd/ateworker-microvm/durable_test.go +++ b/cmd/ateworker-microvm/durable_test.go @@ -22,8 +22,8 @@ import ( "slices" "testing" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/tarutil" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/tarutil" ) func TestHasDurableVolumes(t *testing.T) { diff --git a/cmd/ateworker-microvm/fidelity.go b/cmd/ateworker-microvm/fidelity.go index de9a7049f7..244ce99f79 100644 --- a/cmd/ateworker-microvm/fidelity.go +++ b/cmd/ateworker-microvm/fidelity.go @@ -16,7 +16,7 @@ package main -import "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" +import "go.ate.dev/substrate/internal/proto/ateworkerpb" // The fidelity ladder, lowest to highest, each level including the ones // below it: VOLUMES holds the durable volumes, ROOTFS adds each container's diff --git a/cmd/ateworker-microvm/fidelity_test.go b/cmd/ateworker-microvm/fidelity_test.go index 4634041d14..4c9eee1408 100644 --- a/cmd/ateworker-microvm/fidelity_test.go +++ b/cmd/ateworker-microvm/fidelity_test.go @@ -22,8 +22,8 @@ import ( "google.golang.org/grpc/codes" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/proto/ateworkerpb" ) // TestFidelityLadder pins which layers each level carries: the checkpoint's diff --git a/cmd/ateworker-microvm/hosted.go b/cmd/ateworker-microvm/hosted.go index 0c33ae705b..61978a513e 100644 --- a/cmd/ateworker-microvm/hosted.go +++ b/cmd/ateworker-microvm/hosted.go @@ -23,15 +23,15 @@ import ( "net" "time" - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/kata" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateworker/actornet" - "github.com/agent-substrate/substrate/internal/ateworker/cgroup" - "github.com/agent-substrate/substrate/internal/atunnel" - "github.com/agent-substrate/substrate/internal/netns" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/sizing" - "github.com/agent-substrate/substrate/internal/workerstats" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/kata" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateworker/actornet" + "go.ate.dev/substrate/internal/ateworker/cgroup" + "go.ate.dev/substrate/internal/atunnel/ingress" + "go.ate.dev/substrate/internal/netns" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/sizing" + "go.ate.dev/substrate/internal/workerstats" ) // hostedActor holds one actor's attribution, network, and runtime state. @@ -234,7 +234,7 @@ func (s *AteWorkerService) sandboxNetNS(actorUID string) netns.Handle { } // sandboxDialer connects through the actor's gateway namespace. -func (s *AteWorkerService) sandboxDialer(actorUID string) atunnel.DialFunc { +func (s *AteWorkerService) sandboxDialer(actorUID string) ingress.DialFunc { return func(ctx context.Context, network, address string) (net.Conn, error) { s.actorsMu.RLock() hosted := s.actors[actorUID] diff --git a/cmd/ateworker-microvm/hosted_test.go b/cmd/ateworker-microvm/hosted_test.go index 9c60d5cff1..1ce50ec51b 100644 --- a/cmd/ateworker-microvm/hosted_test.go +++ b/cmd/ateworker-microvm/hosted_test.go @@ -21,8 +21,8 @@ import ( "google.golang.org/grpc/codes" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/resources" ) // Re-admitting an actor that is already hosted keeps its slot, so it succeeds diff --git a/cmd/ateworker-microvm/internal/agentstats/agentstats.go b/cmd/ateworker-microvm/internal/agentstats/agentstats.go index ad9b83ed67..95b0f01c43 100644 --- a/cmd/ateworker-microvm/internal/agentstats/agentstats.go +++ b/cmd/ateworker-microvm/internal/agentstats/agentstats.go @@ -29,7 +29,7 @@ package agentstats import ( - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb" ) // Sample is a point-in-time reading for one container, or the sum of several. diff --git a/cmd/ateworker-microvm/internal/agentstats/agentstats_test.go b/cmd/ateworker-microvm/internal/agentstats/agentstats_test.go index 3bcc79bc48..69f61e39ad 100644 --- a/cmd/ateworker-microvm/internal/agentstats/agentstats_test.go +++ b/cmd/ateworker-microvm/internal/agentstats/agentstats_test.go @@ -19,7 +19,7 @@ import ( "github.com/google/go-cmp/cmp" - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb" ) // cgroupStats builds an agent reading: usage/max bytes, the memory.stat entries diff --git a/cmd/ateworker-microvm/internal/kata/agentclient.go b/cmd/ateworker-microvm/internal/kata/agentclient.go index ff39a0c991..4c00311fe0 100644 --- a/cmd/ateworker-microvm/internal/kata/agentclient.go +++ b/cmd/ateworker-microvm/internal/kata/agentclient.go @@ -25,8 +25,8 @@ import ( "strings" "time" - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb" "github.com/containerd/ttrpc" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb" "google.golang.org/protobuf/types/known/emptypb" ) diff --git a/cmd/ateworker-microvm/internal/kata/agentclient_linux_test.go b/cmd/ateworker-microvm/internal/kata/agentclient_linux_test.go index a27b40b397..d190b0175a 100644 --- a/cmd/ateworker-microvm/internal/kata/agentclient_linux_test.go +++ b/cmd/ateworker-microvm/internal/kata/agentclient_linux_test.go @@ -25,8 +25,8 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb" "github.com/containerd/ttrpc" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb" "google.golang.org/protobuf/types/known/emptypb" ) diff --git a/cmd/ateworker-microvm/internal/kata/cleanup_linux.go b/cmd/ateworker-microvm/internal/kata/cleanup_linux.go index 6f7fd6ae5c..0c2634275c 100644 --- a/cmd/ateworker-microvm/internal/kata/cleanup_linux.go +++ b/cmd/ateworker-microvm/internal/kata/cleanup_linux.go @@ -27,7 +27,7 @@ import ( "golang.org/x/sys/unix" - "github.com/agent-substrate/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/nodepath" ) // CleanupSandboxState removes leftover host-side state for a sandbox id (the diff --git a/cmd/ateworker-microvm/internal/kata/cleanup_linux_test.go b/cmd/ateworker-microvm/internal/kata/cleanup_linux_test.go index 46220baf84..8180307ce8 100644 --- a/cmd/ateworker-microvm/internal/kata/cleanup_linux_test.go +++ b/cmd/ateworker-microvm/internal/kata/cleanup_linux_test.go @@ -20,7 +20,7 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/nodepath" ) func TestIsSandboxProcess(t *testing.T) { diff --git a/cmd/ateworker-microvm/internal/kata/overlay_linux.go b/cmd/ateworker-microvm/internal/kata/overlay_linux.go index 9e06d03e4b..d989644ba8 100644 --- a/cmd/ateworker-microvm/internal/kata/overlay_linux.go +++ b/cmd/ateworker-microvm/internal/kata/overlay_linux.go @@ -38,9 +38,9 @@ import ( "syscall" "time" - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb" - "github.com/agent-substrate/substrate/internal/ocispec" specs "github.com/opencontainers/runtime-spec/specs-go" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb" + "go.ate.dev/substrate/internal/ocispec" "golang.org/x/sys/unix" ) diff --git a/cmd/ateworker-microvm/internal/kata/overlay_linux_test.go b/cmd/ateworker-microvm/internal/kata/overlay_linux_test.go index be491697ce..fb9ca8269f 100644 --- a/cmd/ateworker-microvm/internal/kata/overlay_linux_test.go +++ b/cmd/ateworker-microvm/internal/kata/overlay_linux_test.go @@ -25,7 +25,7 @@ import ( "syscall" "testing" - "github.com/agent-substrate/substrate/internal/roottest" + "go.ate.dev/substrate/internal/roottest" "golang.org/x/sys/unix" ) diff --git a/cmd/ateworker-microvm/internal/kata/specconv.go b/cmd/ateworker-microvm/internal/kata/specconv.go index 87f0f2194c..5f143bb8a6 100644 --- a/cmd/ateworker-microvm/internal/kata/specconv.go +++ b/cmd/ateworker-microvm/internal/kata/specconv.go @@ -17,8 +17,8 @@ package kata import ( - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb" specs "github.com/opencontainers/runtime-spec/specs-go" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb" ) // DefaultCPUPeriodUS is the CFS period assumed when a spec carries a quota but diff --git a/cmd/ateworker-microvm/internal/third_party/kata/PROVENANCE.md b/cmd/ateworker-microvm/internal/third_party/kata/PROVENANCE.md index e9d8cdbd06..b884f7f647 100644 --- a/cmd/ateworker-microvm/internal/third_party/kata/PROVENANCE.md +++ b/cmd/ateworker-microvm/internal/third_party/kata/PROVENANCE.md @@ -50,5 +50,5 @@ ateworker drives a small subset of `AgentService`: 1. Check out the matching kata-containers tag and copy `src/libs/protocols/protos/{agent,oci,types,csi}.proto` into `agentpb/`. 2. Set each `option go_package` to - `github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb;agentpb`. + `go.ate.dev/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb;agentpb`. 3. Regenerate with `protoc --go_out=...` (protoc-gen-go), matching the versions above. diff --git a/cmd/ateworker-microvm/internal/third_party/kata/agentpb/agent.pb.go b/cmd/ateworker-microvm/internal/third_party/kata/agentpb/agent.pb.go index d1d5e570b4..ff93089850 100644 --- a/cmd/ateworker-microvm/internal/third_party/kata/agentpb/agent.pb.go +++ b/cmd/ateworker-microvm/internal/third_party/kata/agentpb/agent.pb.go @@ -4694,7 +4694,7 @@ const file_agent_proto_rawDesc = "" + "\vAddSwapPath\x12\x18.grpc.AddSwapPathRequest\x1a\x16.google.protobuf.Empty\x12E\n" + "\x0eGetVolumeStats\x12\x18.grpc.VolumeStatsRequest\x1a\x19.grpc.VolumeStatsResponse\x12A\n" + "\fResizeVolume\x12\x19.grpc.ResizeVolumeRequest\x1a\x16.google.protobuf.Empty\x12;\n" + - "\tSetPolicy\x12\x16.grpc.SetPolicyRequest\x1a\x16.google.protobuf.EmptyBfZdgithub.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb;agentpbb\x06proto3" + "\tSetPolicy\x12\x16.grpc.SetPolicyRequest\x1a\x16.google.protobuf.EmptyBVZTgo.ate.dev/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb;agentpbb\x06proto3" var ( file_agent_proto_rawDescOnce sync.Once diff --git a/cmd/ateworker-microvm/internal/third_party/kata/agentpb/agent.proto b/cmd/ateworker-microvm/internal/third_party/kata/agentpb/agent.proto index eff52661e4..36bf01e91f 100644 --- a/cmd/ateworker-microvm/internal/third_party/kata/agentpb/agent.proto +++ b/cmd/ateworker-microvm/internal/third_party/kata/agentpb/agent.proto @@ -7,7 +7,7 @@ syntax = "proto3"; -option go_package = "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb;agentpb"; +option go_package = "go.ate.dev/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb;agentpb"; package grpc; diff --git a/cmd/ateworker-microvm/internal/third_party/kata/agentpb/csi.pb.go b/cmd/ateworker-microvm/internal/third_party/kata/agentpb/csi.pb.go index d68026183e..3ff51b5961 100644 --- a/cmd/ateworker-microvm/internal/third_party/kata/agentpb/csi.pb.go +++ b/cmd/ateworker-microvm/internal/third_party/kata/agentpb/csi.pb.go @@ -286,7 +286,7 @@ const file_csi_proto_rawDesc = "" + "\x06INODES\x10\x02\"G\n" + "\x0fVolumeCondition\x12\x1a\n" + "\babnormal\x18\x01 \x01(\bR\babnormal\x12\x18\n" + - "\amessage\x18\x02 \x01(\tR\amessageBfZdgithub.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb;agentpbb\x06proto3" + "\amessage\x18\x02 \x01(\tR\amessageBVZTgo.ate.dev/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb;agentpbb\x06proto3" var ( file_csi_proto_rawDescOnce sync.Once diff --git a/cmd/ateworker-microvm/internal/third_party/kata/agentpb/csi.proto b/cmd/ateworker-microvm/internal/third_party/kata/agentpb/csi.proto index 21d2f89c59..5978a713de 100644 --- a/cmd/ateworker-microvm/internal/third_party/kata/agentpb/csi.proto +++ b/cmd/ateworker-microvm/internal/third_party/kata/agentpb/csi.proto @@ -4,7 +4,7 @@ // syntax = "proto3"; -option go_package = "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb;agentpb"; +option go_package = "go.ate.dev/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb;agentpb"; package grpc; diff --git a/cmd/ateworker-microvm/internal/third_party/kata/agentpb/oci.pb.go b/cmd/ateworker-microvm/internal/third_party/kata/agentpb/oci.pb.go index 7a704e0855..eeb482c3a1 100644 --- a/cmd/ateworker-microvm/internal/third_party/kata/agentpb/oci.pb.go +++ b/cmd/ateworker-microvm/internal/third_party/kata/agentpb/oci.pb.go @@ -2618,7 +2618,7 @@ const file_oci_proto_rawDesc = "" + "\n" + "\bErrnoRet\"5\n" + "\rLinuxIntelRdt\x12$\n" + - "\rL3CacheSchema\x18\x01 \x01(\tR\rL3CacheSchemaBfZdgithub.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb;agentpbb\x06proto3" + "\rL3CacheSchema\x18\x01 \x01(\tR\rL3CacheSchemaBVZTgo.ate.dev/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb;agentpbb\x06proto3" var ( file_oci_proto_rawDescOnce sync.Once diff --git a/cmd/ateworker-microvm/internal/third_party/kata/agentpb/oci.proto b/cmd/ateworker-microvm/internal/third_party/kata/agentpb/oci.proto index f28cae63b0..84d70a7bde 100644 --- a/cmd/ateworker-microvm/internal/third_party/kata/agentpb/oci.proto +++ b/cmd/ateworker-microvm/internal/third_party/kata/agentpb/oci.proto @@ -6,7 +6,7 @@ // syntax = "proto3"; -option go_package = "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb;agentpb"; +option go_package = "go.ate.dev/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb;agentpb"; package grpc; diff --git a/cmd/ateworker-microvm/internal/third_party/kata/agentpb/types.pb.go b/cmd/ateworker-microvm/internal/third_party/kata/agentpb/types.pb.go index 1a600ed555..0eee61126a 100644 --- a/cmd/ateworker-microvm/internal/third_party/kata/agentpb/types.pb.go +++ b/cmd/ateworker-microvm/internal/third_party/kata/agentpb/types.pb.go @@ -508,7 +508,7 @@ const file_types_proto_rawDesc = "" + "\x13FSGroupChangePolicy\x12\n" + "\n" + "\x06Always\x10\x00\x12\x12\n" + - "\x0eOnRootMismatch\x10\x01BfZdgithub.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb;agentpbb\x06proto3" + "\x0eOnRootMismatch\x10\x01BVZTgo.ate.dev/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb;agentpbb\x06proto3" var ( file_types_proto_rawDescOnce sync.Once diff --git a/cmd/ateworker-microvm/internal/third_party/kata/agentpb/types.proto b/cmd/ateworker-microvm/internal/third_party/kata/agentpb/types.proto index 62c08fe7c2..baa8e5e773 100644 --- a/cmd/ateworker-microvm/internal/third_party/kata/agentpb/types.proto +++ b/cmd/ateworker-microvm/internal/third_party/kata/agentpb/types.proto @@ -7,7 +7,7 @@ syntax = "proto3"; -option go_package = "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb;agentpb"; +option go_package = "go.ate.dev/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb;agentpb"; package types; diff --git a/cmd/ateworker-microvm/main.go b/cmd/ateworker-microvm/main.go index b577fc05d2..6f1a34ed6c 100644 --- a/cmd/ateworker-microvm/main.go +++ b/cmd/ateworker-microvm/main.go @@ -37,25 +37,25 @@ import ( "time" "cloud.google.com/go/compute/metadata" - "github.com/agent-substrate/substrate/internal/actorlock" - "github.com/agent-substrate/substrate/internal/actorlog" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateinterceptors" - "github.com/agent-substrate/substrate/internal/ateworker/actornet" - "github.com/agent-substrate/substrate/internal/ateworker/cgroup" - "github.com/agent-substrate/substrate/internal/ateworker/pid1" - "github.com/agent-substrate/substrate/internal/ateworker/register" - "github.com/agent-substrate/substrate/internal/ateworker/tunnel" - "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/otlprelay" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/serverboot" - "github.com/agent-substrate/substrate/internal/version" - "github.com/agent-substrate/substrate/internal/workerstats" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/spf13/pflag" + "go.ate.dev/substrate/internal/actorlock" + "go.ate.dev/substrate/internal/actorlog" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateinterceptors" + "go.ate.dev/substrate/internal/ateworker/actornet" + "go.ate.dev/substrate/internal/ateworker/cgroup" + "go.ate.dev/substrate/internal/ateworker/pid1" + "go.ate.dev/substrate/internal/ateworker/register" + "go.ate.dev/substrate/internal/ateworker/tunnel" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/otlprelay" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/serverboot" + "go.ate.dev/substrate/internal/version" + "go.ate.dev/substrate/internal/workerstats" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "golang.org/x/sys/unix" "google.golang.org/grpc" @@ -182,7 +182,7 @@ func do(ctx context.Context) error { if err := resources.ValidateWorkerPodUID(*podUID); err != nil { return fmt.Errorf("in resources.ValidateWorkerPodUID: %w", err) } - // atelet owns the directory and the sandbox versions file in it; only the + // atelet owns the directory and the sandbox runtimes file in it; only the // socket below is this process's. Both outlive a container restart. if err := os.MkdirAll(ateworkerDir, 0o700); err != nil { return fmt.Errorf("in os.MkdirAll(%q): %w", ateworkerDir, err) @@ -272,11 +272,12 @@ func do(ctx context.Context) error { svr.GracefulStop() }() - // Report what this worker can supply. Nothing else tells the control plane, - // which places no Actor here until it lands, so a worker that cannot report - // is one that will sit idle forever. Report retries every failure it can - // outlast, including the window before the Worker record exists; anything - // that reaches here is a misconfiguration no restart-in-place will fix. + // Report what this worker can supply and the sandbox runtimes it can run. + // Nothing else tells the control plane, which places no Actor here until it + // lands, so a worker that cannot report is one that will sit idle forever. + // Report retries every failure it can outlast, including the window before + // the Worker record exists; anything that reaches here is a misconfiguration + // no restart-in-place will fix. go func() { err := register.Report(ctx, register.ReportConfig{ SocketPath: nodepath.WorkerSupportSocket, @@ -285,9 +286,10 @@ func do(ctx context.Context) error { AteletSPIFFEID: tunnelConfig.BrokerIdentity, Actors: *maxActors, SandboxClass: string(atev1alpha1.SandboxClassMicroVM), + RuntimesFile: nodepath.WorkerSandboxRuntimesPath(*podUID), }) if err != nil && ctx.Err() == nil { - serverboot.Fatal(ctx, "Failed to report worker capacity", err) + serverboot.Fatal(ctx, "Failed to register worker", err) } }() diff --git a/cmd/ateworker-microvm/net.go b/cmd/ateworker-microvm/net.go index ef4703492e..2e177a92fa 100644 --- a/cmd/ateworker-microvm/net.go +++ b/cmd/ateworker-microvm/net.go @@ -24,8 +24,8 @@ import ( "github.com/vishvananda/netlink" - "github.com/agent-substrate/substrate/internal/ateworker/actornet" - "github.com/agent-substrate/substrate/internal/netns" + "go.ate.dev/substrate/internal/ateworker/actornet" + "go.ate.dev/substrate/internal/netns" ) const ( diff --git a/cmd/ateworker-microvm/net_linux_test.go b/cmd/ateworker-microvm/net_linux_test.go index 0f6b220985..7831a7cd99 100644 --- a/cmd/ateworker-microvm/net_linux_test.go +++ b/cmd/ateworker-microvm/net_linux_test.go @@ -22,13 +22,14 @@ import ( "github.com/vishvananda/netlink" - "github.com/agent-substrate/substrate/internal/ateworker/actornet" - "github.com/agent-substrate/substrate/internal/ateworker/tunnel" - "github.com/agent-substrate/substrate/internal/atunnel" - "github.com/agent-substrate/substrate/internal/netns" - "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/roottest" + "go.ate.dev/substrate/internal/ateworker/actornet" + "go.ate.dev/substrate/internal/ateworker/tunnel" + "go.ate.dev/substrate/internal/atunnel/egress" + "go.ate.dev/substrate/internal/linuxsys" + "go.ate.dev/substrate/internal/netns" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/roottest" ) // Hosting an actor that is already hosted replaces its network rather than @@ -36,12 +37,12 @@ import ( func TestHostActorReplacesSameActor(t *testing.T) { roottest.Require(t, "creates network namespaces") ctx := context.Background() - egress, err := atunnel.NewEgress(atunnel.TCPOriginalDestination) + egressProxy, err := egress.New(linuxsys.TCPOriginalDestination) if err != nil { t.Fatal(err) } service := &AteWorkerService{ - tunnel: &tunnel.Tunnel{Egress: egress, EgressPort: 15001}, + tunnel: &tunnel.Tunnel{Egress: egressProxy, EgressPort: 15001}, actors: map[string]*hostedActor{}, maxActors: 1, } diff --git a/cmd/ateworker-microvm/phaselog.go b/cmd/ateworker-microvm/phaselog.go index f277b26cd5..e895371160 100644 --- a/cmd/ateworker-microvm/phaselog.go +++ b/cmd/ateworker-microvm/phaselog.go @@ -16,7 +16,7 @@ package main -import "github.com/agent-substrate/substrate/internal/ateattr" +import "go.ate.dev/substrate/internal/ateattr" // The phase names, suffixed onto the phaselog duration keys. Kept out of // ateattr on purpose: that package's SnapshotPhase* values are the diff --git a/cmd/ateworker-microvm/restore.go b/cmd/ateworker-microvm/restore.go index 02bf2971bf..efa86e976e 100644 --- a/cmd/ateworker-microvm/restore.go +++ b/cmd/ateworker-microvm/restore.go @@ -27,19 +27,19 @@ import ( "strings" "time" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateworker/phaselog" - "github.com/agent-substrate/substrate/internal/workerstats" - - "github.com/agent-substrate/substrate/internal/ateworker/actornet" - - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/ch" - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/kata" - "github.com/agent-substrate/substrate/internal/imagecache" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/sizing" - "github.com/agent-substrate/substrate/internal/wakeupprobe" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateworker/phaselog" + "go.ate.dev/substrate/internal/workerstats" + + "go.ate.dev/substrate/internal/ateworker/actornet" + + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/ch" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/kata" + "go.ate.dev/substrate/internal/imagecache" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/sizing" + "go.ate.dev/substrate/internal/wakeupprobe" ) // restoreMemMode picks how cloud-hypervisor should load guest RAM, from what the VMM @@ -405,10 +405,8 @@ func (s *AteWorkerService) restoreMemoryFidelity(ctx context.Context, p actorBoo // page cache before resuming the VM so guest RAM and snapshot page cache do // not overlap during Resume and wakeup probe gating. // - // Dropping ahead of Resume means a Resume, agent dial, or wakeup failure leaves - // restore-state empty; atelet re-downloads with O_TRUNC and resetActorDirs - // wipes the directory on retry, so this costs a re-download on retry rather - // than correctness. + // When the restore dir is not preserved, dropping ahead of Resume means a + // failure after this point costs a re-download of the snapshot on retry. maybeDropStagedMemoryImage(ctx, restoreDir, memMode, preserveRestoreDir) dropActorSnapshotCacheAsync(restoreDir, memMode == ch.MemRestoreEager) if err := client.Resume(ctx); err != nil { diff --git a/cmd/ateworker-microvm/restore_test.go b/cmd/ateworker-microvm/restore_test.go index a7118617e0..b95ddd2384 100644 --- a/cmd/ateworker-microvm/restore_test.go +++ b/cmd/ateworker-microvm/restore_test.go @@ -27,11 +27,11 @@ import ( "google.golang.org/grpc/codes" - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/ch" - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/kata" - "github.com/agent-substrate/substrate/internal/actorlock" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/ch" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/kata" + "go.ate.dev/substrate/internal/actorlock" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/proto/ateworkerpb" ) // writeSnapshotConfig writes a config.json holding the given fs devices (plus the diff --git a/cmd/ateworker-microvm/rootfsupper.go b/cmd/ateworker-microvm/rootfsupper.go index c8984ebc66..5097135c21 100644 --- a/cmd/ateworker-microvm/rootfsupper.go +++ b/cmd/ateworker-microvm/rootfsupper.go @@ -53,9 +53,9 @@ import ( "path/filepath" "strings" - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/kata" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/tarutil" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/kata" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/tarutil" ) // rootfsUpperTarFile is the snapshot file holding the tar of one container's diff --git a/cmd/ateworker-microvm/rootfsupper_test.go b/cmd/ateworker-microvm/rootfsupper_test.go index 0181e1b251..3869055108 100644 --- a/cmd/ateworker-microvm/rootfsupper_test.go +++ b/cmd/ateworker-microvm/rootfsupper_test.go @@ -22,9 +22,9 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/internal/tarutil" + "go.ate.dev/substrate/internal/tarutil" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/proto/ateworkerpb" ) // upperDirWith returns a rootfs upper directory laid out the way the host diff --git a/cmd/ateworker-microvm/run.go b/cmd/ateworker-microvm/run.go index ea076e6888..55748ff804 100644 --- a/cmd/ateworker-microvm/run.go +++ b/cmd/ateworker-microvm/run.go @@ -29,21 +29,21 @@ import ( "syscall" "time" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/workerstats" - - "github.com/agent-substrate/substrate/internal/ateworker/actornet" - - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/ch" - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/kata" - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb" - "github.com/agent-substrate/substrate/internal/imagecache" - "github.com/agent-substrate/substrate/internal/ocispec" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/sizing" - "github.com/agent-substrate/substrate/internal/wakeupprobe" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/workerstats" + + "go.ate.dev/substrate/internal/ateworker/actornet" + specs "github.com/opencontainers/runtime-spec/specs-go" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/ch" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/kata" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb" + "go.ate.dev/substrate/internal/imagecache" + "go.ate.dev/substrate/internal/ocispec" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/sizing" + "go.ate.dev/substrate/internal/wakeupprobe" "golang.org/x/sys/unix" ) diff --git a/cmd/ateworker-microvm/run_test.go b/cmd/ateworker-microvm/run_test.go index bc23538f3e..34699b0f96 100644 --- a/cmd/ateworker-microvm/run_test.go +++ b/cmd/ateworker-microvm/run_test.go @@ -27,7 +27,7 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/kata" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/kata" ) // A vsock socket that has gone missing means cloud-hypervisor stopped the VM diff --git a/cmd/ateworker-microvm/sandboxnet.go b/cmd/ateworker-microvm/sandboxnet.go index 3749205ce5..298220f3d7 100644 --- a/cmd/ateworker-microvm/sandboxnet.go +++ b/cmd/ateworker-microvm/sandboxnet.go @@ -20,8 +20,7 @@ import ( "fmt" "os" - "github.com/agent-substrate/substrate/internal/ateworker/actornet" - "github.com/agent-substrate/substrate/internal/ateworker/actornet/dns" + "go.ate.dev/substrate/internal/ateworker/actornet" ) // writeActorResolvConf points the guest resolver at its fixed gateway address. @@ -30,5 +29,5 @@ func writeActorResolvConf(rootfs string) error { if err != nil { return fmt.Errorf("reading the worker pod resolv.conf: %w", err) } - return dns.WriteRootfsResolvConf(rootfs, dns.SandboxResolvConf(actornet.ActorVethGateway, pod)) + return actornet.WriteRootfsResolvConf(rootfs, actornet.SandboxResolvConf(actornet.ActorVethGateway, pod)) } diff --git a/cmd/ateworker-microvm/shutdown.go b/cmd/ateworker-microvm/shutdown.go index 2a4e0c427c..e91d5198c7 100644 --- a/cmd/ateworker-microvm/shutdown.go +++ b/cmd/ateworker-microvm/shutdown.go @@ -31,7 +31,7 @@ import ( "syscall" "time" - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/kata" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/kata" ) // workloadGracePeriod is the whole budget for draining the worker on shutdown: diff --git a/cmd/ateworker-microvm/snapshotcache.go b/cmd/ateworker-microvm/snapshotcache.go index 6dbc711082..91ab2618ae 100644 --- a/cmd/ateworker-microvm/snapshotcache.go +++ b/cmd/ateworker-microvm/snapshotcache.go @@ -21,7 +21,7 @@ import ( "log/slog" "strings" - "github.com/agent-substrate/substrate/internal/snapshotcache" + "go.ate.dev/substrate/internal/snapshotcache" ) const memoryRangesFile = "memory-ranges" @@ -43,8 +43,9 @@ func dropStagedMemoryImage(ctx context.Context, restoreDir string) { if len(targets) == 0 { return } - snapshotcache.DropFiles(targets) - slog.InfoContext(ctx, "dropped the staged memory image (eager restore needs no merge base)") + if snapshotcache.DropFiles(targets) { + slog.InfoContext(ctx, "dropped the staged memory image (eager restore needs no merge base)") + } } // dropActorSnapshotCacheAsync pins the actor's snapshot files in restoreDir diff --git a/cmd/ateworker-microvm/snapshotcache_test.go b/cmd/ateworker-microvm/snapshotcache_test.go index f6ab7aaa8c..71f87c25e8 100644 --- a/cmd/ateworker-microvm/snapshotcache_test.go +++ b/cmd/ateworker-microvm/snapshotcache_test.go @@ -22,8 +22,8 @@ import ( "path/filepath" "testing" - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/ch" - "github.com/agent-substrate/substrate/internal/snapshotcache" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/ch" + "go.ate.dev/substrate/internal/snapshotcache" ) const ( diff --git a/cmd/ateworker-microvm/spec.go b/cmd/ateworker-microvm/spec.go index b0a2a88030..21eec6af87 100644 --- a/cmd/ateworker-microvm/spec.go +++ b/cmd/ateworker-microvm/spec.go @@ -22,8 +22,8 @@ import ( specs "github.com/opencontainers/runtime-spec/specs-go" - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/kata" - "github.com/agent-substrate/substrate/internal/apierror" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/kata" + "go.ate.dev/substrate/internal/apierror" ) // guestEnvelope is the ceiling an actor's container limits must fit inside. diff --git a/cmd/ateworker-microvm/spec_test.go b/cmd/ateworker-microvm/spec_test.go index 126dabe2b7..05ec03908d 100644 --- a/cmd/ateworker-microvm/spec_test.go +++ b/cmd/ateworker-microvm/spec_test.go @@ -26,9 +26,9 @@ import ( "google.golang.org/grpc/codes" "k8s.io/utils/ptr" - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/kata" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ocispec" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/kata" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ocispec" ) // Limits the guest can never satisfy must be rejected before the containers diff --git a/cmd/ateworker-microvm/stats.go b/cmd/ateworker-microvm/stats.go index d7e8c61d19..2bc2d5c2a4 100644 --- a/cmd/ateworker-microvm/stats.go +++ b/cmd/ateworker-microvm/stats.go @@ -25,13 +25,13 @@ import ( "sync" "time" - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/agentstats" - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/resources" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/agentstats" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/resources" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" ) // statsCallTimeout bounds one container's guest-agent call. The RPC is polled diff --git a/cmd/ateworker-microvm/stats_test.go b/cmd/ateworker-microvm/stats_test.go index 6340144dd0..85977fa069 100644 --- a/cmd/ateworker-microvm/stats_test.go +++ b/cmd/ateworker-microvm/stats_test.go @@ -30,12 +30,12 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/protobuf/testing/protocmp" - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb" - "github.com/agent-substrate/substrate/internal/actorlock" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/workerstats" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb" + "go.ate.dev/substrate/internal/actorlock" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/workerstats" ) // TestActorBootParamsAttribution pins the mapping from actorBootParams onto the diff --git a/cmd/ateworker-microvm/systeminfo.go b/cmd/ateworker-microvm/systeminfo.go index 32157e07b0..5e946d2f81 100644 --- a/cmd/ateworker-microvm/systeminfo.go +++ b/cmd/ateworker-microvm/systeminfo.go @@ -47,9 +47,9 @@ import ( "os" "path/filepath" - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/kata" - "github.com/agent-substrate/substrate/internal/ocispec" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/kata" + "go.ate.dev/substrate/internal/ocispec" + "go.ate.dev/substrate/internal/proto/ateworkerpb" "golang.org/x/sys/unix" ) diff --git a/cmd/ateworker-microvm/usage_test.go b/cmd/ateworker-microvm/usage_test.go index b78f71482c..2b8a4e2cf4 100644 --- a/cmd/ateworker-microvm/usage_test.go +++ b/cmd/ateworker-microvm/usage_test.go @@ -23,11 +23,11 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/workerstats" - "github.com/agent-substrate/substrate/internal/workerstats/workerstatstest" + "go.ate.dev/substrate/cmd/ateworker-microvm/internal/third_party/kata/agentpb" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/workerstats" + "go.ate.dev/substrate/internal/workerstats/workerstatstest" ) func withUsageRecorder(s *AteWorkerService) *workerstatstest.Recorder { diff --git a/cmd/benchmarking/boomer-worker/main.go b/cmd/benchmarking/boomer-worker/main.go index d4e1acbbf4..29b77a4a63 100644 --- a/cmd/benchmarking/boomer-worker/main.go +++ b/cmd/benchmarking/boomer-worker/main.go @@ -28,18 +28,18 @@ import ( "strings" "time" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/boomerutil" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/dynconfig" - bmetrics "github.com/agent-substrate/substrate/internal/benchmarking/boomer/metrics" - btrace "github.com/agent-substrate/substrate/internal/benchmarking/boomer/trace" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/userclass" "github.com/myzhan/boomer" + "go.ate.dev/substrate/internal/benchmarking/boomer/boomerutil" + "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" + bmetrics "go.ate.dev/substrate/internal/benchmarking/boomer/metrics" + btrace "go.ate.dev/substrate/internal/benchmarking/boomer/trace" + "go.ate.dev/substrate/internal/benchmarking/boomer/userclass" // Register user classes via init(): - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/agentsession" - _ "github.com/agent-substrate/substrate/internal/benchmarking/boomer/glutton" - _ "github.com/agent-substrate/substrate/internal/benchmarking/boomer/resumepaused" - _ "github.com/agent-substrate/substrate/internal/benchmarking/boomer/sweperf" + "go.ate.dev/substrate/internal/benchmarking/boomer/agentsession" + _ "go.ate.dev/substrate/internal/benchmarking/boomer/glutton" + _ "go.ate.dev/substrate/internal/benchmarking/boomer/resumepaused" + _ "go.ate.dev/substrate/internal/benchmarking/boomer/sweperf" ) func main() { diff --git a/cmd/benchmarking/glutton/main.go b/cmd/benchmarking/glutton/main.go index 8f5d8fd09c..60e3bebc2c 100644 --- a/cmd/benchmarking/glutton/main.go +++ b/cmd/benchmarking/glutton/main.go @@ -26,9 +26,9 @@ import ( "github.com/spf13/pflag" - "github.com/agent-substrate/substrate/internal/benchmarking/glutton" - "github.com/agent-substrate/substrate/internal/serverboot" - "github.com/agent-substrate/substrate/internal/version" + "go.ate.dev/substrate/internal/benchmarking/glutton" + "go.ate.dev/substrate/internal/serverboot" + "go.ate.dev/substrate/internal/version" ) var ( diff --git a/cmd/benchmarking/isolate/fake-atelet/herder.go b/cmd/benchmarking/isolate/fake-atelet/herder.go index ec25740f43..1b58808abe 100644 --- a/cmd/benchmarking/isolate/fake-atelet/herder.go +++ b/cmd/benchmarking/isolate/fake-atelet/herder.go @@ -26,10 +26,10 @@ import ( "strings" "time" - "github.com/agent-substrate/substrate/internal/objectstore" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/objectstore" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" ) @@ -95,7 +95,7 @@ func newActorCSR() ([]byte, error) { // egress gateway, as worker does before it starts the workload. Its time is // ate-api-server's, so it adds to the call's delay rather than running inside // it; a failure fails the call, as tunneled egress fails closed in worker. -func (h *herder) mintActorCertificate(ctx context.Context, atespace, name, uid string, gateway *ateletpb.EgressGateway) error { +func (h *herder) mintActorCertificate(ctx context.Context, workerPodUID, atespace, name, uid string, gateway *ateletpb.EgressGateway) error { if h.minter == nil || gateway == nil { return nil } @@ -103,6 +103,8 @@ func (h *herder) mintActorCertificate(ctx context.Context, atespace, name, uid s Actor: &ateapipb.ObjectRef{Atespace: atespace, Name: name}, ActorUid: uid, CertificateSigningRequest: h.csr, + // Workers are global-scoped and named by their pod UID. + Worker: &ateapipb.ObjectRef{Name: workerPodUID}, }); err != nil { return fmt.Errorf("while minting the certificate of actor %s/%s: %w", atespace, name, err) } @@ -150,7 +152,7 @@ func (h *herder) writePlaceholder(ctx context.Context, start time.Time, d time.D } func (h *herder) Run(ctx context.Context, req *ateletpb.RunRequest) (*ateletpb.RunResponse, error) { - if err := h.mintActorCertificate(ctx, req.GetAtespace(), req.GetActorName(), req.GetActorUid(), req.GetEgressGateway()); err != nil { + if err := h.mintActorCertificate(ctx, req.GetWorkerPodUid(), req.GetAtespace(), req.GetActorName(), req.GetActorUid(), req.GetEgressGateway()); err != nil { return nil, err } if err := wait(ctx, time.Now(), h.delays.run); err != nil { @@ -160,7 +162,7 @@ func (h *herder) Run(ctx context.Context, req *ateletpb.RunRequest) (*ateletpb.R } func (h *herder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (*ateletpb.RestoreResponse, error) { - if err := h.mintActorCertificate(ctx, req.GetAtespace(), req.GetActorName(), req.GetActorUid(), req.GetEgressGateway()); err != nil { + if err := h.mintActorCertificate(ctx, req.GetWorkerPodUid(), req.GetAtespace(), req.GetActorName(), req.GetActorUid(), req.GetEgressGateway()); err != nil { return nil, err } if err := wait(ctx, time.Now(), h.delays.restore); err != nil { diff --git a/cmd/benchmarking/isolate/fake-atelet/herder_test.go b/cmd/benchmarking/isolate/fake-atelet/herder_test.go index 0268c0f6c6..a1b0a52bdf 100644 --- a/cmd/benchmarking/isolate/fake-atelet/herder_test.go +++ b/cmd/benchmarking/isolate/fake-atelet/herder_test.go @@ -27,8 +27,8 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "google.golang.org/protobuf/proto" ) @@ -285,7 +285,10 @@ func (m *recordingMinter) mints() []*ateapipb.MintWorkerActorCertificateRequest return slices.Clone(m.reqs) } -const testActorUID = "0f9c1d2e-3b4a-4c5d-8e6f-7a8b9c0d1e2f" +const ( + testActorUID = "0f9c1d2e-3b4a-4c5d-8e6f-7a8b9c0d1e2f" + testWorkerPodUID = "5d4c3b2a-1f0e-4d9c-8b7a-6f5e4d3c2b1a" +) var testGateway = &ateletpb.EgressGateway{Address: "atenet-egress.ate-system.svc:443"} @@ -293,11 +296,11 @@ var testGateway = &ateletpb.EgressGateway{Address: "atenet-egress.ate-system.svc func activations(h *herder, gateway *ateletpb.EgressGateway) map[string]func(context.Context) error { return map[string]func(context.Context) error{ "Run": func(ctx context.Context) error { - _, err := h.Run(ctx, &ateletpb.RunRequest{Atespace: "team-a", ActorName: "sleep-1", ActorUid: testActorUID, EgressGateway: gateway}) + _, err := h.Run(ctx, &ateletpb.RunRequest{WorkerPodUid: testWorkerPodUID, Atespace: "team-a", ActorName: "sleep-1", ActorUid: testActorUID, EgressGateway: gateway}) return err }, "Restore": func(ctx context.Context) error { - _, err := h.Restore(ctx, &ateletpb.RestoreRequest{Atespace: "team-a", ActorName: "sleep-1", ActorUid: testActorUID, EgressGateway: gateway}) + _, err := h.Restore(ctx, &ateletpb.RestoreRequest{WorkerPodUid: testWorkerPodUID, Atespace: "team-a", ActorName: "sleep-1", ActorUid: testActorUID, EgressGateway: gateway}) return err }, } @@ -330,6 +333,10 @@ func TestHerderMintsActorCertificateOnActivation(t *testing.T) { if !proto.Equal(got[0].GetActor(), want) || got[0].GetActorUid() != testActorUID { t.Errorf("%s minted for %v/%q, want %v/%q", name, got[0].GetActor(), got[0].GetActorUid(), want, testActorUID) } + // The control plane mints only for the worker hosting the actor. + if got, want := got[0].GetWorker(), (&ateapipb.ObjectRef{Name: testWorkerPodUID}); !proto.Equal(got, want) { + t.Errorf("%s minted as worker %v, want %v", name, got, want) + } // ate-api-server parses the CSR and checks its signature. csr, err := x509.ParseCertificateRequest(got[0].GetCertificateSigningRequest()) if err != nil { diff --git a/cmd/benchmarking/isolate/fake-atelet/main.go b/cmd/benchmarking/isolate/fake-atelet/main.go index 3f3e7dc381..bcd956cbf7 100644 --- a/cmd/benchmarking/isolate/fake-atelet/main.go +++ b/cmd/benchmarking/isolate/fake-atelet/main.go @@ -37,19 +37,19 @@ import ( "syscall" "time" - "github.com/agent-substrate/substrate/cmd/benchmarking/isolate/internal/fakeworker" - "github.com/agent-substrate/substrate/internal/ateapiauth" - "github.com/agent-substrate/substrate/internal/atelet" - "github.com/agent-substrate/substrate/internal/credbundle" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/serverboot" - "github.com/agent-substrate/substrate/internal/version" - "github.com/agent-substrate/substrate/pkg/objectstorage" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/spf13/pflag" + "go.ate.dev/substrate/cmd/benchmarking/isolate/internal/fakeworker" + "go.ate.dev/substrate/internal/ateapiauth" + "go.ate.dev/substrate/internal/atelet" + "go.ate.dev/substrate/internal/credbundle" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/serverboot" + "go.ate.dev/substrate/internal/version" + "go.ate.dev/substrate/pkg/objectstorage" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "google.golang.org/grpc/credentials" ) diff --git a/cmd/benchmarking/isolate/fake-atelet/relay.go b/cmd/benchmarking/isolate/fake-atelet/relay.go index c25d80a678..eed6d28c00 100644 --- a/cmd/benchmarking/isolate/fake-atelet/relay.go +++ b/cmd/benchmarking/isolate/fake-atelet/relay.go @@ -19,8 +19,8 @@ import ( "crypto/tls" "log/slog" - "github.com/agent-substrate/substrate/cmd/benchmarking/isolate/internal/fakeworker" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/benchmarking/isolate/internal/fakeworker" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" ) diff --git a/cmd/benchmarking/isolate/fake-atelet/relay_test.go b/cmd/benchmarking/isolate/fake-atelet/relay_test.go index 88f70d7304..ba0d2c51ca 100644 --- a/cmd/benchmarking/isolate/fake-atelet/relay_test.go +++ b/cmd/benchmarking/isolate/fake-atelet/relay_test.go @@ -21,9 +21,9 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/cmd/benchmarking/isolate/internal/fakeworker/fakeworkertest" - "github.com/agent-substrate/substrate/internal/credbundle" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/benchmarking/isolate/internal/fakeworker/fakeworkertest" + "go.ate.dev/substrate/internal/credbundle" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" diff --git a/cmd/benchmarking/isolate/fake-workersync/controller.go b/cmd/benchmarking/isolate/fake-workersync/controller.go index 320ecfc7cf..22a7d161dd 100644 --- a/cmd/benchmarking/isolate/fake-workersync/controller.go +++ b/cmd/benchmarking/isolate/fake-workersync/controller.go @@ -27,12 +27,12 @@ import ( "sync" "time" - "github.com/agent-substrate/substrate/cmd/benchmarking/isolate/internal/fakeworker" - "github.com/agent-substrate/substrate/internal/hardware" - "github.com/agent-substrate/substrate/internal/resources" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/uuid" + "go.ate.dev/substrate/cmd/benchmarking/isolate/internal/fakeworker" + "go.ate.dev/substrate/internal/hardware" + "go.ate.dev/substrate/internal/resources" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/pkg/proto/ateapipb" "golang.org/x/sync/errgroup" "google.golang.org/grpc" "google.golang.org/grpc/codes" diff --git a/cmd/benchmarking/isolate/fake-workersync/controller_test.go b/cmd/benchmarking/isolate/fake-workersync/controller_test.go index 43c4a24024..543f5d49f2 100644 --- a/cmd/benchmarking/isolate/fake-workersync/controller_test.go +++ b/cmd/benchmarking/isolate/fake-workersync/controller_test.go @@ -25,9 +25,9 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/benchmarking/isolate/internal/fakeworker" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/benchmarking/isolate/internal/fakeworker" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" diff --git a/cmd/benchmarking/isolate/fake-workersync/main.go b/cmd/benchmarking/isolate/fake-workersync/main.go index d5596f7a07..9a3829cae7 100644 --- a/cmd/benchmarking/isolate/fake-workersync/main.go +++ b/cmd/benchmarking/isolate/fake-workersync/main.go @@ -44,16 +44,16 @@ import ( "syscall" "time" - "github.com/agent-substrate/substrate/cmd/benchmarking/isolate/internal/fakeworker" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/boomerutil" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/internal/serverboot" - "github.com/agent-substrate/substrate/internal/version" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/client/clientset/versioned" - ateinformers "github.com/agent-substrate/substrate/pkg/client/informers/externalversions" - atelisters "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" "github.com/spf13/pflag" + "go.ate.dev/substrate/cmd/benchmarking/isolate/internal/fakeworker" + "go.ate.dev/substrate/internal/benchmarking/boomer/boomerutil" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/serverboot" + "go.ate.dev/substrate/internal/version" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/pkg/client/clientset/versioned" + ateinformers "go.ate.dev/substrate/pkg/client/informers/externalversions" + atelisters "go.ate.dev/substrate/pkg/client/listers/api/v1alpha1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/cmd/benchmarking/isolate/fake-workersync/relayclient.go b/cmd/benchmarking/isolate/fake-workersync/relayclient.go index 3b27c64d27..7d4fe48cad 100644 --- a/cmd/benchmarking/isolate/fake-workersync/relayclient.go +++ b/cmd/benchmarking/isolate/fake-workersync/relayclient.go @@ -23,9 +23,9 @@ import ( "sync" "time" - "github.com/agent-substrate/substrate/cmd/benchmarking/isolate/internal/fakeworker" - "github.com/agent-substrate/substrate/internal/credbundle" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/benchmarking/isolate/internal/fakeworker" + "go.ate.dev/substrate/internal/credbundle" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "google.golang.org/grpc/credentials" ) diff --git a/cmd/benchmarking/isolate/fake-workersync/relayclient_test.go b/cmd/benchmarking/isolate/fake-workersync/relayclient_test.go index f0bdffc756..0cc83ce898 100644 --- a/cmd/benchmarking/isolate/fake-workersync/relayclient_test.go +++ b/cmd/benchmarking/isolate/fake-workersync/relayclient_test.go @@ -26,10 +26,10 @@ import ( "sync" "testing" - "github.com/agent-substrate/substrate/cmd/benchmarking/isolate/internal/fakeworker" - "github.com/agent-substrate/substrate/cmd/benchmarking/isolate/internal/fakeworker/fakeworkertest" - "github.com/agent-substrate/substrate/internal/credbundle" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/cmd/benchmarking/isolate/internal/fakeworker" + "go.ate.dev/substrate/cmd/benchmarking/isolate/internal/fakeworker/fakeworkertest" + "go.ate.dev/substrate/internal/credbundle" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" diff --git a/cmd/benchmarking/isolate/internal/fakeworker/fakeworkertest/fakeworkertest.go b/cmd/benchmarking/isolate/internal/fakeworker/fakeworkertest/fakeworkertest.go index 6b1365bdd5..7339ebfaa7 100644 --- a/cmd/benchmarking/isolate/internal/fakeworker/fakeworkertest/fakeworkertest.go +++ b/cmd/benchmarking/isolate/internal/fakeworker/fakeworkertest/fakeworkertest.go @@ -33,7 +33,7 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/localca" + "go.ate.dev/substrate/internal/localca" ) // CA is a throwaway signer whose files live in a test's temp dir. diff --git a/cmd/credential-provider/kubernetes-secrets/kubeprovider.go b/cmd/credential-provider/kubernetes-secrets/kubeprovider.go index 598560695c..054c0ca75e 100644 --- a/cmd/credential-provider/kubernetes-secrets/kubeprovider.go +++ b/cmd/credential-provider/kubernetes-secrets/kubeprovider.go @@ -33,8 +33,8 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/credproviderpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/credproviderpb" ) // ProviderName is the ate-secret:// URI host this backend serves. diff --git a/cmd/credential-provider/kubernetes-secrets/kubeprovider_test.go b/cmd/credential-provider/kubernetes-secrets/kubeprovider_test.go index 1f2758aa81..6199be449e 100644 --- a/cmd/credential-provider/kubernetes-secrets/kubeprovider_test.go +++ b/cmd/credential-provider/kubernetes-secrets/kubeprovider_test.go @@ -34,7 +34,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/agent-substrate/substrate/pkg/proto/credproviderpb" + "go.ate.dev/substrate/pkg/proto/credproviderpb" ) func TestParseURI(t *testing.T) { diff --git a/cmd/credential-provider/kubernetes-secrets/main.go b/cmd/credential-provider/kubernetes-secrets/main.go index 6ce789e698..2dcc57b858 100644 --- a/cmd/credential-provider/kubernetes-secrets/main.go +++ b/cmd/credential-provider/kubernetes-secrets/main.go @@ -41,12 +41,12 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "github.com/agent-substrate/substrate/internal/credbundle" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/serverboot" - "github.com/agent-substrate/substrate/internal/version" - "github.com/agent-substrate/substrate/pkg/proto/credproviderpb" + "go.ate.dev/substrate/internal/credbundle" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/serverboot" + "go.ate.dev/substrate/internal/version" + "go.ate.dev/substrate/pkg/proto/credproviderpb" ) const serviceName = "credprovider" diff --git a/cmd/credential-provider/kubernetes-secrets/main_test.go b/cmd/credential-provider/kubernetes-secrets/main_test.go index 944194a72e..a1cf7ed8d7 100644 --- a/cmd/credential-provider/kubernetes-secrets/main_test.go +++ b/cmd/credential-provider/kubernetes-secrets/main_test.go @@ -31,9 +31,9 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/internal/localca" - "github.com/agent-substrate/substrate/pkg/proto/credproviderpb" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/localca" + "go.ate.dev/substrate/pkg/proto/credproviderpb" "google.golang.org/grpc" "google.golang.org/grpc/credentials" corev1 "k8s.io/api/core/v1" diff --git a/cmd/credential-provider/kubernetes-secrets/nsauthz.go b/cmd/credential-provider/kubernetes-secrets/nsauthz.go index 46ebf5c840..21995ff7fc 100644 --- a/cmd/credential-provider/kubernetes-secrets/nsauthz.go +++ b/cmd/credential-provider/kubernetes-secrets/nsauthz.go @@ -19,7 +19,7 @@ import ( "os" "sort" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/resources" "k8s.io/apimachinery/pkg/util/validation" "sigs.k8s.io/yaml" diff --git a/cmd/credential-provider/kubernetes-secrets/statusz.go b/cmd/credential-provider/kubernetes-secrets/statusz.go index 45e2f52896..df586ff021 100644 --- a/cmd/credential-provider/kubernetes-secrets/statusz.go +++ b/cmd/credential-provider/kubernetes-secrets/statusz.go @@ -29,7 +29,7 @@ import ( "github.com/spf13/pflag" - "github.com/agent-substrate/substrate/internal/version" + "go.ate.dev/substrate/internal/version" ) // statusContext is the /statusz payload; the JSON form is the contract tests diff --git a/cmd/credential-provider/kubernetes-secrets/statusz_test.go b/cmd/credential-provider/kubernetes-secrets/statusz_test.go index 4155f4389d..184ec08e01 100644 --- a/cmd/credential-provider/kubernetes-secrets/statusz_test.go +++ b/cmd/credential-provider/kubernetes-secrets/statusz_test.go @@ -27,7 +27,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" - "github.com/agent-substrate/substrate/pkg/proto/credproviderpb" + "go.ate.dev/substrate/pkg/proto/credproviderpb" ) func TestGrants(t *testing.T) { diff --git a/cmd/kubectl-ate/README.md b/cmd/kubectl-ate/README.md index 5b058683a5..6c07170a49 100644 --- a/cmd/kubectl-ate/README.md +++ b/cmd/kubectl-ate/README.md @@ -322,7 +322,9 @@ kubectl ate admin make-ca-pool \ # Generate a new JWT authority pool and push it to a Kubernetes Secret. The # key is ES256 (--alg RS256, a 4096-bit RSA key, for relying parties that don't # support ES256) and its ID defaults to the base64url SHA-256 of the public -# key's PKIX encoding (--key-id to override). +# key's PKIX encoding (--key-id to override). The Secret holds the pool under +# "pool" and its public keys as a JWK set under "jwks", for verifiers that +# must not see the signing keys. Every command below keeps the two in step. kubectl ate admin make-jwt-pool \ --name actor-id-jwt-pool \ --secret-namespace ate-system diff --git a/cmd/kubectl-ate/internal/cmd/access_policy.go b/cmd/kubectl-ate/internal/cmd/access_policy.go new file mode 100644 index 0000000000..092fd8fd56 --- /dev/null +++ b/cmd/kubectl-ate/internal/cmd/access_policy.go @@ -0,0 +1,389 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + + "github.com/spf13/cobra" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/encoding/protojson" + + "go.ate.dev/substrate/cmd/kubectl-ate/internal/printer" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" +) + +var accessPolicyFlags struct { + atespace string + global bool + filename string + uid string + version int64 +} + +const accessPolicyManifestHelp = `The manifest is a YAML or JSON AccessPolicy, as printed by +"kubectl ate get access-policy -o yaml". Its metadata may be omitted. Members +are principals with a "user:" prefix, e.g. +"user:system:serviceaccount:team-a:alice". + + bindings: + - role: owner + members: + - user:system:serviceaccount:team-a:alice` + +var getAccessPolicyCmd = &cobra.Command{ + Use: "access-policy (-a | --global)", + Aliases: []string{"access-policies"}, + Short: "Get the access policy of an atespace or the global access policy", + Args: cobra.NoArgs, + RunE: runGetAccessPolicy, +} + +var createAccessPolicyCmd = &cobra.Command{ + Use: "access-policy (-a | --global) -f ", + Aliases: []string{"access-policies"}, + Short: "Create an access policy from a manifest", + Long: "Create the access policy of an atespace, or the global access policy.\n\n" + accessPolicyManifestHelp, + Args: cobra.NoArgs, + RunE: runCreateAccessPolicy, +} + +var updateAccessPolicyCmd = &cobra.Command{ + Use: "access-policy (-a | --global) -f ", + Aliases: []string{"access-policies"}, + Short: "Replace an access policy from a manifest", + Long: "Replace the access policy of an atespace, or the global access policy.\n\n" + accessPolicyManifestHelp + ` + +Its metadata.uid and metadata.version are required preconditions; the server +rejects the update when they no longer match those of the stored policy.`, + Args: cobra.NoArgs, + RunE: runUpdateAccessPolicy, +} + +var deleteAccessPolicyCmd = &cobra.Command{ + Use: "access-policy -a ", + Aliases: []string{"access-policies"}, + Short: "Delete the access policy of an atespace", + Long: `Delete the access policy of an atespace. The global access policy cannot be +deleted; update it with no bindings instead.`, + Args: cobra.NoArgs, + RunE: runDeleteAccessPolicy, +} + +// accessPolicyScope is the parent of an access policy: one atespace, or the +// global singleton when atespace is empty. +type accessPolicyScope struct { + atespace string +} + +func (s accessPolicyScope) global() bool { return s.atespace == "" } + +func (s accessPolicyScope) String() string { + if s.global() { + return "global" + } + return fmt.Sprintf("atespace %q", s.atespace) +} + +func (s accessPolicyScope) ref() *ateapipb.ObjectRef { + return &ateapipb.ObjectRef{Name: s.atespace} +} + +// scopeFromFlags requires exactly one of --atespace and --global. +func scopeFromFlags(atespace string, global bool) (accessPolicyScope, error) { + switch { + case atespace != "" && global: + return accessPolicyScope{}, errors.New("--atespace and --global are mutually exclusive") + case atespace == "" && !global: + return accessPolicyScope{}, errors.New("one of --atespace or --global is required") + } + return accessPolicyScope{atespace: atespace}, nil +} + +// accessPolicyFromManifest parses a single protojson-shaped YAML or JSON +// document into an AccessPolicy. Parsing is strict: unknown fields are an +// error. +func accessPolicyFromManifest(data []byte) (*ateapipb.AccessPolicy, error) { + jsonData, err := manifestToJSON(data) + if err != nil { + return nil, err + } + policy := &ateapipb.AccessPolicy{} + if err := protojson.Unmarshal(jsonData, policy); err != nil { + return nil, fmt.Errorf("invalid AccessPolicy: %w", err) + } + return policy, nil +} + +// loadAccessPolicyManifest reads a manifest from filename, or from in when +// filename is "-", parses it, and checks its metadata. An access policy's +// metadata.atespace is always empty, whatever its scope, and its name is +// always "default"; omitted metadata is filled in. +func loadAccessPolicyManifest(in io.Reader, filename string) (*ateapipb.AccessPolicy, error) { + data, err := readFileOrStdin(in, filename) + if err != nil { + return nil, err + } + policy, err := accessPolicyFromManifest(data) + if err != nil { + return nil, fmt.Errorf("failed to parse access policy manifest %q: %w", filename, err) + } + if policy.Metadata == nil { + policy.Metadata = &ateapipb.ResourceMetadata{} + } + if policy.Metadata.Name == "" { + policy.Metadata.Name = resources.SingletonName + } + if policy.Metadata.Name != resources.SingletonName { + return nil, fmt.Errorf("manifest metadata.name %q must be %q", policy.Metadata.Name, resources.SingletonName) + } + if policy.Metadata.Atespace != "" { + return nil, fmt.Errorf("manifest metadata.atespace must be empty, got %q; choose the atespace with --atespace", policy.Metadata.Atespace) + } + return policy, nil +} + +// atespaceGetter abstracts the atespace read that tells a missing atespace +// from a missing access policy. +type atespaceGetter interface { + GetAtespace(ctx context.Context, req *ateapipb.GetAtespaceRequest, opts ...grpc.CallOption) (*ateapipb.Atespace, error) +} + +// requireScope settles a NotFound from an access policy RPC. For an atespace +// the server answers NotFound for a missing atespace and for an atespace +// without a policy alike, so it reads the atespace. It returns nil when the +// scope exists, which means the policy is what is missing. +func requireScope(ctx context.Context, getter atespaceGetter, scope accessPolicyScope) error { + if scope.global() { + return nil + } + if _, err := getter.GetAtespace(ctx, &ateapipb.GetAtespaceRequest{Atespace: scope.ref()}); err != nil { + if status.Code(err) == codes.NotFound { + return fmt.Errorf("atespace %q not found", scope.atespace) + } + return fmt.Errorf("failed to get atespace %q: %w", scope.atespace, err) + } + return nil +} + +// accessPolicyClient is the subset of the Control API the access policy +// commands call. +type accessPolicyClient interface { + GetGlobalAccessPolicy(ctx context.Context, req *ateapipb.GetGlobalAccessPolicyRequest, opts ...grpc.CallOption) (*ateapipb.AccessPolicy, error) + CreateGlobalAccessPolicy(ctx context.Context, req *ateapipb.CreateGlobalAccessPolicyRequest, opts ...grpc.CallOption) (*ateapipb.AccessPolicy, error) + UpdateGlobalAccessPolicy(ctx context.Context, req *ateapipb.UpdateGlobalAccessPolicyRequest, opts ...grpc.CallOption) (*ateapipb.AccessPolicy, error) + GetAtespaceAccessPolicy(ctx context.Context, req *ateapipb.GetAtespaceAccessPolicyRequest, opts ...grpc.CallOption) (*ateapipb.AccessPolicy, error) + CreateAtespaceAccessPolicy(ctx context.Context, req *ateapipb.CreateAtespaceAccessPolicyRequest, opts ...grpc.CallOption) (*ateapipb.AccessPolicy, error) + UpdateAtespaceAccessPolicy(ctx context.Context, req *ateapipb.UpdateAtespaceAccessPolicyRequest, opts ...grpc.CallOption) (*ateapipb.AccessPolicy, error) + DeleteAtespaceAccessPolicy(ctx context.Context, req *ateapipb.DeleteAtespaceAccessPolicyRequest, opts ...grpc.CallOption) (*ateapipb.AccessPolicy, error) + atespaceGetter +} + +// accessPolicyRunner executes the access policy commands. +type accessPolicyRunner struct { + client accessPolicyClient + scope accessPolicyScope + outputFmt string + stdout io.Writer + stderr io.Writer +} + +func (r *accessPolicyRunner) get(ctx context.Context) error { + var policy *ateapipb.AccessPolicy + var err error + if r.scope.global() { + policy, err = r.client.GetGlobalAccessPolicy(ctx, &ateapipb.GetGlobalAccessPolicyRequest{Name: resources.SingletonName}) + } else { + policy, err = r.client.GetAtespaceAccessPolicy(ctx, &ateapipb.GetAtespaceAccessPolicyRequest{Atespace: r.scope.ref(), Name: resources.SingletonName}) + } + if status.Code(err) == codes.NotFound { + if err := requireScope(ctx, r.client, r.scope); err != nil { + return err + } + // No policy is a valid state, not a failure: it grants nothing. + fmt.Fprintf(r.stderr, "%s has no access policy\n", r.scope) + return nil + } + if err != nil { + return fmt.Errorf("failed to get %s access policy: %w", r.scope, err) + } + return printer.PrintAccessPolicyTo(r.stdout, r.scope.atespace, policy, r.outputFmt) +} + +func (r *accessPolicyRunner) create(ctx context.Context, policy *ateapipb.AccessPolicy) error { + var created *ateapipb.AccessPolicy + var err error + if r.scope.global() { + created, err = r.client.CreateGlobalAccessPolicy(ctx, &ateapipb.CreateGlobalAccessPolicyRequest{AccessPolicy: policy}) + } else { + created, err = r.client.CreateAtespaceAccessPolicy(ctx, &ateapipb.CreateAtespaceAccessPolicyRequest{Atespace: r.scope.ref(), AccessPolicy: policy}) + } + if status.Code(err) == codes.AlreadyExists { + return fmt.Errorf(`%s already has an access policy; change it with "kubectl ate update access-policy": %w`, r.scope, err) + } + if err != nil { + return fmt.Errorf("failed to create %s access policy: %w", r.scope, err) + } + return printer.PrintAccessPolicyTo(r.stdout, r.scope.atespace, created, r.outputFmt) +} + +func (r *accessPolicyRunner) update(ctx context.Context, policy *ateapipb.AccessPolicy) error { + var updated *ateapipb.AccessPolicy + var err error + if r.scope.global() { + updated, err = r.client.UpdateGlobalAccessPolicy(ctx, &ateapipb.UpdateGlobalAccessPolicyRequest{AccessPolicy: policy}) + } else { + updated, err = r.client.UpdateAtespaceAccessPolicy(ctx, &ateapipb.UpdateAtespaceAccessPolicyRequest{Atespace: r.scope.ref(), AccessPolicy: policy}) + } + if status.Code(err) == codes.NotFound { + if err := requireScope(ctx, r.client, r.scope); err != nil { + return err + } + return fmt.Errorf(`%s has no access policy to update; create it with "kubectl ate create access-policy"`, r.scope) + } + if status.Code(err) == codes.Aborted { + return fmt.Errorf(`manifest's uid and version preconditions do not match those of the stored %s access policy (changed since it was read, or read from another scope); re-run "get access-policy -o yaml" and reapply the edit: %w`, r.scope, err) + } + if err != nil { + return fmt.Errorf("failed to update %s access policy: %w", r.scope, err) + } + return printer.PrintAccessPolicyTo(r.stdout, r.scope.atespace, updated, r.outputFmt) +} + +func (r *accessPolicyRunner) delete(ctx context.Context, options *ateapipb.DeleteOptions) error { + _, err := r.client.DeleteAtespaceAccessPolicy(ctx, &ateapipb.DeleteAtespaceAccessPolicyRequest{Atespace: r.scope.ref(), Name: resources.SingletonName, Options: options}) + if status.Code(err) == codes.NotFound { + if err := requireScope(ctx, r.client, r.scope); err != nil { + return err + } + return fmt.Errorf("%s has no access policy", r.scope) + } + if err != nil { + return fmt.Errorf("failed to delete %s access policy: %w", r.scope, err) + } + _, err = fmt.Fprintf(r.stdout, "access policy for %s deleted\n", r.scope) + return err +} + +// newAccessPolicyRunner connects to ate-api-server. The returned close func +// releases the connection. +func newAccessPolicyRunner(cmd *cobra.Command, scope accessPolicyScope) (*accessPolicyRunner, func(), error) { + apiClient, err := newAPIClient(cmd) + if err != nil { + return nil, nil, fmt.Errorf("failed to connect to ate-api-server: %w", err) + } + return &accessPolicyRunner{ + client: apiClient, + scope: scope, + outputFmt: outputFmt, + stdout: cmd.OutOrStdout(), + stderr: cmd.ErrOrStderr(), + }, apiClient.Close, nil +} + +func runGetAccessPolicy(cmd *cobra.Command, _ []string) error { + scope, err := scopeFromFlags(accessPolicyFlags.atespace, accessPolicyFlags.global) + if err != nil { + return err + } + runner, closeFn, err := newAccessPolicyRunner(cmd, scope) + if err != nil { + return err + } + defer closeFn() + return runner.get(cmd.Context()) +} + +func runCreateAccessPolicy(cmd *cobra.Command, _ []string) error { + scope, err := scopeFromFlags(accessPolicyFlags.atespace, accessPolicyFlags.global) + if err != nil { + return err + } + policy, err := loadAccessPolicyManifest(cmd.InOrStdin(), accessPolicyFlags.filename) + if err != nil { + return err + } + runner, closeFn, err := newAccessPolicyRunner(cmd, scope) + if err != nil { + return err + } + defer closeFn() + return runner.create(cmd.Context(), policy) +} + +func runUpdateAccessPolicy(cmd *cobra.Command, _ []string) error { + scope, err := scopeFromFlags(accessPolicyFlags.atespace, accessPolicyFlags.global) + if err != nil { + return err + } + policy, err := loadAccessPolicyManifest(cmd.InOrStdin(), accessPolicyFlags.filename) + if err != nil { + return err + } + // Checked before dialing, so an unusable manifest costs no connection. + if policy.GetMetadata().GetUid() == "" || policy.GetMetadata().GetVersion() == 0 { + return fmt.Errorf(`manifest for the %s access policy lacks the metadata.uid and metadata.version preconditions that "get access-policy -o yaml" prints`, scope) + } + runner, closeFn, err := newAccessPolicyRunner(cmd, scope) + if err != nil { + return err + } + defer closeFn() + return runner.update(cmd.Context(), policy) +} + +func runDeleteAccessPolicy(cmd *cobra.Command, _ []string) error { + options, err := deleteOptionsFromFlags(deleteGuardFlags{ + uidSet: cmd.Flags().Changed("uid"), + uid: accessPolicyFlags.uid, + versionSet: cmd.Flags().Changed("version"), + version: accessPolicyFlags.version, + }) + if err != nil { + return err + } + runner, closeFn, err := newAccessPolicyRunner(cmd, accessPolicyScope{atespace: accessPolicyFlags.atespace}) + if err != nil { + return err + } + defer closeFn() + return runner.delete(cmd.Context(), options) +} + +func init() { + for _, c := range []*cobra.Command{getAccessPolicyCmd, createAccessPolicyCmd, updateAccessPolicyCmd} { + c.Flags().StringVarP(&accessPolicyFlags.atespace, "atespace", "a", "", "Atespace whose access policy to use") + c.Flags().BoolVar(&accessPolicyFlags.global, "global", false, "Use the global access policy instead of an atespace's") + } + for _, c := range []*cobra.Command{createAccessPolicyCmd, updateAccessPolicyCmd} { + c.Flags().StringVarP(&accessPolicyFlags.filename, "filename", "f", "", "Manifest file holding one AccessPolicy; use - for stdin (required)") + _ = c.MarkFlagRequired("filename") + } + getCmd.AddCommand(getAccessPolicyCmd) + createCmd.AddCommand(createAccessPolicyCmd) + updateCmd.AddCommand(updateAccessPolicyCmd) + + deleteAccessPolicyCmd.Flags().StringVarP(&accessPolicyFlags.atespace, "atespace", "a", "", "Atespace whose access policy to delete (required)") + _ = deleteAccessPolicyCmd.MarkFlagRequired("atespace") + deleteAccessPolicyCmd.Flags().StringVar(&accessPolicyFlags.uid, "uid", "", "Delete only if the policy's metadata.uid matches; from get -o yaml") + deleteAccessPolicyCmd.Flags().Int64Var(&accessPolicyFlags.version, "version", 0, "Delete only if the policy's metadata.version matches; from get -o yaml") + deleteCmd.AddCommand(deleteAccessPolicyCmd) +} diff --git a/cmd/kubectl-ate/internal/cmd/access_policy_test.go b/cmd/kubectl-ate/internal/cmd/access_policy_test.go new file mode 100644 index 0000000000..9288cac71c --- /dev/null +++ b/cmd/kubectl-ate/internal/cmd/access_policy_test.go @@ -0,0 +1,331 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "bytes" + "context" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/testing/protocmp" + + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" +) + +func TestScopeFromFlags(t *testing.T) { + t.Parallel() + tests := []struct { + name string + atespace string + global bool + want accessPolicyScope + wantErr string + }{ + {name: "atespace", atespace: "team-a", want: accessPolicyScope{atespace: "team-a"}}, + {name: "global", global: true, want: accessPolicyScope{}}, + {name: "neither", wantErr: "one of --atespace or --global is required"}, + {name: "both", atespace: "team-a", global: true, wantErr: "mutually exclusive"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got, err := scopeFromFlags(tc.atespace, tc.global) + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("scopeFromFlags() error = %v, want %q", err, tc.wantErr) + } + return + } + if err != nil { + t.Fatalf("scopeFromFlags() error = %v", err) + } + if got != tc.want { + t.Errorf("scopeFromFlags() = %+v, want %+v", got, tc.want) + } + }) + } +} + +func TestLoadAccessPolicyManifest(t *testing.T) { + t.Parallel() + ownerAlice := []*ateapipb.Binding{{Role: "owner", Members: []string{"user:alice"}}} + tests := []struct { + name string + manifest string + want *ateapipb.AccessPolicy + wantErr string + }{ + { + name: "metadata omitted is filled in", + manifest: "bindings:\n- role: owner\n members: [\"user:alice\"]\n", + want: &ateapipb.AccessPolicy{Metadata: &ateapipb.ResourceMetadata{Name: "default"}, Bindings: ownerAlice}, + }, + { + name: "preconditions preserved", + manifest: `metadata: {name: default, uid: u-1, version: "3"} +bindings: [{role: owner, members: ["user:alice"]}] +`, + want: &ateapipb.AccessPolicy{Metadata: &ateapipb.ResourceMetadata{Name: "default", Uid: "u-1", Version: 3}, Bindings: ownerAlice}, + }, + { + name: "no bindings", + manifest: "metadata: {name: default}\n", + want: &ateapipb.AccessPolicy{Metadata: &ateapipb.ResourceMetadata{Name: "default"}}, + }, + {name: "wrong name", manifest: "metadata: {name: other}\n", wantErr: `must be "default"`}, + {name: "atespace set", manifest: "metadata: {atespace: team-a}\n", wantErr: "metadata.atespace must be empty"}, + {name: "unknown field", manifest: "bindingz: []\n", wantErr: "invalid AccessPolicy"}, + {name: "empty", manifest: "", wantErr: "manifest is empty"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got, err := loadAccessPolicyManifest(strings.NewReader(tc.manifest), "-") + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("loadAccessPolicyManifest() error = %v, want %q", err, tc.wantErr) + } + return + } + if err != nil { + t.Fatalf("loadAccessPolicyManifest() error = %v", err) + } + if diff := cmp.Diff(tc.want, got, protocmp.Transform()); diff != "" { + t.Errorf("loadAccessPolicyManifest() (-want +got):\n%s", diff) + } + }) + } +} + +// fakeAccessPolicyClient records the last RPC and answers with policy or err. +type fakeAccessPolicyClient struct { + policy *ateapipb.AccessPolicy + err error + atespaceErr error + + method string + req any +} + +func (f *fakeAccessPolicyClient) answer(method string, req any) (*ateapipb.AccessPolicy, error) { + f.method, f.req = method, req + return f.policy, f.err +} + +func (f *fakeAccessPolicyClient) GetGlobalAccessPolicy(_ context.Context, req *ateapipb.GetGlobalAccessPolicyRequest, _ ...grpc.CallOption) (*ateapipb.AccessPolicy, error) { + return f.answer("GetGlobalAccessPolicy", req) +} + +func (f *fakeAccessPolicyClient) CreateGlobalAccessPolicy(_ context.Context, req *ateapipb.CreateGlobalAccessPolicyRequest, _ ...grpc.CallOption) (*ateapipb.AccessPolicy, error) { + return f.answer("CreateGlobalAccessPolicy", req) +} + +func (f *fakeAccessPolicyClient) UpdateGlobalAccessPolicy(_ context.Context, req *ateapipb.UpdateGlobalAccessPolicyRequest, _ ...grpc.CallOption) (*ateapipb.AccessPolicy, error) { + return f.answer("UpdateGlobalAccessPolicy", req) +} + +func (f *fakeAccessPolicyClient) GetAtespaceAccessPolicy(_ context.Context, req *ateapipb.GetAtespaceAccessPolicyRequest, _ ...grpc.CallOption) (*ateapipb.AccessPolicy, error) { + return f.answer("GetAtespaceAccessPolicy", req) +} + +func (f *fakeAccessPolicyClient) CreateAtespaceAccessPolicy(_ context.Context, req *ateapipb.CreateAtespaceAccessPolicyRequest, _ ...grpc.CallOption) (*ateapipb.AccessPolicy, error) { + return f.answer("CreateAtespaceAccessPolicy", req) +} + +func (f *fakeAccessPolicyClient) UpdateAtespaceAccessPolicy(_ context.Context, req *ateapipb.UpdateAtespaceAccessPolicyRequest, _ ...grpc.CallOption) (*ateapipb.AccessPolicy, error) { + return f.answer("UpdateAtespaceAccessPolicy", req) +} + +func (f *fakeAccessPolicyClient) DeleteAtespaceAccessPolicy(_ context.Context, req *ateapipb.DeleteAtespaceAccessPolicyRequest, _ ...grpc.CallOption) (*ateapipb.AccessPolicy, error) { + return f.answer("DeleteAtespaceAccessPolicy", req) +} + +func (f *fakeAccessPolicyClient) GetAtespace(_ context.Context, req *ateapipb.GetAtespaceRequest, _ ...grpc.CallOption) (*ateapipb.Atespace, error) { + if f.atespaceErr != nil { + return nil, f.atespaceErr + } + return &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: req.GetAtespace().GetName()}}, nil +} + +func TestAccessPolicyRunner(t *testing.T) { + t.Parallel() + stored := &ateapipb.AccessPolicy{ + Metadata: &ateapipb.ResourceMetadata{Name: "default", Uid: "u-1", Version: 2}, + Bindings: []*ateapipb.Binding{{Role: "owner", Members: []string{"user:alice"}}}, + } + edit := &ateapipb.AccessPolicy{Metadata: &ateapipb.ResourceMetadata{Name: "default", Uid: "u-1", Version: 2}} + teamA := accessPolicyScope{atespace: "team-a"} + global := accessPolicyScope{} + notFound := status.Error(codes.NotFound, "not found") + + tests := []struct { + name string + scope accessPolicyScope + client *fakeAccessPolicyClient + run func(context.Context, *accessPolicyRunner) error + wantMethod string + wantReq any + wantStdout string + wantStderr string + wantErr string + wantNoCalls bool + }{ + { + name: "get atespace", + scope: teamA, + client: &fakeAccessPolicyClient{policy: stored}, + run: func(ctx context.Context, r *accessPolicyRunner) error { return r.get(ctx) }, + wantMethod: "GetAtespaceAccessPolicy", + wantReq: &ateapipb.GetAtespaceAccessPolicyRequest{Atespace: &ateapipb.ObjectRef{Name: "team-a"}, Name: resources.SingletonName}, + wantStdout: "team-a owner user:alice", + }, + { + name: "get global", + scope: global, + client: &fakeAccessPolicyClient{policy: stored}, + run: func(ctx context.Context, r *accessPolicyRunner) error { return r.get(ctx) }, + wantMethod: "GetGlobalAccessPolicy", + wantReq: &ateapipb.GetGlobalAccessPolicyRequest{Name: resources.SingletonName}, + wantStdout: " owner user:alice", + }, + { + name: "get atespace without a policy", + scope: teamA, + client: &fakeAccessPolicyClient{err: notFound}, + run: func(ctx context.Context, r *accessPolicyRunner) error { return r.get(ctx) }, + wantMethod: "GetAtespaceAccessPolicy", + wantStderr: `atespace "team-a" has no access policy`, + }, + { + name: "get missing atespace", + scope: teamA, + client: &fakeAccessPolicyClient{err: notFound, atespaceErr: notFound}, + run: func(ctx context.Context, r *accessPolicyRunner) error { return r.get(ctx) }, + wantMethod: "GetAtespaceAccessPolicy", + wantErr: `atespace "team-a" not found`, + }, + { + name: "get permission denied", + scope: teamA, + client: &fakeAccessPolicyClient{err: status.Error(codes.PermissionDenied, "denied")}, + run: func(ctx context.Context, r *accessPolicyRunner) error { return r.get(ctx) }, + wantMethod: "GetAtespaceAccessPolicy", + wantErr: "PermissionDenied", + }, + { + name: "create atespace", + scope: teamA, + client: &fakeAccessPolicyClient{policy: stored}, + run: func(ctx context.Context, r *accessPolicyRunner) error { return r.create(ctx, edit) }, + wantMethod: "CreateAtespaceAccessPolicy", + wantReq: &ateapipb.CreateAtespaceAccessPolicyRequest{Atespace: &ateapipb.ObjectRef{Name: "team-a"}, AccessPolicy: edit}, + wantStdout: "team-a owner user:alice", + }, + { + name: "create global", + scope: global, + client: &fakeAccessPolicyClient{policy: stored}, + run: func(ctx context.Context, r *accessPolicyRunner) error { return r.create(ctx, edit) }, + wantMethod: "CreateGlobalAccessPolicy", + wantReq: &ateapipb.CreateGlobalAccessPolicyRequest{AccessPolicy: edit}, + }, + { + name: "create existing", + scope: teamA, + client: &fakeAccessPolicyClient{err: status.Error(codes.AlreadyExists, "exists")}, + run: func(ctx context.Context, r *accessPolicyRunner) error { return r.create(ctx, edit) }, + wantMethod: "CreateAtespaceAccessPolicy", + wantErr: "kubectl ate update access-policy", + }, + { + name: "update global", + scope: global, + client: &fakeAccessPolicyClient{policy: stored}, + run: func(ctx context.Context, r *accessPolicyRunner) error { return r.update(ctx, edit) }, + wantMethod: "UpdateGlobalAccessPolicy", + wantReq: &ateapipb.UpdateGlobalAccessPolicyRequest{AccessPolicy: edit}, + }, + { + name: "update stale", + scope: teamA, + client: &fakeAccessPolicyClient{err: status.Error(codes.Aborted, "version mismatch")}, + run: func(ctx context.Context, r *accessPolicyRunner) error { return r.update(ctx, edit) }, + wantMethod: "UpdateAtespaceAccessPolicy", + wantErr: "re-run", + }, + { + name: "update without a policy", + scope: teamA, + client: &fakeAccessPolicyClient{err: notFound}, + run: func(ctx context.Context, r *accessPolicyRunner) error { return r.update(ctx, edit) }, + wantMethod: "UpdateAtespaceAccessPolicy", + wantErr: "kubectl ate create access-policy", + }, + { + name: "delete", + scope: teamA, + client: &fakeAccessPolicyClient{policy: stored}, + run: func(ctx context.Context, r *accessPolicyRunner) error { + return r.delete(ctx, &ateapipb.DeleteOptions{Version: 2}) + }, + wantMethod: "DeleteAtespaceAccessPolicy", + wantReq: &ateapipb.DeleteAtespaceAccessPolicyRequest{Atespace: &ateapipb.ObjectRef{Name: "team-a"}, Name: resources.SingletonName, Options: &ateapipb.DeleteOptions{Version: 2}}, + wantStdout: `access policy for atespace "team-a" deleted`, + }, + { + name: "delete without a policy", + scope: teamA, + client: &fakeAccessPolicyClient{err: notFound}, + run: func(ctx context.Context, r *accessPolicyRunner) error { return r.delete(ctx, nil) }, + wantMethod: "DeleteAtespaceAccessPolicy", + wantErr: `atespace "team-a" has no access policy`, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + var stdout, stderr bytes.Buffer + r := &accessPolicyRunner{client: tc.client, scope: tc.scope, outputFmt: "table", stdout: &stdout, stderr: &stderr} + err := tc.run(t.Context(), r) + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("error = %v, want it to contain %q", err, tc.wantErr) + } + } else if err != nil { + t.Fatalf("error = %v", err) + } + if tc.client.method != tc.wantMethod { + t.Errorf("called %q, want %q", tc.client.method, tc.wantMethod) + } + if tc.wantReq != nil { + if diff := cmp.Diff(tc.wantReq, tc.client.req, protocmp.Transform()); diff != "" { + t.Errorf("request (-want +got):\n%s", diff) + } + } + if !strings.Contains(stdout.String(), tc.wantStdout) { + t.Errorf("stdout = %q, want it to contain %q", stdout.String(), tc.wantStdout) + } + if !strings.Contains(stderr.String(), tc.wantStderr) { + t.Errorf("stderr = %q, want it to contain %q", stderr.String(), tc.wantStderr) + } + }) + } +} diff --git a/cmd/kubectl-ate/internal/cmd/actor.go b/cmd/kubectl-ate/internal/cmd/actor.go index 20b59a3ad1..be7b57c711 100644 --- a/cmd/kubectl-ate/internal/cmd/actor.go +++ b/cmd/kubectl-ate/internal/cmd/actor.go @@ -27,12 +27,12 @@ import ( "sync" "time" - "github.com/agent-substrate/substrate/cmd/kubectl-ate/internal/printer" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/ocispec" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/spf13/cobra" + "go.ate.dev/substrate/cmd/kubectl-ate/internal/printer" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/ocispec" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" diff --git a/cmd/kubectl-ate/internal/cmd/actor_test.go b/cmd/kubectl-ate/internal/cmd/actor_test.go index ce4c31bc2b..634369c2b8 100644 --- a/cmd/kubectl-ate/internal/cmd/actor_test.go +++ b/cmd/kubectl-ate/internal/cmd/actor_test.go @@ -24,9 +24,9 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" diff --git a/cmd/kubectl-ate/internal/cmd/actortemplate.go b/cmd/kubectl-ate/internal/cmd/actortemplate.go index a27d48fa47..8a73095942 100644 --- a/cmd/kubectl-ate/internal/cmd/actortemplate.go +++ b/cmd/kubectl-ate/internal/cmd/actortemplate.go @@ -19,9 +19,9 @@ import ( "io" "os" - "github.com/agent-substrate/substrate/cmd/kubectl-ate/internal/printer" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/spf13/cobra" + "go.ate.dev/substrate/cmd/kubectl-ate/internal/printer" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/encoding/protojson" "sigs.k8s.io/yaml" ) diff --git a/cmd/kubectl-ate/internal/cmd/actortemplate_test.go b/cmd/kubectl-ate/internal/cmd/actortemplate_test.go index 6243b5a2b5..ec0033e2e9 100644 --- a/cmd/kubectl-ate/internal/cmd/actortemplate_test.go +++ b/cmd/kubectl-ate/internal/cmd/actortemplate_test.go @@ -20,8 +20,8 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/testing/protocmp" ) @@ -41,7 +41,7 @@ workerSelector: workload: counter containers: - name: counter - image: ko://github.com/agent-substrate/substrate/demos/counter + image: ko://go.ate.dev/substrate/demos/counter command: ["/ko-app/counter", "--extra-port=9090"] wakeupProbe: httpGet: @@ -78,7 +78,7 @@ func TestActorTemplateFromManifest(t *testing.T) { WorkerSelector: &ateapipb.Selector{MatchLabels: map[string]string{"workload": "counter"}}, Containers: []*ateapipb.Container{{ Name: "counter", - Image: "ko://github.com/agent-substrate/substrate/demos/counter", + Image: "ko://go.ate.dev/substrate/demos/counter", Command: []string{"/ko-app/counter", "--extra-port=9090"}, WakeupProbe: &ateapipb.WakeupProbe{ HttpGet: &ateapipb.HttpGetAction{Path: "/readyz", Port: 80}, diff --git a/cmd/kubectl-ate/internal/cmd/admin.go b/cmd/kubectl-ate/internal/cmd/admin.go index 3c9c0c1faf..8b73cd227b 100644 --- a/cmd/kubectl-ate/internal/cmd/admin.go +++ b/cmd/kubectl-ate/internal/cmd/admin.go @@ -21,9 +21,9 @@ import ( "text/tabwriter" "time" - "github.com/agent-substrate/substrate/internal/localca" - "github.com/agent-substrate/substrate/internal/localjwtauthority" "github.com/spf13/cobra" + "go.ate.dev/substrate/internal/localca" + "go.ate.dev/substrate/internal/localjwtauthority" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -264,7 +264,13 @@ func updateJWTPool(ctx context.Context, errOut io.Writer, secrets typedcorev1.Se if err != nil { return fmt.Errorf("while marshaling pool: %w", err) } + jwks, err := pool.JWKS() + if err != nil { + return fmt.Errorf("while publishing the pool's public keys: %w", err) + } + // Written together so a verifier reading "jwks" never lags "pool". secret.Data["pool"] = wire + secret.Data["jwks"] = jwks if _, err := secrets.Update(ctx, secret, metav1.UpdateOptions{}); err != nil { return fmt.Errorf("while writing pool secret: %w", err) } @@ -324,6 +330,10 @@ func newJWTPoolSecret(namespace, name, algorithm, keyID string) (*corev1.Secret, if err != nil { return nil, "", fmt.Errorf("while generating JWT authority pool: %w", err) } + jwks, err := localjwtauthority.PublicJWKS(poolBytes) + if err != nil { + return nil, "", fmt.Errorf("while publishing the pool's public keys: %w", err) + } return &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ @@ -332,6 +342,7 @@ func newJWTPoolSecret(namespace, name, algorithm, keyID string) (*corev1.Secret, }, Data: map[string][]byte{ "pool": poolBytes, + "jwks": jwks, }, }, id, nil } diff --git a/cmd/kubectl-ate/internal/cmd/admin_test.go b/cmd/kubectl-ate/internal/cmd/admin_test.go index a7a5fe681e..2cab7ec11b 100644 --- a/cmd/kubectl-ate/internal/cmd/admin_test.go +++ b/cmd/kubectl-ate/internal/cmd/admin_test.go @@ -15,6 +15,7 @@ package cmd import ( + "bytes" "context" "io" "strings" @@ -29,7 +30,7 @@ import ( "k8s.io/client-go/kubernetes/fake" k8stesting "k8s.io/client-go/testing" - "github.com/agent-substrate/substrate/internal/localjwtauthority" + "go.ate.dev/substrate/internal/localjwtauthority" ) func TestNewJWTPoolSecretWithFlagDefaults(t *testing.T) { @@ -94,6 +95,9 @@ func poolState(t *testing.T, kc *fake.Clientset) ([]string, string) { if err != nil { t.Fatal(err) } + if jwks, err := pool.JWKS(); err != nil || !bytes.Equal(jwks, secret.Data["jwks"]) { + t.Errorf("jwks = %s, want the pool's public keys (%v)", secret.Data["jwks"], err) + } var ids []string for _, a := range pool.Authorities { ids = append(ids, a.ID) diff --git a/cmd/kubectl-ate/internal/cmd/atespace.go b/cmd/kubectl-ate/internal/cmd/atespace.go index 8d208bdc69..48d673b08d 100644 --- a/cmd/kubectl-ate/internal/cmd/atespace.go +++ b/cmd/kubectl-ate/internal/cmd/atespace.go @@ -17,9 +17,9 @@ package cmd import ( "fmt" - "github.com/agent-substrate/substrate/cmd/kubectl-ate/internal/printer" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/spf13/cobra" + "go.ate.dev/substrate/cmd/kubectl-ate/internal/printer" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) var getAtespacesCmd = &cobra.Command{ diff --git a/cmd/kubectl-ate/internal/cmd/egress_policy.go b/cmd/kubectl-ate/internal/cmd/egress_policy.go index fb06708e8e..bc2c45ac46 100644 --- a/cmd/kubectl-ate/internal/cmd/egress_policy.go +++ b/cmd/kubectl-ate/internal/cmd/egress_policy.go @@ -21,10 +21,10 @@ import ( "fmt" "io" - "github.com/agent-substrate/substrate/cmd/kubectl-ate/internal/printer" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/spf13/cobra" + "go.ate.dev/substrate/cmd/kubectl-ate/internal/printer" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" diff --git a/cmd/kubectl-ate/internal/cmd/egress_policy_test.go b/cmd/kubectl-ate/internal/cmd/egress_policy_test.go index b740d5724a..8faae33123 100644 --- a/cmd/kubectl-ate/internal/cmd/egress_policy_test.go +++ b/cmd/kubectl-ate/internal/cmd/egress_policy_test.go @@ -23,11 +23,11 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/kubectl-ate/internal/printer" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" "github.com/spf13/cobra" + "go.ate.dev/substrate/cmd/kubectl-ate/internal/printer" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" diff --git a/cmd/kubectl-ate/internal/cmd/resourceref.go b/cmd/kubectl-ate/internal/cmd/resourceref.go index 27191be8ca..b2f43c59cf 100644 --- a/cmd/kubectl-ate/internal/cmd/resourceref.go +++ b/cmd/kubectl-ate/internal/cmd/resourceref.go @@ -18,8 +18,8 @@ import ( "fmt" "strings" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // parseAtespacedName parses a reference to an atespace-scoped resource, diff --git a/cmd/kubectl-ate/internal/cmd/resourceref_test.go b/cmd/kubectl-ate/internal/cmd/resourceref_test.go index c269494079..8decc3ee9d 100644 --- a/cmd/kubectl-ate/internal/cmd/resourceref_test.go +++ b/cmd/kubectl-ate/internal/cmd/resourceref_test.go @@ -17,8 +17,8 @@ package cmd import ( "testing" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/testing/protocmp" ) diff --git a/cmd/kubectl-ate/internal/cmd/revert_actor.go b/cmd/kubectl-ate/internal/cmd/revert_actor.go index d4890c9539..87ce5290c7 100644 --- a/cmd/kubectl-ate/internal/cmd/revert_actor.go +++ b/cmd/kubectl-ate/internal/cmd/revert_actor.go @@ -17,10 +17,10 @@ package cmd import ( "fmt" - "github.com/agent-substrate/substrate/cmd/kubectl-ate/internal/printer" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/spf13/cobra" + "go.ate.dev/substrate/cmd/kubectl-ate/internal/printer" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) var revertAtespaceFlag string diff --git a/cmd/kubectl-ate/internal/cmd/root.go b/cmd/kubectl-ate/internal/cmd/root.go index e4d1ea1a07..14e8db801c 100644 --- a/cmd/kubectl-ate/internal/cmd/root.go +++ b/cmd/kubectl-ate/internal/cmd/root.go @@ -23,8 +23,8 @@ import ( "github.com/spf13/cobra" "google.golang.org/grpc/metadata" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/version" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/version" ) var ( diff --git a/cmd/kubectl-ate/internal/cmd/root_test.go b/cmd/kubectl-ate/internal/cmd/root_test.go index 61f5b86fd3..ae9c65a27b 100644 --- a/cmd/kubectl-ate/internal/cmd/root_test.go +++ b/cmd/kubectl-ate/internal/cmd/root_test.go @@ -20,9 +20,9 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/internal/preview" "github.com/google/go-cmp/cmp" "github.com/spf13/cobra" + "go.ate.dev/substrate/internal/preview" "google.golang.org/grpc/metadata" ) diff --git a/cmd/kubectl-ate/internal/cmd/tag.go b/cmd/kubectl-ate/internal/cmd/tag.go index 9f34843457..a99355971a 100644 --- a/cmd/kubectl-ate/internal/cmd/tag.go +++ b/cmd/kubectl-ate/internal/cmd/tag.go @@ -19,9 +19,9 @@ import ( "fmt" "strings" - "github.com/agent-substrate/substrate/cmd/kubectl-ate/internal/printer" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/spf13/cobra" + "go.ate.dev/substrate/cmd/kubectl-ate/internal/printer" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" ) diff --git a/cmd/kubectl-ate/internal/cmd/tag_test.go b/cmd/kubectl-ate/internal/cmd/tag_test.go index fe63752ea5..52c3053e40 100644 --- a/cmd/kubectl-ate/internal/cmd/tag_test.go +++ b/cmd/kubectl-ate/internal/cmd/tag_test.go @@ -18,8 +18,8 @@ import ( "context" "testing" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/testing/protocmp" diff --git a/cmd/kubectl-ate/internal/cmd/verbose.go b/cmd/kubectl-ate/internal/cmd/verbose.go index 0a718a6218..c6ef24a6e9 100644 --- a/cmd/kubectl-ate/internal/cmd/verbose.go +++ b/cmd/kubectl-ate/internal/cmd/verbose.go @@ -23,8 +23,8 @@ import ( "sync" "time" - "github.com/agent-substrate/substrate/internal/ateclient" "github.com/spf13/cobra" + "go.ate.dev/substrate/internal/ateclient" "google.golang.org/grpc" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" @@ -75,6 +75,7 @@ func loadKubeConfig(errOut io.Writer, kubeconfigPath, contextName string) (*rest func newAPIClient(cmd *cobra.Command) (*ateclient.Client, error) { ctx := cmd.Context() errOut := cmd.ErrOrStderr() + warnSharedIdentity(errOut, tokenFile) if verbosity >= vFlow { vlogf(errOut, vFlow, "Connecting to ate-api-server (endpoint=%q, kubeconfig=%q, context=%q, token-file=%q, trace=%t)", @@ -105,6 +106,18 @@ func newAPIClient(cmd *cobra.Command) (*ateclient.Client, error) { return cli, nil } +// warnSharedIdentity warns on errOut when the command will act as the shared +// fallback ServiceAccount rather than as the user. Everyone relying on the +// fallback is the same principal, holding whatever that principal is granted: +// global owner wherever the install names it a bootstrap owner. +func warnSharedIdentity(errOut io.Writer, tokenFile string) { + if tokenFile != "" { + return + } + fmt.Fprintf(errOut, "Warning: no --token-file given; acting as the shared identity %s rather than as yourself. Pass --token-file to act as yourself.\n", + ateclient.FallbackPrincipal()) +} + func formatProtoCompact(v any) string { msg, ok := v.(proto.Message) if !ok || msg == nil { diff --git a/cmd/kubectl-ate/internal/cmd/verbose_test.go b/cmd/kubectl-ate/internal/cmd/verbose_test.go index c036ae93fd..ad36577387 100644 --- a/cmd/kubectl-ate/internal/cmd/verbose_test.go +++ b/cmd/kubectl-ate/internal/cmd/verbose_test.go @@ -24,8 +24,8 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "google.golang.org/protobuf/proto" corev1 "k8s.io/api/core/v1" @@ -287,3 +287,23 @@ func TestTopWorkersRunner_VerboseSurfacesMetricsIssues(t *testing.T) { } }) } + +func TestWarnSharedIdentity(t *testing.T) { + t.Run("warns without a token file", func(t *testing.T) { + var buf bytes.Buffer + warnSharedIdentity(&buf, "") + got := buf.String() + for _, want := range []string{"Warning:", "ate-client", "shared identity", "--token-file"} { + if !strings.Contains(got, want) { + t.Errorf("warning %q does not mention %q", got, want) + } + } + }) + t.Run("silent with a token file", func(t *testing.T) { + var buf bytes.Buffer + warnSharedIdentity(&buf, "/path/to/token") + if got := buf.String(); got != "" { + t.Errorf("warning = %q, want none", got) + } + }) +} diff --git a/cmd/kubectl-ate/internal/cmd/worker.go b/cmd/kubectl-ate/internal/cmd/worker.go index cd499985dc..518ab4bb38 100644 --- a/cmd/kubectl-ate/internal/cmd/worker.go +++ b/cmd/kubectl-ate/internal/cmd/worker.go @@ -19,10 +19,10 @@ import ( "fmt" "io" - "github.com/agent-substrate/substrate/cmd/kubectl-ate/internal/printer" - "github.com/agent-substrate/substrate/internal/ateclient" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/spf13/cobra" + "go.ate.dev/substrate/cmd/kubectl-ate/internal/printer" + "go.ate.dev/substrate/internal/ateclient" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" diff --git a/cmd/kubectl-ate/internal/cmd/worker_test.go b/cmd/kubectl-ate/internal/cmd/worker_test.go index 04a33d8bc3..adc5c39d8f 100644 --- a/cmd/kubectl-ate/internal/cmd/worker_test.go +++ b/cmd/kubectl-ate/internal/cmd/worker_test.go @@ -21,9 +21,9 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/kubectl-ate/internal/printer" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/cmd/kubectl-ate/internal/printer" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "google.golang.org/protobuf/types/known/timestamppb" corev1 "k8s.io/api/core/v1" diff --git a/cmd/kubectl-ate/internal/printer/access_policy.go b/cmd/kubectl-ate/internal/printer/access_policy.go new file mode 100644 index 0000000000..a260e02897 --- /dev/null +++ b/cmd/kubectl-ate/internal/printer/access_policy.go @@ -0,0 +1,50 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package printer + +import ( + "fmt" + "io" + "strings" + "text/tabwriter" + + "go.ate.dev/substrate/pkg/proto/ateapipb" +) + +// PrintAccessPolicyTo prints an access policy, one table row per binding. +// json and yaml emit the bare AccessPolicy. The policy's metadata carries no +// atespace, so the caller passes it; empty means the global policy. +func PrintAccessPolicyTo(out io.Writer, atespace string, policy *ateapipb.AccessPolicy, format string) error { + switch format { + case "json", "yaml": + return printProto(out, policy, format) + case "table": + scope := atespace + if scope == "" { + scope = "" + } + w := tabwriter.NewWriter(out, 0, 0, 3, ' ', 0) + fmt.Fprintln(w, "SCOPE\tROLE\tMEMBERS") + if len(policy.GetBindings()) == 0 { + fmt.Fprintf(w, "%s\t\t\n", scope) + } + for _, b := range policy.GetBindings() { + fmt.Fprintf(w, "%s\t%s\t%s\n", scope, b.GetRole(), strings.Join(b.GetMembers(), ",")) + } + return w.Flush() + default: + return fmt.Errorf("unsupported format %q", format) + } +} diff --git a/cmd/kubectl-ate/internal/printer/access_policy_test.go b/cmd/kubectl-ate/internal/printer/access_policy_test.go new file mode 100644 index 0000000000..e068c025ce --- /dev/null +++ b/cmd/kubectl-ate/internal/printer/access_policy_test.go @@ -0,0 +1,79 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package printer + +import ( + "bytes" + "strings" + "testing" + + "go.ate.dev/substrate/pkg/proto/ateapipb" +) + +func TestPrintAccessPolicyTo(t *testing.T) { + policy := &ateapipb.AccessPolicy{ + Metadata: &ateapipb.ResourceMetadata{Name: "default", Uid: "u-1", Version: 2}, + Bindings: []*ateapipb.Binding{ + {Role: "owner", Members: []string{"user:alice", "user:bob"}}, + {Role: "viewer", Members: []string{"user:carol"}}, + }, + } + tests := []struct { + name string + atespace string + policy *ateapipb.AccessPolicy + format string + want []string + wantErr bool + }{ + { + name: "table", atespace: "team-a", policy: policy, format: "table", + want: []string{"SCOPE", "team-a owner user:alice,user:bob", "team-a viewer user:carol"}, + }, + { + name: "global table", policy: policy, format: "table", + want: []string{" owner"}, + }, + { + name: "no bindings", atespace: "team-a", policy: &ateapipb.AccessPolicy{}, format: "table", + want: []string{"team-a "}, + }, + { + name: "yaml carries preconditions", atespace: "team-a", policy: policy, format: "yaml", + want: []string{"uid: u-1", `version: "2"`, "role: owner"}, + }, + {name: "unknown format", policy: policy, format: "xml", wantErr: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var out bytes.Buffer + err := PrintAccessPolicyTo(&out, tc.atespace, tc.policy, tc.format) + if tc.wantErr { + if err == nil { + t.Fatal("PrintAccessPolicyTo() succeeded, want an error") + } + return + } + if err != nil { + t.Fatalf("PrintAccessPolicyTo() error = %v", err) + } + for _, w := range tc.want { + if !strings.Contains(out.String(), w) { + t.Errorf("output = %q, want it to contain %q", out.String(), w) + } + } + }) + } +} diff --git a/cmd/kubectl-ate/internal/printer/egress_policy.go b/cmd/kubectl-ate/internal/printer/egress_policy.go index 153b0cc77f..7af33d316e 100644 --- a/cmd/kubectl-ate/internal/printer/egress_policy.go +++ b/cmd/kubectl-ate/internal/printer/egress_policy.go @@ -19,7 +19,7 @@ import ( "io" "text/tabwriter" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // PrintEgressPolicyTo prints one actor's egress policy. json and yaml emit the diff --git a/cmd/kubectl-ate/internal/printer/egress_policy_test.go b/cmd/kubectl-ate/internal/printer/egress_policy_test.go index 58c1c2b97d..e1ee36cd67 100644 --- a/cmd/kubectl-ate/internal/printer/egress_policy_test.go +++ b/cmd/kubectl-ate/internal/printer/egress_policy_test.go @@ -19,8 +19,8 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/types/known/timestamppb" ) diff --git a/cmd/kubectl-ate/internal/printer/printer.go b/cmd/kubectl-ate/internal/printer/printer.go index f77cbcecf6..7b7cf83412 100644 --- a/cmd/kubectl-ate/internal/printer/printer.go +++ b/cmd/kubectl-ate/internal/printer/printer.go @@ -24,7 +24,7 @@ import ( "text/tabwriter" "time" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" diff --git a/cmd/kubectl-ate/internal/printer/printer_test.go b/cmd/kubectl-ate/internal/printer/printer_test.go index 08c9b76366..fb78ec1261 100644 --- a/cmd/kubectl-ate/internal/printer/printer_test.go +++ b/cmd/kubectl-ate/internal/printer/printer_test.go @@ -19,8 +19,8 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/types/known/timestamppb" ) diff --git a/cmd/kubectl-ate/main.go b/cmd/kubectl-ate/main.go index 1adface558..3109c5e745 100644 --- a/cmd/kubectl-ate/main.go +++ b/cmd/kubectl-ate/main.go @@ -15,7 +15,7 @@ package main import ( - "github.com/agent-substrate/substrate/cmd/kubectl-ate/internal/cmd" + "go.ate.dev/substrate/cmd/kubectl-ate/internal/cmd" ) func main() { diff --git a/cmd/objectstore-plugin/README.md b/cmd/objectstore-plugin/README.md new file mode 100644 index 0000000000..9e0b5d2fcb --- /dev/null +++ b/cmd/objectstore-plugin/README.md @@ -0,0 +1,44 @@ +# Object-store plugin upload memory + +The node plugin shares one upload memory budget across all snapshot requests. +`--upload-memory-budget` accepts a Kubernetes quantity and defaults to `512Mi`. +Uploads wait before allocating compressor or backend buffers. Waiting requests +cancel with their RPC, and failed uploads release their reservations after their +compression and upload workers stop. + +Reservations cover parallel compression input buffers, queued compressed frames, +zstd state, and the storage client's upload buffers. GCS range uploads reserve +serial encoders and resumable-upload buffers together. Large files reduce their +compression or range parallelism to fit the budget; file size does not need to fit +in memory. The sparse format and download readers are unchanged. + +With the current buffer sizes, the minimum budget is `384Mi` for GCS and `224Mi` +for S3. Startup rejects smaller values. The accounting is conservative: GCS +reserves for a composite upload even when an object ultimately fits in one +request. Raise the budget to trade memory for more simultaneous uploads. + +The budget is **not a process memory limit**. The Go runtime, garbage awaiting +collection, connections, downloads, and asset transfers need additional memory. +Size the sidecar's Kubernetes memory request and limit for those costs as well; +the existing `128Mi` request is not a bound on its usage. If setting `GOMEMLIMIT`, +leave headroom below the container limit for memory outside the Go heap. + +A manifest included with data files is uploaded only after all those files +succeed. The existing two-RPC flow (data, then manifest) also uses the same budget. + +## Validation + +```sh +go test -race ./pkg/objectstorage ./internal/objectstoreplugin ./cmd/objectstore-plugin -count=2 +go test ./pkg/objectstorage -run '^$' -bench '^BenchmarkConcurrentSnapshotUploads$' -benchtime=1x -count=1 +``` + +The tests cover ten overlapping snapshots with eight data files each, canceled +waiters, source and destination failures, retry after failure, manifest ordering, +large sparse sources, and slow uploads through the real S3 upload manager and GCS +writers. Their network peers are simulated; they do not require cloud credentials. + +The benchmark compares admitting all 80 tiny-file uploads with sharing the +budget, using a destination that stalls for 20 ms. It samples peak heap growth +and reports files per second. It isolates compression backpressure; it is not a +measurement of cloud throughput or the sidecar's total resident memory. diff --git a/cmd/objectstore-plugin/main.go b/cmd/objectstore-plugin/main.go index fba93439c2..6aa3469f5c 100644 --- a/cmd/objectstore-plugin/main.go +++ b/cmd/objectstore-plugin/main.go @@ -31,15 +31,15 @@ import ( "time" "cloud.google.com/go/storage" - "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/objectstore" - "github.com/agent-substrate/substrate/internal/objectstoreplugin" - "github.com/agent-substrate/substrate/internal/pluginsocket" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/serverboot" - "github.com/agent-substrate/substrate/pkg/objectstorage" - objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" "github.com/spf13/pflag" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/objectstore" + "go.ate.dev/substrate/internal/objectstoreplugin" + "go.ate.dev/substrate/internal/pluginsocket" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/serverboot" + "go.ate.dev/substrate/pkg/objectstorage" + objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "google.golang.org/grpc/health" @@ -58,6 +58,7 @@ func main() { flags := pflag.NewFlagSet(mode, pflag.ExitOnError) socket := flags.String("socket", "", "Unix socket to serve on, or to check in healthcheck mode") root := flags.String("root", nodepath.BasePath, "node mode: the only directory tree local snapshot and asset files may be read from or written to") + uploadMemory := flags.String("upload-memory-budget", resource.NewQuantity(objectstorage.DefaultUploadMemory, resource.BinarySI).String(), "node mode: shared memory reservations for snapshot compression and upload buffers, as a Kubernetes quantity; uploads wait for capacity") assetStagingDir := flags.String("asset-staging-dir", "/var/lib/objectstore-plugin/asset-staging", "node mode: directory sandbox assets are downloaded into and verified before they are copied below --root; must not be below --root or readable by the caller") assetStagingCapacity := flags.String("asset-staging-capacity", "8Gi", "node mode: the most sandbox asset downloads in --asset-staging-dir may hold at once, as a Kubernetes quantity; downloads wait for space, and an asset larger than this is refused") timeout := flags.Duration("timeout", 5*time.Second, "healthcheck mode: how long to wait for the plugin to report that it is serving") @@ -98,9 +99,13 @@ func main() { if err != nil { serverboot.Fatal(ctx, "Failed to set up the object storage backend", err) } - plugin, err := objectstoreplugin.NewNodePlugin(objects, *root) + uploadCapacity, err := parseCapacity(*uploadMemory) if err != nil { - serverboot.Fatal(ctx, "Invalid --root", err) + serverboot.Fatal(ctx, "Invalid --upload-memory-budget", err) + } + plugin, err := objectstoreplugin.NewNodePlugin(objects, *root, uploadCapacity) + if err != nil { + serverboot.Fatal(ctx, "Invalid node upload configuration", err) } objectstorev1.RegisterNodeProviderServer(srv, plugin) capacity, err := parseCapacity(*assetStagingCapacity) diff --git a/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go b/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go index ffd03105af..50ab670afb 100644 --- a/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go +++ b/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go @@ -26,14 +26,13 @@ import ( "path" "time" - "github.com/agent-substrate/substrate/cmd/podcertcontroller/internal/podcertificate" - "github.com/agent-substrate/substrate/cmd/podcertcontroller/internal/signercontroller" - "github.com/agent-substrate/substrate/internal/localca" - "github.com/agent-substrate/substrate/internal/substratex509" + "go.ate.dev/substrate/cmd/podcertcontroller/internal/podcertificate" + "go.ate.dev/substrate/cmd/podcertcontroller/internal/signercontroller" + "go.ate.dev/substrate/internal/localca" + "go.ate.dev/substrate/internal/substratex509" certsv1 "k8s.io/api/certificates/v1" certsv1beta1 "k8s.io/api/certificates/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" "k8s.io/utils/ptr" ) @@ -41,14 +40,12 @@ const Name = "podidentity.podcert.ate.dev/identity" const CTBPrefix = "podidentity.podcert.ate.dev:identity:" type Impl struct { - kc kubernetes.Interface pcrClient *podcertificate.Client caPool localca.Pool } -func NewImpl(kc kubernetes.Interface, caPool localca.Pool, pcrClient *podcertificate.Client) *Impl { +func NewImpl(caPool localca.Pool, pcrClient *podcertificate.Client) *Impl { return &Impl{ - kc: kc, pcrClient: pcrClient, caPool: caPool, } @@ -96,16 +93,11 @@ func (h *Impl) DesiredClusterTrustBundles() ([]*certsv1.ClusterTrustBundle, erro } func (h *Impl) MakeCert(ctx context.Context, pcr *certsv1beta1.PodCertificateRequest) error { - // Fetch the pod to get its ServiceAccount - pod, err := h.kc.CoreV1().Pods(pcr.ObjectMeta.Namespace).Get(ctx, pcr.Spec.PodName, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("while getting pod %s/%s: %w", pcr.ObjectMeta.Namespace, pcr.Spec.PodName, err) - } - - if pod.ObjectMeta.UID != pcr.Spec.PodUID { - return fmt.Errorf("pod UID mismatch: expected %s, got %s", pcr.Spec.PodUID, pod.ObjectMeta.UID) - } - + // The identity comes from the PCR spec alone. kube-apiserver attests it + // for kubelet-created requests (the pod, ServiceAccount and node exist and + // belong together); anyone else needs RBAC permission to create + // PodCertificateRequests and may name any identity, which is how + // non-kubelet clients such as the e2e suites obtain certificates. subjectPublicKey, err := podcertificate.PublicKey(pcr) if err != nil { return err diff --git a/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner_test.go b/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner_test.go index 0de8d3d0e5..5254c3c14a 100644 --- a/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner_test.go +++ b/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner_test.go @@ -27,11 +27,10 @@ import ( "testing/synctest" "time" - "github.com/agent-substrate/substrate/cmd/podcertcontroller/internal/podcertificate" - "github.com/agent-substrate/substrate/internal/localca" - "github.com/agent-substrate/substrate/internal/substratex509" + "go.ate.dev/substrate/cmd/podcertcontroller/internal/podcertificate" + "go.ate.dev/substrate/internal/localca" + "go.ate.dev/substrate/internal/substratex509" certsv1beta1 "k8s.io/api/certificates/v1beta1" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -40,25 +39,19 @@ import ( "k8s.io/utils/ptr" ) -// makePodAndPCR returns a pod and a matching PodCertificateRequest with no -// key material set; callers fill in StubPKCS10Request. -func makePodAndPCR(namespace, podName, serviceAccount string, maxExpirationSeconds int32) (*corev1.Pod, *certsv1beta1.PodCertificateRequest) { - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: podName, - UID: types.UID("pod-uid-1"), - }, - } - pcr := &certsv1beta1.PodCertificateRequest{ +// makePCR returns a PodCertificateRequest with no key material set; callers +// fill in StubPKCS10Request. The pod, ServiceAccount and node it names exist +// nowhere: the signer takes the identity from the request alone. +func makePCR(namespace, podName, serviceAccount string, maxExpirationSeconds int32) *certsv1beta1.PodCertificateRequest { + return &certsv1beta1.PodCertificateRequest{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, Name: "req-1", }, Spec: certsv1beta1.PodCertificateRequestSpec{ SignerName: Name, - PodName: pod.ObjectMeta.Name, - PodUID: pod.ObjectMeta.UID, + PodName: podName, + PodUID: types.UID("pod-uid-1"), ServiceAccountName: serviceAccount, ServiceAccountUID: types.UID("sa-uid-1"), NodeName: types.NodeName("node-1"), @@ -66,7 +59,6 @@ func makePodAndPCR(namespace, podName, serviceAccount string, maxExpirationSecon MaxExpirationSeconds: ptr.To(maxExpirationSeconds), }, } - return pod, pcr } // stubCSR returns a stub PKCS#10 request carrying priv's public key. @@ -85,7 +77,6 @@ func TestMakeCert(t *testing.T) { namespace string podName string serviceAccount string - podLabels map[string]string maxExpirationSeconds int32 wantLifetime time.Duration wantURI string @@ -118,7 +109,6 @@ func TestMakeCert(t *testing.T) { namespace: "ate-demo-counter", podName: "counter-abcde", serviceAccount: "default", - podLabels: map[string]string{"ate.dev/worker-pool": "counter"}, maxExpirationSeconds: 86400, wantLifetime: 24 * time.Hour, wantURI: "spiffe://cluster.local/ns/ate-demo-counter/sa/default", @@ -206,12 +196,12 @@ func TestMakeCert(t *testing.T) { t.Fatalf("while generating subject key: %v", err) } - pod, pcr := makePodAndPCR(tc.namespace, tc.podName, tc.serviceAccount, tc.maxExpirationSeconds) - pod.ObjectMeta.Labels = tc.podLabels + pcr := makePCR(tc.namespace, tc.podName, tc.serviceAccount, tc.maxExpirationSeconds) pcr.Spec.StubPKCS10Request = stubCSR(t, subjectPriv) - kc := fake.NewSimpleClientset(pod, pcr) - impl := NewImpl(kc, caPool, betaClient(t, kc)) + // Only the PCR exists: the signer must not need the pod. + kc := fake.NewSimpleClientset(pcr) + impl := NewImpl(caPool, betaClient(t, kc)) if err := impl.MakeCert(context.Background(), pcr); err != nil { t.Fatalf("MakeCert: %v", err) @@ -296,28 +286,15 @@ func TestMakeCert(t *testing.T) { func TestMakeCertErrors(t *testing.T) { testCases := []struct { name string - omitPod bool - podUID types.UID omitKey bool failUpdate bool }{ - { - name: "pod not found", - omitPod: true, - podUID: "pod-uid-1", - }, - { - name: "pod UID mismatch", - podUID: "other-uid", - }, { name: "no key material in PCR", - podUID: "pod-uid-1", omitKey: true, }, { name: "status update fails", - podUID: "pod-uid-1", failUpdate: true, }, } @@ -330,8 +307,7 @@ func TestMakeCertErrors(t *testing.T) { } caPool := &localca.ConcretePool{CAs: []*localca.CA{ca}} - pod, pcr := makePodAndPCR("ate-system", "atelet-abcde", "atelet", 86400) - pod.ObjectMeta.UID = tc.podUID + pcr := makePCR("ate-system", "atelet-abcde", "atelet", 86400) if !tc.omitKey { _, subjectPriv, err := ed25519.GenerateKey(rand.Reader) if err != nil { @@ -340,17 +316,13 @@ func TestMakeCertErrors(t *testing.T) { pcr.Spec.StubPKCS10Request = stubCSR(t, subjectPriv) } - objects := []runtime.Object{pcr} - if !tc.omitPod { - objects = append(objects, pod) - } - kc := fake.NewSimpleClientset(objects...) + kc := fake.NewSimpleClientset(pcr) if tc.failUpdate { kc.PrependReactor("update", "podcertificaterequests", func(action k8stesting.Action) (bool, runtime.Object, error) { return true, nil, errors.New("injected update failure") }) } - impl := NewImpl(kc, caPool, betaClient(t, kc)) + impl := NewImpl(caPool, betaClient(t, kc)) if err := impl.MakeCert(context.Background(), pcr); err == nil { t.Fatalf("MakeCert: got nil error, want error") @@ -377,7 +349,7 @@ func TestDesiredClusterTrustBundles(t *testing.T) { t.Fatalf("while generating CA 2: %v", err) } caPool := &localca.ConcretePool{CAs: []*localca.CA{ca1, ca2}} - impl := NewImpl(nil, caPool, nil) + impl := NewImpl(caPool, nil) ctbs, err := impl.DesiredClusterTrustBundles() if err != nil { diff --git a/cmd/podcertcontroller/internal/postgressigner/postgressigner.go b/cmd/podcertcontroller/internal/postgressigner/postgressigner.go index 61d12ee432..48b87e8084 100644 --- a/cmd/podcertcontroller/internal/postgressigner/postgressigner.go +++ b/cmd/podcertcontroller/internal/postgressigner/postgressigner.go @@ -26,10 +26,10 @@ import ( "strings" "time" - "github.com/agent-substrate/substrate/cmd/podcertcontroller/internal/podcertificate" - "github.com/agent-substrate/substrate/cmd/podcertcontroller/internal/signercontroller" - "github.com/agent-substrate/substrate/internal/localca" - "github.com/agent-substrate/substrate/pkg/postgressetup" + "go.ate.dev/substrate/cmd/podcertcontroller/internal/podcertificate" + "go.ate.dev/substrate/cmd/podcertcontroller/internal/signercontroller" + "go.ate.dev/substrate/internal/localca" + "go.ate.dev/substrate/pkg/postgressetup" certsv1 "k8s.io/api/certificates/v1" certsv1beta1 "k8s.io/api/certificates/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/cmd/podcertcontroller/internal/postgressigner/postgressigner_test.go b/cmd/podcertcontroller/internal/postgressigner/postgressigner_test.go index 8807c52128..ffa3a6781e 100644 --- a/cmd/podcertcontroller/internal/postgressigner/postgressigner_test.go +++ b/cmd/podcertcontroller/internal/postgressigner/postgressigner_test.go @@ -25,10 +25,10 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/cmd/podcertcontroller/internal/podcertificate" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/internal/localca" - "github.com/agent-substrate/substrate/pkg/postgressetup" + "go.ate.dev/substrate/cmd/podcertcontroller/internal/podcertificate" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/localca" + "go.ate.dev/substrate/pkg/postgressetup" certsv1beta1 "k8s.io/api/certificates/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go b/cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go index d482f3ed23..9e57142828 100644 --- a/cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go +++ b/cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go @@ -22,9 +22,9 @@ import ( "fmt" "time" - "github.com/agent-substrate/substrate/cmd/podcertcontroller/internal/podcertificate" - "github.com/agent-substrate/substrate/cmd/podcertcontroller/internal/signercontroller" - "github.com/agent-substrate/substrate/internal/localca" + "go.ate.dev/substrate/cmd/podcertcontroller/internal/podcertificate" + "go.ate.dev/substrate/cmd/podcertcontroller/internal/signercontroller" + "go.ate.dev/substrate/internal/localca" certsv1 "k8s.io/api/certificates/v1" certsv1beta1 "k8s.io/api/certificates/v1beta1" corev1 "k8s.io/api/core/v1" diff --git a/cmd/podcertcontroller/internal/signercontroller/signercontroller.go b/cmd/podcertcontroller/internal/signercontroller/signercontroller.go index b768086247..97cfa16dd1 100644 --- a/cmd/podcertcontroller/internal/signercontroller/signercontroller.go +++ b/cmd/podcertcontroller/internal/signercontroller/signercontroller.go @@ -21,9 +21,9 @@ import ( "log/slog" "time" - "github.com/agent-substrate/substrate/cmd/podcertcontroller/internal/podcertificate" - "github.com/agent-substrate/substrate/cmd/podcertcontroller/internal/rendezvous" - "github.com/agent-substrate/substrate/internal/clustertrustbundle" + "go.ate.dev/substrate/cmd/podcertcontroller/internal/podcertificate" + "go.ate.dev/substrate/cmd/podcertcontroller/internal/rendezvous" + "go.ate.dev/substrate/internal/clustertrustbundle" certsv1 "k8s.io/api/certificates/v1" certsv1beta1 "k8s.io/api/certificates/v1beta1" apiequality "k8s.io/apimachinery/pkg/api/equality" diff --git a/cmd/podcertcontroller/internal/signercontroller/signercontroller_test.go b/cmd/podcertcontroller/internal/signercontroller/signercontroller_test.go index 2b89cd7aa3..01490f45a0 100644 --- a/cmd/podcertcontroller/internal/signercontroller/signercontroller_test.go +++ b/cmd/podcertcontroller/internal/signercontroller/signercontroller_test.go @@ -18,7 +18,7 @@ import ( "context" "testing" - "github.com/agent-substrate/substrate/internal/clustertrustbundle" + "go.ate.dev/substrate/internal/clustertrustbundle" certsv1 "k8s.io/api/certificates/v1" certsv1beta1 "k8s.io/api/certificates/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/cmd/podcertcontroller/main.go b/cmd/podcertcontroller/main.go index 16922b292b..1a782cd700 100644 --- a/cmd/podcertcontroller/main.go +++ b/cmd/podcertcontroller/main.go @@ -31,18 +31,18 @@ import ( "path/filepath" "syscall" - "github.com/agent-substrate/substrate/cmd/podcertcontroller/internal/podcertificate" - "github.com/agent-substrate/substrate/cmd/podcertcontroller/internal/podidentitysigner" - "github.com/agent-substrate/substrate/cmd/podcertcontroller/internal/postgressigner" - "github.com/agent-substrate/substrate/cmd/podcertcontroller/internal/rendezvous" - "github.com/agent-substrate/substrate/cmd/podcertcontroller/internal/servicednssigner" - "github.com/agent-substrate/substrate/cmd/podcertcontroller/internal/signercontroller" - "github.com/agent-substrate/substrate/internal/clustertrustbundle" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/internal/localca" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/version" "github.com/spf13/pflag" + "go.ate.dev/substrate/cmd/podcertcontroller/internal/podcertificate" + "go.ate.dev/substrate/cmd/podcertcontroller/internal/podidentitysigner" + "go.ate.dev/substrate/cmd/podcertcontroller/internal/postgressigner" + "go.ate.dev/substrate/cmd/podcertcontroller/internal/rendezvous" + "go.ate.dev/substrate/cmd/podcertcontroller/internal/servicednssigner" + "go.ate.dev/substrate/cmd/podcertcontroller/internal/signercontroller" + "go.ate.dev/substrate/internal/clustertrustbundle" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/localca" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/version" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -221,7 +221,7 @@ func main() { slog.ErrorContext(ctx, "Error loading podidentity.podcert.ate.dev/identity CA pool state", slog.Any("err", err)) os.Exit(1) } - podIdentitySignerController := signercontroller.New(clock.RealClock{}, podidentitysigner.NewImpl(kc, podIdentityCAPool, pcrClient), hasher, pcrClient, trustBundles) + podIdentitySignerController := signercontroller.New(clock.RealClock{}, podidentitysigner.NewImpl(podIdentityCAPool, pcrClient), hasher, pcrClient, trustBundles) postgresCAPool, err := localca.NewRefreshingPool(*postgresCAPoolFile) if err != nil { slog.ErrorContext(ctx, "Error loading PostgreSQL CA pool state", slog.Any("err", err)) diff --git a/demos/autoscaled-workerpool/autoscaled-workerpool-template.yaml.tmpl b/demos/autoscaled-workerpool/autoscaled-workerpool-template.yaml.tmpl index d870abf04a..04ac32f934 100644 --- a/demos/autoscaled-workerpool/autoscaled-workerpool-template.yaml.tmpl +++ b/demos/autoscaled-workerpool/autoscaled-workerpool-template.yaml.tmpl @@ -25,7 +25,7 @@ workerSelector: workload: counter-autoscaled containers: - name: counter - image: ko://github.com/agent-substrate/substrate/demos/counter + image: ko://go.ate.dev/substrate/demos/counter command: - /ko-app/counter wakeupProbe: diff --git a/demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl b/demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl index 04aa6672ec..b60beb231f 100644 --- a/demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl +++ b/demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl @@ -37,6 +37,6 @@ metadata: workload: counter-autoscaled spec: replicas: 5 - workerImage: ko://github.com/agent-substrate/substrate/cmd/ateworker-gvisor + workerImage: ko://go.ate.dev/substrate/cmd/ateworker-gvisor sandboxClasses: - name: gvisor diff --git a/demos/claude-code-multiplex/claude-code-multiplex.yaml.tmpl b/demos/claude-code-multiplex/claude-code-multiplex.yaml.tmpl index 5267e91017..005a859c88 100644 --- a/demos/claude-code-multiplex/claude-code-multiplex.yaml.tmpl +++ b/demos/claude-code-multiplex/claude-code-multiplex.yaml.tmpl @@ -34,6 +34,6 @@ metadata: workload: claude-multiplex spec: replicas: 2 - workerImage: ko://github.com/agent-substrate/substrate/cmd/ateworker-gvisor + workerImage: ko://go.ate.dev/substrate/cmd/ateworker-gvisor sandboxClasses: - name: gvisor diff --git a/demos/claude-code-multiplex/ui/server.go b/demos/claude-code-multiplex/ui/server.go index c0881e9d2f..734f0d9e10 100644 --- a/demos/claude-code-multiplex/ui/server.go +++ b/demos/claude-code-multiplex/ui/server.go @@ -58,8 +58,8 @@ import ( "sync" "time" - "github.com/agent-substrate/substrate/internal/ateclient" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/ateclient" + "go.ate.dev/substrate/pkg/proto/ateapipb" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" diff --git a/demos/counter/counter-microvm-csi-test-template.yaml b/demos/counter/counter-microvm-csi-test-template.yaml index fd20354aae..293d0c407d 100644 --- a/demos/counter/counter-microvm-csi-test-template.yaml +++ b/demos/counter/counter-microvm-csi-test-template.yaml @@ -24,7 +24,7 @@ workerSelector: workload: counter-microvm-csi containers: - name: counter - image: ko://github.com/agent-substrate/substrate/demos/counter + image: ko://go.ate.dev/substrate/demos/counter wakeupProbe: httpGet: path: /readyz diff --git a/demos/counter/counter-microvm-csi-test.yaml b/demos/counter/counter-microvm-csi-test.yaml index 55bfbac1b9..06916b100c 100644 --- a/demos/counter/counter-microvm-csi-test.yaml +++ b/demos/counter/counter-microvm-csi-test.yaml @@ -34,4 +34,4 @@ spec: replicas: 1 sandboxClasses: - name: microvm - workerImage: ko://github.com/agent-substrate/substrate/cmd/ateworker-microvm + workerImage: ko://go.ate.dev/substrate/cmd/ateworker-microvm diff --git a/demos/counter/counter-microvm-template.yaml.tmpl b/demos/counter/counter-microvm-template.yaml.tmpl index 77d67fdcc5..e694e3d11d 100644 --- a/demos/counter/counter-microvm-template.yaml.tmpl +++ b/demos/counter/counter-microvm-template.yaml.tmpl @@ -24,7 +24,7 @@ workerSelector: workload: counter-microvm containers: - name: counter - image: ko://github.com/agent-substrate/substrate/demos/counter + image: ko://go.ate.dev/substrate/demos/counter command: - /ko-app/counter # Exercises atenet-router's arbitrary-port ingress support: a second diff --git a/demos/counter/counter-microvm.yaml.tmpl b/demos/counter/counter-microvm.yaml.tmpl index ba7b252886..a3b33e23c5 100644 --- a/demos/counter/counter-microvm.yaml.tmpl +++ b/demos/counter/counter-microvm.yaml.tmpl @@ -43,7 +43,7 @@ spec: - name: microvm configRef: name: microvm - workerImage: ko://github.com/agent-substrate/substrate/cmd/ateworker-microvm + workerImage: ko://go.ate.dev/substrate/cmd/ateworker-microvm template: # Per-worker resources: these size the worker POD and advertise its # scheduling capacity (the per-actor ceiling). The micro-VM sandbox itself is diff --git a/demos/counter/counter-template.yaml.tmpl b/demos/counter/counter-template.yaml.tmpl index 7ab64ca520..9aca738597 100644 --- a/demos/counter/counter-template.yaml.tmpl +++ b/demos/counter/counter-template.yaml.tmpl @@ -27,7 +27,7 @@ workerSelector: workload: counter containers: - name: counter - image: ko://github.com/agent-substrate/substrate/demos/counter + image: ko://go.ate.dev/substrate/demos/counter command: - /ko-app/counter # Exercises atenet-router's arbitrary-port ingress support: a second diff --git a/demos/counter/counter.yaml.tmpl b/demos/counter/counter.yaml.tmpl index 9919666b35..007016795e 100644 --- a/demos/counter/counter.yaml.tmpl +++ b/demos/counter/counter.yaml.tmpl @@ -37,7 +37,7 @@ spec: # pool that cannot place every replica is worse than a smaller one -- the # deploy blocks on a rollout that never completes. replicas: 3 - workerImage: ko://github.com/agent-substrate/substrate/cmd/ateworker-gvisor + workerImage: ko://go.ate.dev/substrate/cmd/ateworker-gvisor # No configRef: the pool follows the class default, the newest SandboxConfig # of the class annotated sandboxconfig.ate.dev/is-class-default=true # (gvisor-default as installed), and moves to a new default when one appears. diff --git a/demos/egress/egress-microvm-template.yaml.tmpl b/demos/egress/egress-microvm-template.yaml.tmpl index ad4400f831..dc0e70a9ff 100644 --- a/demos/egress/egress-microvm-template.yaml.tmpl +++ b/demos/egress/egress-microvm-template.yaml.tmpl @@ -32,7 +32,7 @@ workerSelector: workload: egress-microvm containers: - name: egress - image: ko://github.com/agent-substrate/substrate/demos/egress + image: ko://go.ate.dev/substrate/demos/egress command: ["/ko-app/egress"] # The demo fetches with a stock net/http client. SSL_CERT_FILE adds the # gateway CA, and Go still scans /etc/ssl/certs, so the image's public diff --git a/demos/egress/egress-microvm.yaml.tmpl b/demos/egress/egress-microvm.yaml.tmpl index 7f515c9b70..e45313d9b9 100644 --- a/demos/egress/egress-microvm.yaml.tmpl +++ b/demos/egress/egress-microvm.yaml.tmpl @@ -38,7 +38,7 @@ spec: replicas: 2 sandboxClasses: - name: microvm - workerImage: ko://github.com/agent-substrate/substrate/cmd/ateworker-microvm + workerImage: ko://go.ate.dev/substrate/cmd/ateworker-microvm # No template.resources, unlike the counter demo's micro-VM variant: an # unlimited worker container reports no capacity, which the scheduler reads as # unconstrained, and the pod requests nothing so it stays placeable next to diff --git a/demos/egress/egress-template.yaml.tmpl b/demos/egress/egress-template.yaml.tmpl index 81271c18af..09cf6b9775 100644 --- a/demos/egress/egress-template.yaml.tmpl +++ b/demos/egress/egress-template.yaml.tmpl @@ -30,7 +30,7 @@ workerSelector: workload: egress containers: - name: egress - image: ko://github.com/agent-substrate/substrate/demos/egress + image: ko://go.ate.dev/substrate/demos/egress command: ["/ko-app/egress"] env: - name: SSL_CERT_DIR diff --git a/demos/egress/egress.yaml.tmpl b/demos/egress/egress.yaml.tmpl index 09a798a4c9..234ad18330 100644 --- a/demos/egress/egress.yaml.tmpl +++ b/demos/egress/egress.yaml.tmpl @@ -33,6 +33,6 @@ metadata: workload: egress spec: replicas: 2 - workerImage: ko://github.com/agent-substrate/substrate/cmd/ateworker-gvisor + workerImage: ko://go.ate.dev/substrate/cmd/ateworker-gvisor sandboxClasses: - name: gvisor diff --git a/demos/egress/main.go b/demos/egress/main.go index da40483663..4e32cdedec 100644 --- a/demos/egress/main.go +++ b/demos/egress/main.go @@ -42,7 +42,7 @@ import ( "google.golang.org/grpc/credentials/insecure" grpcstatus "google.golang.org/grpc/status" - "github.com/agent-substrate/substrate/internal/proto/grpcechopb" + "go.ate.dev/substrate/internal/proto/grpcechopb" ) const ( diff --git a/demos/egress/main_test.go b/demos/egress/main_test.go index 82c1dc72d8..80778bbcdc 100644 --- a/demos/egress/main_test.go +++ b/demos/egress/main_test.go @@ -34,8 +34,8 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" - "github.com/agent-substrate/substrate/internal/proto/grpcechopb" - "github.com/agent-substrate/substrate/internal/testcert" + "go.ate.dev/substrate/internal/proto/grpcechopb" + "go.ate.dev/substrate/internal/testcert" ) func TestFetch(t *testing.T) { diff --git a/demos/jupyter/jupyter.yaml.tmpl b/demos/jupyter/jupyter.yaml.tmpl index fb9fcaef2b..e8382d39e8 100644 --- a/demos/jupyter/jupyter.yaml.tmpl +++ b/demos/jupyter/jupyter.yaml.tmpl @@ -32,7 +32,7 @@ metadata: workload: jupyter spec: replicas: 1 - workerImage: ko://github.com/agent-substrate/substrate/cmd/ateworker-gvisor + workerImage: ko://go.ate.dev/substrate/cmd/ateworker-gvisor sandboxClasses: - name: gvisor diff --git a/demos/multi-template/counter-template.yaml.tmpl b/demos/multi-template/counter-template.yaml.tmpl index c37a608dc4..a84a2469fe 100644 --- a/demos/multi-template/counter-template.yaml.tmpl +++ b/demos/multi-template/counter-template.yaml.tmpl @@ -26,7 +26,7 @@ workerSelector: workload: multi-template-shared containers: - name: counter - image: ko://github.com/agent-substrate/substrate/demos/counter + image: ko://go.ate.dev/substrate/demos/counter # Sandbox size. The pool sets no worker limits, so the worker offers the # node's allocatable and these only need to fit the node. resources: diff --git a/demos/multi-template/fspersist-template.yaml.tmpl b/demos/multi-template/fspersist-template.yaml.tmpl index 8701ca002d..9ceb6881b2 100644 --- a/demos/multi-template/fspersist-template.yaml.tmpl +++ b/demos/multi-template/fspersist-template.yaml.tmpl @@ -26,7 +26,7 @@ workerSelector: workload: multi-template-shared containers: - name: fspersist - image: ko://github.com/agent-substrate/substrate/demos/multi-template/fspersist + image: ko://go.ate.dev/substrate/demos/multi-template/fspersist # Sandbox size. The pool sets no worker limits, so the worker offers the # node's allocatable and these only need to fit the node. resources: diff --git a/demos/multi-template/multi-template.yaml.tmpl b/demos/multi-template/multi-template.yaml.tmpl index d24aa04946..b4465c20f5 100644 --- a/demos/multi-template/multi-template.yaml.tmpl +++ b/demos/multi-template/multi-template.yaml.tmpl @@ -36,6 +36,6 @@ metadata: workload: multi-template-shared spec: replicas: 3 - workerImage: ko://github.com/agent-substrate/substrate/cmd/ateworker-gvisor + workerImage: ko://go.ate.dev/substrate/cmd/ateworker-gvisor sandboxClasses: - name: gvisor diff --git a/demos/parking/parking-template.yaml.tmpl b/demos/parking/parking-template.yaml.tmpl index e926fd9e21..eac6c76796 100644 --- a/demos/parking/parking-template.yaml.tmpl +++ b/demos/parking/parking-template.yaml.tmpl @@ -24,7 +24,7 @@ workerSelector: workload: parking containers: - name: counter - image: ko://github.com/agent-substrate/substrate/demos/counter + image: ko://go.ate.dev/substrate/demos/counter command: ["/ko-app/counter"] # Sandbox size. The pool sets no worker limits, so the worker offers the # node's allocatable and these only need to fit the node. diff --git a/demos/parking/parking.yaml.tmpl b/demos/parking/parking.yaml.tmpl index 9c908f093e..16a01db8ca 100644 --- a/demos/parking/parking.yaml.tmpl +++ b/demos/parking/parking.yaml.tmpl @@ -43,6 +43,6 @@ spec: # Intentionally small: 2 workers, so 3+ concurrently-active actors saturate # the pool and exercise the parking path. replicas: 2 - workerImage: ko://github.com/agent-substrate/substrate/cmd/ateworker-gvisor + workerImage: ko://go.ate.dev/substrate/cmd/ateworker-gvisor sandboxClasses: - name: gvisor diff --git a/demos/sandbox/client/main.go b/demos/sandbox/client/main.go index 65a8cfc953..05dfdf3ed5 100644 --- a/demos/sandbox/client/main.go +++ b/demos/sandbox/client/main.go @@ -28,10 +28,10 @@ import ( "strings" "syscall" - "github.com/agent-substrate/substrate/internal/atenet" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/spf13/pflag" + "go.ate.dev/substrate/internal/atenet" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "google.golang.org/grpc/credentials" ) diff --git a/demos/sandbox/manual-test-multi-template.yaml b/demos/sandbox/manual-test-multi-template.yaml index 3adaa45c92..3811c37684 100644 --- a/demos/sandbox/manual-test-multi-template.yaml +++ b/demos/sandbox/manual-test-multi-template.yaml @@ -24,7 +24,7 @@ workerSelector: workload: sandbox containers: - name: sandbox - image: ko://github.com/agent-substrate/substrate/demos/sandbox + image: ko://go.ate.dev/substrate/demos/sandbox command: ["/ko-app/sandbox"] env: - name: PORT diff --git a/demos/sandbox/manual-test-multi.yaml b/demos/sandbox/manual-test-multi.yaml index 3b19ff5213..5dbb69627b 100644 --- a/demos/sandbox/manual-test-multi.yaml +++ b/demos/sandbox/manual-test-multi.yaml @@ -30,6 +30,6 @@ metadata: workload: sandbox spec: replicas: 1 - workerImage: ko://github.com/agent-substrate/substrate/cmd/ateworker-gvisor + workerImage: ko://go.ate.dev/substrate/cmd/ateworker-gvisor sandboxClasses: - name: gvisor diff --git a/demos/sandbox/sandbox-template.yaml.tmpl b/demos/sandbox/sandbox-template.yaml.tmpl index 7ab19834d9..c638376bb7 100644 --- a/demos/sandbox/sandbox-template.yaml.tmpl +++ b/demos/sandbox/sandbox-template.yaml.tmpl @@ -24,7 +24,7 @@ workerSelector: workload: sandbox containers: - name: sandbox - image: ko://github.com/agent-substrate/substrate/demos/sandbox + image: ko://go.ate.dev/substrate/demos/sandbox env: - name: PORT value: "80" diff --git a/demos/sandbox/sandbox.yaml.tmpl b/demos/sandbox/sandbox.yaml.tmpl index 6b60b6a152..d51fe8ff9b 100644 --- a/demos/sandbox/sandbox.yaml.tmpl +++ b/demos/sandbox/sandbox.yaml.tmpl @@ -30,7 +30,7 @@ metadata: workload: sandbox spec: replicas: 2 - workerImage: ko://github.com/agent-substrate/substrate/cmd/ateworker-gvisor + workerImage: ko://go.ate.dev/substrate/cmd/ateworker-gvisor sandboxClasses: - name: gvisor template: diff --git a/docs/api-guide.md b/docs/api-guide.md index 8958b99338..075bfa045b 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -11,7 +11,7 @@ The `WorkerPool` defines the pool of physical "warm" compute capacity. It manage | Field | Type | Description | | :--- | :--- | :--- | | `replicas` | `int32` | **Required.** Number of physical standby pods to maintain in the cluster. | -| `workerImage` | `string` | **Required.** The container image for the `ateworker` herder process (e.g. `ko://github.com/agent-substrate/substrate/cmd/ateworker-gvisor`). | +| `workerImage` | `string` | **Required.** The container image for the `ateworker` herder process (e.g. `ko://go.ate.dev/substrate/cmd/ateworker-gvisor`). | | `sandboxClasses` | `[]WorkerPoolSandboxClass` | **Required.** The sandbox runtime families the pool runs; exactly one entry today. Each entry has `name` (**required**, `gvisor` or `microvm`, immutable), which drives the worker pod shape (e.g. KVM device mounts, node placement), and `configRef.name` (optional), which names a cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) of the same class. Without `configRef`, the pool uses the class default (see [Class default](#class-default)). The controller records the result in `status.sandboxClasses`. The sandbox binaries an actor boots with still come from the `SandboxConfig` its `ActorTemplate` selects. | | `template` | `WorkerPoolPodTemplate` | **Optional.** Metadata, scheduling, and resource settings for worker workloads. | @@ -26,7 +26,7 @@ The `WorkerPool` defines the pool of physical "warm" compute capacity. It manage | `priorityClassName` | `string` | `spec.priorityClassName` | | `nodeAffinity` | `NodeAffinity` | `spec.affinity.nodeAffinity` | | `resources` | `ResourceRequirements` | `spec.containers[].resources` | -| `serviceAccountName` | `string` | `spec.serviceAccountName` | +| `serviceAccountName` | `string` | `spec.serviceAccountName`; defaults to `ateworker`, which the controller creates | Keys in `ate.dev/` and its subdomains (for example, `policy.ate.dev/`) are reserved for controllers and cannot be set in `template.labels` or @@ -57,7 +57,7 @@ metadata: workload: secret-agent spec: replicas: 10 - workerImage: ko://github.com/agent-substrate/substrate/cmd/ateworker-gvisor + workerImage: ko://go.ate.dev/substrate/cmd/ateworker-gvisor sandboxClasses: - name: gvisor configRef: @@ -102,7 +102,7 @@ The `ActorTemplate` defines the code, environment, and state-management policies | `sandboxConfig` | `SandboxConfig` | **Required.** The sandbox runtime selection: `sandboxClass` (**required**, `gvisor` or `microvm`) picks the runtime family this template's actors require — only `WorkerPool`s whose `sandboxClasses` entry matches are eligible — and `configName` (**required**) names the cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) object supplying the sandbox binaries. It must reference an existing config of the matching class; `CreateActorTemplate` rejects the template otherwise. | | `workerSelector` | `*Selector` | Optional. Gates which `WorkerPool`s actors from this template may use, by matching against each pool's labels (`matchLabels`). If unset, all pools are eligible (subject to the actor's own `worker_selector`). | | `snapshotConfig` | `SnapshotConfig` | **Required.** The base object-storage location snapshots are written under, the snapshot fidelity (`preferredFidelity`) every Pause and Suspend captures (`SNAPSHOT_FIDELITY_ROOTFS` requires `sandboxClass: microvm`), and automated golden snapshot creation (`goldenSnapshotConfig.mode`). See [Snapshot Storage Layout](#snapshot-storage-layout). | -| `volumes` | `[]Volume` | Optional. Volumes the containers may mount, each a `durableDir`, an `externalVolumeTemplate` (see [CSI Volumes Guide](csi-volumes.md)), or a `systemInfo` volume (see [SystemInfo Volumes](#systeminfo-volumes)). Every declared volume must be mounted by at least one container. A `microvm` template may declare several `durableDir` volumes; a `gvisor` template is limited to one. | +| `volumes` | `[]Volume` | Optional. Volumes the containers may mount, each a `durableDir`, an `externalVolumeTemplate` (see [CSI Volumes Guide](csi-volumes.md)), or a `systemInfo` volume (see [SystemInfo Volumes](#systeminfo-volumes)). Every declared volume must be mounted by at least one container. A `microvm` template may declare several `durableDir` volumes; a `gvisor` template is limited to one. A `durableDir` is created `0777`, as Kubernetes creates an emptyDir, so a container running as any user can write it. A snapshot restores it with the mode it had, so the container's own chmod survives suspend and resume. | | `resources` | `*ResourceRequirements` | Required. Declares each actor's compute size via `limits`, which must set both `cpu` and `memory` — see [Sandbox Right-Sizing](#sandbox-right-sizing-resources). Immutable, like the rest of the template. | The sandbox itself — the binaries (e.g. the gVisor `runsc` binary) and the `pauseImage` holding the sandbox's namespaces — comes from the cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) object the template names via `sandboxConfig.configName`. An actor always resolves the config from its current template — repointing the actor at another template requires the same config. @@ -402,7 +402,9 @@ a `Snapshot` with the following fields: an eligible worker whose `defaultRuntime` or `restorableRuntimes` is compatible with `sandboxRuntime` (same sandbox class, same schema version, and every attribute recorded on the snapshot present on the worker with the same - value; attributes only the worker reports are ignored). When no compatible worker has + value, except `cpu_features`, whose hexadecimal bitmask on the snapshot must be + a bitwise subset of the worker's; attributes only the worker reports are + ignored). When no compatible worker has capacity: - A durable snapshot — the actor's own, a `sourceTag` clone, or the template's golden snapshot a new actor starts from — falls back to any eligible diff --git a/docs/api-style-guide.md b/docs/api-style-guide.md index 641c110a11..c78189b57f 100644 --- a/docs/api-style-guide.md +++ b/docs/api-style-guide.md @@ -282,7 +282,7 @@ Rules: - **Immutable** fields — caller-set at creation but fixed thereafter (`atespace`, `name`, and resource-specific ones such as an actor's `source_snapshot_tag`). A request that changes one - including by omitting it, which would clear it - **must** return `INVALID_ARGUMENT` naming the field. - The embedded resource field **must** be named after the resource's own snake_case type name (e.g. `actor` for `Actor`), not a generic name like `resource` or `body`. - The request **must** have no other fields, apart from its parent's `ObjectRef` for a subresource (see section #2.4). -- Unknown fields in the request are preserved, so a client built against a newer schema does not lose data by round-tripping through an older one. +- A request carrying a field the server does not know, at any level of nesting, **must** return `INVALID_ARGUMENT` naming the field, as for every RPC. Fields only the client does not know are a different case: a client that modifies the resource it read sends them back unchanged, so an update from an older client does not clear fields a newer server set. - The resource's `atespace` and `name` identify the resource to update; they are not themselves updatable. - If the resource does not exist: return `NOT_FOUND`. - The `version` and `uid` fields in the embedded resource's `metadata` are **required** preconditions (see section #7). A request that leaves either unset is a blind write and **must** return `INVALID_ARGUMENT`. They are control fields the client echoes back as guards, not fields it edits. diff --git a/docs/architecture.md b/docs/architecture.md index bd1a0b4a87..a0f6f193bd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -4,7 +4,7 @@ NOTE: Much of this architecture is aspirational, and is not yet implemented! ## Overview -Agent Substrate is a secure-by-default agent execution runtime engineered to run millions of sandboxes with 10x higher density than standard container runtimes. The agent substrate control plane provides full lifecycle management for agent sandboxes, delivering sub-second agent resume/suspend operations, and allows heavy multiplexing of agents onto the same compute infrastructure. It supports multiple sandbox technologies including microVMs and gVisor, enabling consistent lifecycle operations for all sandbox types. +Agent Substrate is a security-hardened, zero-trust runtime for sandboxed agents, engineered to safely run millions of agents and tools with 10x higher density than standard containers. Purpose-built for the era of autonomous agents, Substrate delivers fast resume operations in less than 400ms, with more than 1500 activations per second. Agents get isolated compute, identity, networking, and storage. Sensitive credentials are kept out of reach from untrusted code. It supports multiple sandbox technologies, including microVMs and gVisor, while enabling a consistent lifecycle and management of all sandboxes. At its core, Agent Substrate maps a larger set of “actors” (applications such as agents) onto a smaller set of ready “workers”, relying on @@ -256,9 +256,9 @@ and Performance: SandboxConfigs) as Kubernetes objects allows platform teams to apply familiar RBAC, auditing, and policy enforcement to the underlying infrastructure, while workload definitions (ActorTemplates) are managed - through the substrate API instead, which authenticates callers itself - but does not yet implement authorization (see - [authentication.md](authentication.md)). + through the substrate API instead, which authenticates and authorizes + callers itself (see [authentication.md](authentication.md) and + [authorization.md](authorization.md)). ### Resource Model diff --git a/docs/authentication.md b/docs/authentication.md index 4ebc6ad08d..e7c6c0fd56 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -24,11 +24,10 @@ matches. `certificateAuthorityFile` and `discoveryTokenFile` are optional and are needed for OIDC discovery against some private Kubernetes API servers. `actorIdentityJWTProvider` identifies the provider allowed to call -`ActorIdentity.MintJWT`. Other authenticated providers can call every RPC. -Authorization and RBAC are not implemented yet, so only configure providers -whose users should have full control of the entire control plane: every -atespace, actor, actor template, egress policy, snapshot and worker in the -cluster. +`ActorIdentity.MintJWT`. What an authenticated caller may do is decided by +authorization; see [authorization.md](authorization.md). Its principal ID does +not record which provider issued the token, so only configure providers you +trust not to issue a token whose subject is another principal's ID. ## Google Cloud CLI tokens diff --git a/docs/authorization.md b/docs/authorization.md new file mode 100644 index 0000000000..6943ada4c5 --- /dev/null +++ b/docs/authorization.md @@ -0,0 +1,347 @@ +# ate-api Authorization + +> [!WARNING] +> **Do not keep `ate-client` as the bootstrap owner outside development.** +> `ate-setup` makes the shared `ate-client` ServiceAccount the default +> bootstrap owner so the demos, benchmarks and e2e tests work. Anyone who can +> mint a token for it, including every `kubectl ate` user who doesn't pass +> `--token-file`, is then a global owner. For a shared or production install, +> set [bootstrap owners](#bootstrap-owners) to your operators' own principals. + +`ate-api-server` checks every API call against an +[OpenFGA](https://openfga.dev) model before running it. OpenFGA is embedded in +`ate-api-server` and stores its data in the same PostgreSQL database. For how +callers authenticate, see [authentication.md](authentication.md). + +## How a call is checked + +1. **Authentication** turns the credential into a principal ID, or rejects it + with `Unauthenticated`. +2. **Authorization** allows the call if the principal is a bootstrap owner, + holds a role from an access policy that permits it, or is a Substrate + component with the system role it needs. Otherwise, or if the RPC has no + rule, it returns `PermissionDenied`. + +## Settings + +These are `ate-setup` settings, so the environment variable is read by +`ate-setup`. It passes them to `ate-api-server` as the +`--experimental-enable-authz` and `--authz-bootstrap-owners` flags on its +container; an install that doesn't use `ate-setup` sets those flags in its +manifest. A change rolls `ate-api-server`. + +| Setting | Environment variable | Example | +|---|---|---| +| `ateapi.authz.enabled` | `ATE_API_EXPERIMENTAL_ENABLE_AUTHZ` | `true` (the default) | +| `ateapi.authz.bootstrapOwners` | `ATE_API_AUTHZ_BOOTSTRAP_OWNERS` | `alice@example.com,bob@example.com` | + +If you don't set `ateapi.authz.bootstrapOwners`, `ate-setup` uses the shared +`ate-client` ServiceAccount (`system:serviceaccount::ate-client`), +so the demos, benchmarks and e2e tests work. See the warning at the top of +this page. + +```sh +ATE_API_EXPERIMENTAL_ENABLE_AUTHZ=false hack/install-ate.sh --deploy-ate-system # turn it off +``` + +With authorization off, access policy RPCs are still checked, so nobody can +grant themselves access while it is off and keep it once it is on. Bootstrap +owners still apply, so they can set up policies before turning it on. + +## Principals + +A principal is the ID authentication produces. Access policies name it as +`user:`; bootstrap owners list the bare ``. + +| Caller | `` | +|---|---| +| Kubernetes ServiceAccount token | `system:serviceaccount::` | +| Other JWT provider, such as Google | The token's `sub` (for Google, a numeric account ID, not the email) | +| Pod certificate (mTLS) | Its SPIFFE ID, e.g. `spiffe://cluster.local/ns/benchmarking/sa/default` | + +A denied call names the principal in the error with `:` percent-encoded +(`user:system%3Aserviceaccount%3A...`); decode it before using it. + +## Bootstrap owners + +Bootstrap owners always hold `owner` on the global scope, whatever the access +policies say. They write an install's first access policies and recover access +if those break. + +- They come from configuration, not the API. `ate-api-server` reads them at + startup and exits if authorization is on and the list is empty. Restarting + it with a new list takes effect immediately. +- Any principal `ate-api-server` can authenticate can be one. +- `ate-setup` warns whenever `ate-client` is one. +- Keep the list short. Grant day-to-day admins `owner` in the global access + policy instead, where it can be revoked through the API. + +For example, with a JWT provider whose tokens carry the user's email as `sub`: + +```sh +ATE_API_AUTHZ_BOOTSTRAP_OWNERS=alice@example.com,bob@example.com hack/install-ate.sh --deploy-ate-system +``` + +To make your Google account a bootstrap owner, use its numeric `sub`: + +```sh +gcloud auth print-identity-token | cut -d. -f2 | base64 -d 2>/dev/null | jq -r .sub # your sub +ATE_API_AUTHZ_BOOTSTRAP_OWNERS=,system:serviceaccount:ate-system:ate-client \ + hack/install-ate.sh --deploy-ate-system # keep ate-client for demos and benchmarks +``` + +Then add the Google provider to the `ate-api-authentication` ConfigMap and +restart `ate-api-server` +([authentication.md](authentication.md#google-cloud-cli-tokens)), and act as +yourself: + +```sh +gcloud auth print-identity-token > me.token +kubectl ate --token-file me.token get atespaces +``` + +## `kubectl ate` and `ate-setup` identities + +`kubectl ate` sends the token from `--token-file` (`-` for stdin). Without +one, it uses your kubeconfig to mint a token for `ate-client` (or +`ATE_CLIENT_SERVICE_ACCOUNT`) and warns that it is acting as a shared +identity. It still needs Kubernetes rights to port-forward to `ate-api-server` +unless you pass `--endpoint`. + +`ate-setup` calls the API only to deploy and delete demos and benchmarks, as +`ate-client`. If you leave `ate-client` out of the bootstrap owners, grant it +access in a policy or those steps are denied. + +## Roles + +Roles are granted on the **global** scope, which covers every atespace, or on +one **atespace**. + +| Permission | Global owner | Global viewer | Atespace owner | Atespace editor | Atespace viewer | +|---|:-:|:-:|:-:|:-:|:-:| +| Create atespaces; list all atespaces | ✅ | list only | | | | +| Get the atespace | ✅ | ✅ | ✅ | ✅ | ✅ | +| Delete the atespace | ✅ | | ✅ | | | +| Get access policies | ✅ | ✅ | ✅ (its own) | ✅ (its own) | ✅ (its own) | +| Create, update, delete access policies | ✅ | | ✅ (its own) | | | +| Get and list actors, actor templates, tags, egress policies | ✅ | ✅ | ✅ | ✅ | ✅ | +| Create, update, delete actors and actor templates | ✅ | | ✅ | ✅ | | +| Pause, resume, suspend, revert actors | ✅ | | ✅ | ✅ | | +| Create, update, delete egress policies | ✅ | | ✅ | ✅ | | +| Create, publish, delete tags | ✅ | | ✅ | ✅ | | +| Create an actor from a tag (`sourceTag`) | ✅ | | ✅ | ✅ | | +| Workers: get, list, list assignments | ✅ | ✅ | | | | +| Workers: create, update, delete, drain | ✅ | | | | | + +### System roles + +Substrate's components hold roles that access policies cannot grant, and only +through their pod certificates. `ate-api-server`'s `--controller-spiffe-id`, +`--egress-gateway-spiffe-id` and `--ingress-router-spiffe-id` flags bind them, +defaulting to the install's; `atelet`'s comes from `--atelet-service-account`. + +| Role | Held by | Allows | +|---|---|---| +| `controller` | `ate-controller` | Create, get, list, update, delete and drain workers | +| `egress_gateway` | `atenet-egress` | Get actors and their egress policies; list actors across all atespaces; mint actor JWTs and certificates | +| `ingress_router` | `atenet-router` | Resume actors; list actors across all atespaces | +| `atelet` | `atelet` | Get actors' egress policies, for the worker that enforces them | + +Only `egress_gateway` can mint actor credentials: not even a global owner can, +since a credential lets its holder act as the actor. + +## Checks per RPC + +A gRPC interceptor makes every check before the handler runs +([`registry.go`](../cmd/ateapi/internal/authz/registry.go), +[`model.fga`](../cmd/ateapi/internal/authz/model.fga)). When an RPC lists +several checks, all must pass. + +Roles inherit: an owner is also an editor and a viewer, an editor is also a +viewer, and a global role holds the same role in every atespace. So +"atespace editor" below also means atespace owner, global owner, and so on. + +### Atespaces and access policies + +| RPC | Permission, on | Allowed for | +|---|---|---| +| `CreateAtespace` | `can_create_atespace`, global | Global owner | +| `ListAtespaces` | `can_list_atespaces`, global | Global viewer | +| `GetAtespace` | `can_get`, the atespace | Atespace viewer | +| `DeleteAtespace` | `can_delete`, the atespace | Atespace owner | +| `GetGlobalAccessPolicy` | `can_get_access_policy`, global | Global viewer | +| `CreateGlobalAccessPolicy`, `UpdateGlobalAccessPolicy` | `can_create_access_policy` / `can_update_access_policy`, global | Global owner | +| `GetAtespaceAccessPolicy` | `can_get_access_policy`, the atespace | Atespace viewer | +| `CreateAtespaceAccessPolicy`, `UpdateAtespaceAccessPolicy`, `DeleteAtespaceAccessPolicy` | `can_create_access_policy` / `can_update_access_policy` / `can_delete_access_policy`, the atespace | Atespace owner | + +The access policy RPCs are checked even with authorization off. + +### Actor templates and actors + +| RPC | Permission, on | Allowed for | +|---|---|---| +| `CreateActorTemplate` | `can_create_actor_template`, `metadata.atespace` | Atespace editor | +| `GetActorTemplate` | `can_get`, the template | Viewer of its atespace | +| `DeleteActorTemplate` | `can_delete`, the template | Editor of its atespace | +| `ListActorTemplates` | `can_list_actor_templates`, the atespace, or global if none is given | Atespace viewer; global viewer for all atespaces | +| `CreateActor` | 1. `can_create_actor`, the actor's atespace
2. `can_use_template`, the actor template
3. `can_use_tag`, the `sourceTag`, if one is given | 1. Atespace editor
2. Viewer of the template's atespace, which may be another one
3. Editor of the tag's atespace. Golden tags skip this check: the handler only accepts one built from the actor's own template. | +| `UpdateActor` | 1. `can_update`, the actor
2. `can_use_template`, the template in the request | 1. Editor of its atespace
2. Viewer of the template's atespace, so an actor can't be switched to a template its editor can't use | +| `GetActor` | `can_get`, the actor | Viewer of its atespace; `atenet-egress` | +| `DeleteActor` | `can_delete`, the actor | Editor of its atespace | +| `ListActors` | `can_list_actors`, the atespace, or global if none is given | Atespace viewer; for all atespaces, global viewer, `atenet-router` or `atenet-egress` | +| `PauseActor`, `SuspendActor`, `RevertActor` | `can_pause` / `can_suspend` / `can_revert`, the actor | Editor of its atespace | +| `ResumeActor` | `can_resume`, the actor | Editor of its atespace; `atenet-router` | +| `GetActorEgressPolicy` | `can_get_egress_policy`, the actor | Viewer of its atespace; `atenet-egress`; `atelet` | +| `CreateActorEgressPolicy`, `UpdateActorEgressPolicy`, `DeleteActorEgressPolicy` | `can_create_egress_policy` / `can_update_egress_policy` / `can_delete_egress_policy`, the actor | Editor of its atespace | +| `MintActorJwt`, `MintActorCertificate` | `can_mint_actor_jwt` / `can_mint_actor_certificate`, the actor | `atenet-egress` only | + +### Tags + +| RPC | Permission, on | Allowed for | +|---|---|---| +| `CreateTag` | `can_create_tag`, the source actor's atespace, where the tag is stored | Atespace editor | +| `GetTag` | `can_get`, the tag | Viewer of its atespace | +| `UpdateTag` (including publishing) | `can_update`, the tag | Editor of its atespace | +| `DeleteTag` | `can_delete`, the tag | Editor of its atespace | +| `ListTags` | `can_list_tags`, the atespace, or global if none is given | Atespace viewer; global viewer for all atespaces | + +### Workers + +Workers have no atespace; their permissions come from the global scope. + +| RPC | Permission, on | Allowed for | +|---|---|---| +| `CreateWorker` | `can_create_worker`, global | Global owner; `ate-controller` | +| `ListWorkers` | `can_list_workers`, global | Global viewer; `ate-controller` | +| `GetWorker` | `can_get`, the worker | Global viewer; `ate-controller` | +| `UpdateWorker`, `MarkWorkerAvailable`, `MarkWorkerUnavailable` | `can_update`, the worker | Global owner; `ate-controller` | +| `DeleteWorker` | `can_delete`, the worker | Global owner; `ate-controller` | +| `DrainWorker` | `can_drain`, the worker | Global owner; `ate-controller` | +| `ListWorkerActorAssignments` | `can_list_actor_assignments`, the worker | Global viewer | + +### WorkerService + +The interceptor skips these; each handler requires `atelet`'s pod +certificate. + +| RPC | The caller must be | +|---|---| +| `RegisterWorker` | `atelet` on the worker's node | +| `RequestActorSuspend` | `atelet` on the node of the worker hosting the actor | +| `MintWorkerActorJwt` | `atelet` on the node hosting the actor | +| `MintWorkerActorCertificate` | `atelet` on the node of the worker the actor is assigned to | + +`ReportActorSnapshotState` has a rule but is not served yet. + +### Every RPC + +With authorization on: + +- An RPC with no rule is denied, and a test fails the build if a served RPC + has none. +- An unauthenticated caller gets `Unauthenticated` before its request is read. +- A missing or invalid resource name returns `InvalidArgument`; no check is + skipped. + +## Managing access + +Each scope has one access policy, named `default`, managed with `kubectl ate` +using `--global` or `-a `: + +```yaml +# policy.yaml +metadata: + name: default +bindings: +- role: editor + members: + - user:system:serviceaccount:team-a:alice +- role: viewer + members: + - user:system:serviceaccount:team-a:bob +``` + +```sh +kubectl ate --token-file me.token create access-policy -a team-a -f policy.yaml +kubectl ate --token-file me.token get access-policy -a team-a -o yaml > current.yaml +kubectl ate --token-file me.token update access-policy -a team-a -f current.yaml +kubectl ate --token-file me.token delete access-policy -a team-a +``` + +- Global policies accept `owner` and `viewer`; atespace policies also accept + `editor`. +- `update` replaces the whole policy and needs the `uid` and `version` from + `get -o yaml`. It fails if the policy changed, or was deleted and recreated, + since. +- The global policy can't be deleted; update it with no bindings instead. +- Deleting an atespace deletes its policy, so a recreated atespace starts + with none. +- Changes take effect on the next call. + +## Recovering access + +If no principal anyone can authenticate as holds `owner` (for example, a +Google account listed by email instead of its numeric `sub`), anyone who can +redeploy `ate-api-server` can recover: redeploy with a bootstrap owner they +can use, fix the policies, then redeploy with the intended owners. + +```sh +ATE_API_AUTHZ_BOOTSTRAP_OWNERS=alice@example.com hack/install-ate.sh --deploy-ate-system +``` + +## Production readiness + +For a shared cluster, beyond replacing the default bootstrap owner: + +- **Give people their own identities** in access policies, and have them pass + `--token-file` with a token from a trusted JWT provider or their own + ServiceAccount. To let a user mint tokens only for their own ServiceAccount: + + ```yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: {name: mint-alice-token, namespace: team-a} + rules: + - apiGroups: [""] + resources: ["serviceaccounts/token"] + resourceNames: ["alice"] + verbs: ["create"] + ``` + + ```sh + kubectl create token -n team-a alice --audience api.ate-system.svc > alice.token + kubectl ate --token-file alice.token get actors -a team-a + ``` + +- **Limit who can act as a ServiceAccount bootstrap owner.** Anyone who can + create `serviceaccounts/token`, pods or Secrets in its namespace, or change + RBAC there, can get its token. A + `kubernetes.io/service-account-token` Secret gets one that never expires. + Check who can, and alert on the audit events: + + ```sh + kubectl auth can-i create serviceaccounts/ate-client --subresource=token -n ate-system --as alice@example.com + ``` + +### Not yet supported + +- **Signing in.** `kubectl ate` has no login flow or credential plugin; people + obtain tokens themselves. +- **JWT providers at install time.** `ate-setup` configures only the cluster's + ServiceAccount issuer; add others to the ConfigMap afterwards. +- **Groups.** Policy members are individual principals. + +## Known limitations + +- **Tags can copy an actor's data.** A tag is a durable copy of an actor's + memory and disk, including secrets, and outlives the actor. Any atespace + editor can tag, publish and clone any actor in the atespace. +- **Principals are not qualified by issuer.** If `ate-api-server` trusts more + than one JWT provider, any of them that can issue a token whose `sub` is + another principal's ID, a bootstrap owner's included, acts as that principal. + System roles are unaffected: they need the component's pod certificate. +- **The atespace is the only boundary.** There are no per-actor or + per-template grants. +- **Any atelet can read any actor's egress policy.** Every node's atelet + shares one identity, so the `atelet` role isn't limited to actors on its + node ([#1564](https://github.com/agent-substrate/substrate/issues/1564)). diff --git a/docs/csi-volumes.md b/docs/csi-volumes.md index 59b77af85f..9997644c4e 100644 --- a/docs/csi-volumes.md +++ b/docs/csi-volumes.md @@ -161,7 +161,7 @@ metadata: workload: stateful-agent spec: replicas: 5 - workerImage: ko://github.com/agent-substrate/substrate/cmd/ateworker-gvisor + workerImage: ko://go.ate.dev/substrate/cmd/ateworker-gvisor sandboxClasses: - name: gvisor ``` diff --git a/docs/dev/best-practices/tracing.md b/docs/dev/best-practices/tracing.md index 3f9ffd9eee..a13aa2a9a9 100644 --- a/docs/dev/best-practices/tracing.md +++ b/docs/dev/best-practices/tracing.md @@ -82,7 +82,7 @@ address for whichever environment it is deployed to: ```yaml containers: - name: ateapi - image: ko://github.com/agent-substrate/substrate/cmd/ateapi + image: ko://go.ate.dev/substrate/cmd/ateapi ports: - containerPort: 443 # Supplies OTEL_EXPORTER_OTLP_ENDPOINT (and, on kind, the metric diff --git a/docs/dev/code-layout.md b/docs/dev/code-layout.md index a527f0aa7f..b06ce2f72e 100644 --- a/docs/dev/code-layout.md +++ b/docs/dev/code-layout.md @@ -26,12 +26,12 @@ substrate/ **`pkg/`** is for Go packages with an external API contract — code that users or third-party tools may import directly from -`github.com/agent-substrate/substrate/pkg/...`. Putting a package here signals +`go.ate.dev/substrate/pkg/...`. Putting a package here signals a commitment to backwards-compatibility and discoverability. **`internal/`** is for Go packages that are shared across multiple binaries within this module but are not part of any external API. The Go toolchain -enforces that nothing outside `github.com/agent-substrate/substrate` can import +enforces that nothing outside `go.ate.dev/substrate` can import these. The exception is [`internal/plugins/`](../../internal/plugins/README.md): self-contained plugins, each a Go module of its own, in a temporary location that nothing else in the repo should depend on or import. diff --git a/docs/metrics/registry/metrics.yaml b/docs/metrics/registry/metrics.yaml index 3d4a2f0f32..850cb3d1e0 100644 --- a/docs/metrics/registry/metrics.yaml +++ b/docs/metrics/registry/metrics.yaml @@ -150,11 +150,7 @@ groups: - id: suspend stability: development value: suspend - brief: ateapi wrote the durable snapshot and released the worker. - - id: pause - stability: development - value: pause - brief: ateapi wrote a node-local snapshot. + brief: ateapi wrote a snapshot, durable or node-local, and released the worker. - id: delete stability: development value: delete @@ -167,9 +163,9 @@ groups: stability: development value: unknown brief: > - No lifecycle operation is running, such as when a running or - paused actor crashes because its worker is lost, or the - operation is not in this list. This is the default value. + No lifecycle operation is running, such as when a running actor + crashes because its worker is lost, or the operation is not in + this list. This is the default value. - id: registry.ate.template type: attribute_group @@ -817,14 +813,14 @@ groups: conditionally_required: > The operation is a create, or ateapi loaded the actor. Thus this key is absent when a delete fails or when the load step fails. For a - resume, a suspend and a pause, the load step also reads the + resume and a suspend, the load step also reads the ActorTemplate. - ref: ate.template.name requirement_level: conditionally_required: > The operation is a create, or ateapi loaded the actor. Thus this key is absent when a delete fails or when the load step fails. For a - resume, a suspend and a pause, the load step also reads the + resume and a suspend, the load step also reads the ActorTemplate. - ref: ate.workerpool.namespace requirement_level: @@ -835,7 +831,7 @@ groups: - ref: ate.sandbox.class requirement_level: conditionally_required: > - The operation is resume, suspend, pause or revert, and ateapi found + The operation is resume, suspend or revert, and ateapi found the ActorTemplate. A revert continues when the ActorTemplate no longer exists, and then omits this key. - ref: ate.snapshot.kind @@ -843,7 +839,7 @@ groups: conditionally_required: The operation is a resume and the snapshot source is known. Other operations do not restore, thus this key is absent. - ref: ate.snapshot.fidelity requirement_level: - conditionally_required: The operation is resume, suspend or pause, and the fidelity is known. + conditionally_required: The operation is resume or suspend, and the fidelity is known. - ref: error.type requirement_level: conditionally_required: The operation failed. If this key is absent, the operation was correct. diff --git a/docs/metrics/substrate.yaml b/docs/metrics/substrate.yaml index 6eb455f2a9..a7e229ddf4 100644 --- a/docs/metrics/substrate.yaml +++ b/docs/metrics/substrate.yaml @@ -229,6 +229,14 @@ blind_spots: - area: podcertcontroller code: cmd/podcertcontroller brief: This component has no metrics. + - area: egress gateway access log + code: manifests/ate-install/atenet-egress.yaml + brief: > + The MITM and cleartext chains log only requests with status 400 or + above, gRPC requests with a grpc-status other than OK, any response + flag, and a 1% sample of the rest. The runtime keys + access_log.min_status and access_log.sample on the admin port restore + full logging while you debug. - area: GCP Secret Manager credential provider code: internal/plugins/gcp-secret-manager brief: > diff --git a/docs/observability.md b/docs/observability.md index a62fc38be1..ce6ad10fe0 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -173,7 +173,7 @@ ateapi's `Actor state changed` is written once per committed actor state transit `ate.actor.state` takes the `ateapipb.ActorState` values lowercased, so the log vocabulary and the state machine cannot fork. The last record for an actor's uid is the state it is in now, and its timestamp is when that state began. Query it per actor, not in aggregate: neither key is a metric label, because both only ever appear beside actor identity, which [the cardinality rules](#the-metric-registry) keep off metrics entirely. -`ate.actor.operation.name` says which operation drove the transition, which the state alone does not: an actor reaches `suspended` from a suspend and `paused` from a pause, and the two differ in whether the worker was released. +`ate.actor.operation.name` says which operation drove the transition, which the state alone does not: an actor reaches `suspended` both when it is created and when it is suspended. PauseActor and SuspendActor are one operation and are both labeled `suspend`. The record goes out after the store commit, never before, and the state is read straight off the committed record rather than named by the caller. Every state commit has a version check too, so if two writers race, the one that lost writes nothing. You will never see a state here that the store did not actually hold. @@ -222,7 +222,7 @@ Three `event.name` values, which is the OTLP LogRecord's own field rather than a A crash is its own name because an event name promises a set of attributes and a crash has a different severity and shape. There is no name per state: `ate.actor.state` already says which transition happened, so a consumer still selects on that one attribute and needs no map from a name to a state. All three names are in [`docs/metrics/registry/events.yaml`](metrics/registry/events.yaml), which `hack/verify-all.sh` checks. -The attributes are the same flat `ate.*` keys as the stdout record, so they arrive as real log attributes with no transform in front of them. Trace context is not among them: it goes on the record's own `TraceId` and `SpanId` fields, where the stdout record's top-level `trace_id`/`span_id` would be mapped to anyway. The instrumentation scope is `github.com/agent-substrate/substrate/internal/actorevent` for every actor event. Select or drop a stream by `event.name`: the lifecycle events must never be sampled, the usage samples may be. +The attributes are the same flat `ate.*` keys as the stdout record, so they arrive as real log attributes with no transform in front of them. Trace context is not among them: it goes on the record's own `TraceId` and `SpanId` fields, where the stdout record's top-level `trace_id`/`span_id` would be mapped to anyway. The instrumentation scope is `go.ate.dev/substrate/internal/actorevent` for every actor event. Select or drop a stream by `event.name`: the lifecycle events must never be sampled, the usage samples may be. **Never sample or filter the lifecycle stream.** A consumer takes the last event for an actor's uid, so one dropped record reports a stale state with no sign that anything is missing. This is the one stream where a sampling policy is a correctness bug rather than a cost trade. @@ -279,7 +279,7 @@ Agent Substrate emits foundational OpenTelemetry system and server metrics to mo | `ate.workerpool.ready_workers` | atecontroller | up/down counter | number of worker pods currently ready for a WorkerPool, from `status.readyReplicas` (labels `ate.workerpool.namespace`, `ate.workerpool.name`) | | `ate.workerpool.workers` | ateapi | up/down counter | live worker count per pool, split by state (`idle`/`partial`/`at_capacity`/`unschedulable`) and sandbox class to provide fleet capacity and saturation at a glance | -| `ate.actor.lifecycle.operation.duration` | ateapi | histogram | how long each actor operation (create/resume/suspend/pause/delete/revert) takes and whether it failed (`error.type` present = failure, absent = success); labeled by operation, template, pool (`ate.workerpool.namespace` + `ate.workerpool.name`), sandbox class, snapshot kind on resume, and snapshot fidelity on resume, suspend, and pause; already-running resume no-ops are not recorded so the histogram tracks actual activations, not router traffic | +| `ate.actor.lifecycle.operation.duration` | ateapi | histogram | how long each actor operation (create/resume/suspend/delete/revert; a pause is labeled suspend) takes and whether it failed (`error.type` present = failure, absent = success); labeled by operation, template, pool (`ate.workerpool.namespace` + `ate.workerpool.name`), sandbox class, snapshot kind on resume, and snapshot fidelity on resume and suspend; already-running resume no-ops are not recorded so the histogram tracks actual activations, not router traffic | | `ate.scheduler.assignment.duration` | ateapi | histogram | time of one attempt to assign a worker to an actor, with one record for each resume (ateapi retries an attempt that loses a race for a worker and records only the last attempt), with the outcome (`assigned` / `no_capacity` / `error`), the assigned pool (`ate.workerpool.namespace` + `ate.workerpool.name`) and sandbox class to catch scheduling latency and capacity starvation problems | | `ate.actor.restore.duration` | atelet | histogram | how long each phase of a restore takes on the worker node, which is where cold-start latency actually goes once ateapi hands off (labels `ate.snapshot.phase`, `ate.snapshot.kind`, `ate.snapshot.fidelity`, `ate.template.atespace`, `ate.template.name`, `ate.sandbox.class`) | | `ate.actor.checkpoint.duration` | atelet | histogram | the same phase breakdown for writing a snapshot, so a slow suspend can be attributed to ateworker or to the upload (same labels as the restore histogram) | diff --git a/docs/operator-install.md b/docs/operator-install.md index ad84e7cb50..08bfc93d68 100644 --- a/docs/operator-install.md +++ b/docs/operator-install.md @@ -263,6 +263,8 @@ changes it. Defaults are also shown by `ate-setup --help` and by `--help` on the owning command. A setting with no default is unset unless something supplies it. +The `ateapi.authz.*` settings are described in +[authorization.md](authorization.md). Settings for the demos, benchmarks, and e2e tests are listed with them: the [counter](../demos/counter/README.md#settings) and @@ -274,7 +276,10 @@ demos, the [benchmarks](../benchmarking/README.md#settings), and the |----------------------------------------------|-------------------------------------------------|----------------------------------------------------|-----------------------|------------------------------------------| | `actorJWT.algorithm` | `ACTOR_JWT_ALGORITHM` | `--actor-jwt-algorithm` | `ES256` | (global) | | `actorJWT.issuer` | `ACTOR_JWT_ISSUER` | `--actor-jwt-issuer` | — | (global) | +| `ateapi.authz.bootstrapOwners` | `ATE_API_AUTHZ_BOOTSTRAP_OWNERS` | `--ateapi-authz-bootstrap-owners` | `system:serviceaccount::ate-client` | (global) | +| `ateapi.authz.enabled` | `ATE_API_EXPERIMENTAL_ENABLE_AUTHZ` | `--ateapi-experimental-enable-authz` | `true` | (global) | | `ateapi.expectedJWTIssuer` | `EXPECTED_JWT_ISSUER` | `--ateapi-expected-jwt-issuer` | — | (global) | +| `ateapi.logLevel` | `ATE_API_LOG_LEVEL` | `--ateapi-log-level` | — | (global) | | `ateapi.postgres.cloudsql.gsa` | `ATE_API_POSTGRES_CLOUDSQL_GSA` | `--ateapi-postgres-cloudsql-gsa` | — | (global) | | `ateapi.postgres.cloudsql.iamAuth` | `ATE_API_POSTGRES_CLOUDSQL_IAM_AUTH` | `--ateapi-postgres-cloudsql-iam-auth` | — | (global) | | `ateapi.postgres.cloudsql.instance` | `ATE_API_POSTGRES_CLOUDSQL_INSTANCE` | `--ateapi-postgres-cloudsql-instance` | — | (global) | diff --git a/docs/roadmap.md b/docs/roadmap.md index c460bd0e0d..c9e0426df9 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -2,7 +2,9 @@ ## Overview -Agent Substrate delivers a performant, high density runtime environment for large scale agent deployments. The agent substrate control plane provides full lifecycle management for agent sandboxes, delivering sub-second agent resume/suspend operations, and allows heavy multiplexing of agents onto the same compute infrastructure. It supports multiple sandbox technologies including microVMs and gVisor. Substrate builds on sandboxed, snapshottable Pods by adding the ability to update running containers in the Pod leveraging the high scale and low latency Agent Substrate control plane. Substrate enables these containers to rapidly cycle between active and inactive lifecycle phases with fast suspend and resume from snapshots in inexpensive cloud object storage, without losing in-memory or filesystem state. This is especially critical for agentic workloads, which frequently oscillate between active (handling user input) and idle (waiting for LLMs or tool calls to complete). With Substrate, resources can be reclaimed during these idle moments for use by other workloads. This enables both greater efficiency and greater scale than using Pods alone, while still leveraging Kubernetes to provision capacity, configure networking and storage, and manage Substrate itself. +Agent Substrate is a security-hardened, zero-trust runtime for sandboxed agents, engineered to safely run millions of agents and tools with 10x higher density than standard containers. Purpose-built for the era of autonomous agents, Substrate delivers fast resume operations in less than 400ms, with more than 1500 activations per second. Agents get isolated compute, identity, networking, and storage. Sensitive credentials are kept out of reach from untrusted code. It supports multiple sandbox technologies, including microVMs and gVisor, while enabling a consistent lifecycle and management of all sandboxes. + +Substrate builds on sandboxed, snapshottable Pods by adding the ability to update running containers in the Pod leveraging the high scale and low latency Agent Substrate control plane. Substrate enables these containers to rapidly cycle between active and inactive lifecycle phases with fast suspend and resume from snapshots in inexpensive cloud object storage, without losing in-memory or filesystem state. This is especially critical for agentic workloads, which frequently oscillate between active (handling user input) and idle (waiting for LLMs or tool calls to complete). With Substrate, resources can be reclaimed during these idle moments for use by other workloads. This enables both greater efficiency and greater scale than using Pods alone, while still leveraging Kubernetes to provision capacity, configure networking and storage, and manage Substrate itself. This project is trying to move very quickly to find the right set of capabilities for the ever-changing agentic workloads market. Below are our priorities. Any efforts which are not aligned with these priorities should probably be deferred. diff --git a/docs/security.md b/docs/security.md index dbde5ec01f..afbcf3a180 100644 --- a/docs/security.md +++ b/docs/security.md @@ -1,6 +1,8 @@ # Substrate Security -Agent Substrate executes untrusted actors at scale in a multi-tenant environment where workloads are dynamic and memory states migrate continuously across physical infrastructure. All security controls must operate within Substrate's sub-100ms cold-start and resume latency budget. This document defines the high-level security principles. More detailed security risks are documented [in the threat model](threat-model.md). +Substrate delivers zero trust for agents. Every agent has an identity, but no credentials, no open network, and no way out of the sandbox. + +Agent Substrate executes untrusted actors at scale in a multi-tenant environment where workloads are dynamic and memory states migrate continuously across physical infrastructure. All security controls must operate within Substrate's cold-start and resume latency budget. This document defines the high-level security principles. More detailed security risks are documented [in the threat model](threat-model.md). ## Security Scope diff --git a/docs/threat-model.md b/docs/threat-model.md index a8da7edf18..65db8822d0 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -40,7 +40,7 @@ Substrate is an early, fast moving product. It is full of debate and subject to * **Actor:** The core compute primitive, gets scheduled to/from worker via Run for cold start and Resume for snapshot resume. * **Actor Network:** `ateworker` creates a private point-to-point veth network for the active Actor inside a worker Pod. The Actor is not served directly from the worker Pod's port 80; ingress enters through `atunnel`'s authenticated listeners: mTLS `CONNECT` on port 8443, which the router uses by default, and HTTPS on port 443. * **Actor Routing:** Requests sent to `atenet-router` identify the target Actor with `ate-target-actor: /`. Host and authority values are application metadata and do not select the Actor. -* **atenet-router:** Substrate runs Envoy with an `ext_proc` external processor to handle Actor ingress. With `--ingress-auth-mode=static-mtls`, Envoy accepts connections only on its TLS listeners and only from clients presenting a certificate that chains to the configured client CA bundle and carries one of a static list of SPIFFE IDs; any listed client may reach any Actor. The default, `deprecated-insecure`, rejects a TLS client whose certificate fails those checks but accepts one that presents none, and also serves plaintext listeners that accept any client. The ext_proc reads and validates `ate-target-actor`, calls the Substrate API to resume the Actor and obtain its current worker assignment, and selects that worker as a dynamic backend. It overwrites the routing header before forwarding. The router then connects with mTLS to `atunnel`; `atunnel` validates the router identity and authorizes the header against the Actor currently assigned to that worker. +* **atenet-router:** Substrate runs Envoy with an `ext_proc` external processor to handle Actor ingress. With `--ingress-auth-mode=static-mtls`, Envoy accepts connections only on its TLS listeners and only from clients presenting a certificate that chains to the configured client CA bundle and carries one of a static list of SPIFFE IDs; any listed client may reach any Actor. The default, `deprecated-insecure`, rejects a TLS client whose certificate fails those checks but accepts one that presents none, and also serves plaintext listeners that accept any client. The ext_proc reads and validates `ate-target-actor`, calls the Substrate API to resume the Actor and obtain its current worker assignment, and selects that worker as a dynamic backend. It overwrites the routing header before forwarding. The router then connects with mTLS to `atunnel` and accepts it only with the SPIFFE ID of the assigned worker pod's namespace and ServiceAccount, so another pod running as that ServiceAccount in that namespace can still pose as the worker; `atunnel` validates the router identity and authorizes the header against the Actor currently assigned to that worker. * **Object Storage:** Used to store actor snapshots. * **Filesystem support:** Container local filesystem is saved in snapshots, future integrations likely to include networked storage. * **Substrate Database:** PostgreSQL. On GCP, PostgreSQL may be Cloud SQL reached through the Cloud SQL Auth Proxy sidecar (see [tools/setup-gcp/cloud-sql.md](../tools/setup-gcp/cloud-sql.md)); that path adds two egress flows from ate-api-server: HTTPS to `sqladmin.googleapis.com:443` and the proxy tunnel to the instance IP on port 3307, both authenticated via IAM and encrypted. diff --git a/go.mod b/go.mod index 0c6d2892c3..23358d21b1 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/agent-substrate/substrate +module go.ate.dev/substrate go 1.27.0 @@ -63,6 +63,7 @@ require ( go.opentelemetry.io/otel/trace v1.45.0 go.opentelemetry.io/proto/otlp v1.11.0 golang.org/x/net v0.60.0 + golang.org/x/oauth2 v0.36.0 golang.org/x/sync v0.23.0 golang.org/x/sys v0.48.0 golang.org/x/time v0.15.0 @@ -243,7 +244,6 @@ require ( go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/crypto v0.57.0 // indirect golang.org/x/exp v0.0.0-20260718201538-764159d718ef // indirect - golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/term v0.46.0 // indirect golang.org/x/text v0.42.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect diff --git a/hack/create-kind-cluster.sh b/hack/create-kind-cluster.sh index 9d5a0e42e5..e908917121 100755 --- a/hack/create-kind-cluster.sh +++ b/hack/create-kind-cluster.sh @@ -29,8 +29,9 @@ if [[ $# -gt 0 ]]; then echo "Creates the kind cluster '${KIND_CLUSTER_NAME}' and a local registry container on port ${reg_port}." echo echo "Configured through the environment:" - echo " KIND_CLUSTER_NAME Name of the cluster to create (default: kind)." - echo " IP_FAMILY Address families for pods and Services: ipv4, ipv6 or dual (default: ipv4)." + echo " KIND_CLUSTER_NAME Name of the cluster to create (default: kind)." + echo " IP_FAMILY Address families for pods and Services: ipv4, ipv6 or dual (default: ipv4)." + echo " ATE_KIND_RESERVED_CPU CPU reserved for the kubelet and the system on each node (default: none)." exit 0 ;; esac @@ -124,6 +125,17 @@ kubeadmConfigPatches: serializeImagePulls: false maxParallelImagePulls: 4 EOF +# Hundreds of sandboxes per node can starve the kubelet's node heartbeat; after +# the 300 s toleration the taint manager evicts every pod on the node. Opt in to +# a CPU reservation for the kubelet and the system, e.g. ATE_KIND_RESERVED_CPU=2. +if [[ -n "${ATE_KIND_RESERVED_CPU:-}" ]]; then +cat <> "${ROOT}/bin/kind-config.yaml" + systemReserved: + cpu: "${ATE_KIND_RESERVED_CPU}" + kubeReserved: + cpu: "${ATE_KIND_RESERVED_CPU}" +EOF +fi echo "Deleting existing kind cluster '${KIND_CLUSTER_NAME}' if it exists..." "${ROOT}"/hack/kind.sh delete cluster --name "${KIND_CLUSTER_NAME}" || true diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 1438b1f8ee..2d0c3c4635 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -112,6 +112,8 @@ usage() { echo " ports; static-mtls requires the certificate and --atenet-dataplane=envoy." echo " --rollout-timeout DURATION Per-workload readiness wait timeout, kubectl-style Go duration (default: 60s)" echo " --otlp-endpoint URL Send all control plane telemetry to URL, not to the cluster default (see benchmarking/telemetry/README.md)" + echo " --ateapi-log-level LEVEL ate-api-server log level: debug, info, warn, or error" + echo " (default: keep the installed level, else info)" echo "" echo "Experiments:" echo "" @@ -147,6 +149,7 @@ usage() { echo " --create-podcertificate-controller-cas Create podcertificate controller CAs" echo " --create-api-server-env-vars Create ate-api-server env vars" echo " --create-api-authentication-config Create the default ate-api-server authentication config" + echo " --create-api-config Create the ate-api-server config file (log level from --ateapi-log-level)" echo "" echo "PostgreSQL configuration (a connection DSN or Cloud SQL instance selects" echo "an external database and skips the bundled instance):" @@ -178,6 +181,13 @@ usage() { echo " ACTOR_JWT_ISSUER iss claim of actor JWTs, verbatim. Relying parties fetch /.well-known/openid-configuration." echo " Default: https://idp..svc" echo "" + echo "Authorization configuration:" + echo "" + echo " ATE_API_EXPERIMENTAL_ENABLE_AUTHZ false turns off ate-api-server's runtime authorization. Default: true" + echo " ATE_API_AUTHZ_BOOTSTRAP_OWNERS Comma-separated principals that are always global owners." + echo " Default: kubectl-ate's, system:serviceaccount::ate-client" + echo " See docs/authorization.md." + echo "" echo "Benchmarks (see benchmarking/README.md for details and customization):" echo "" echo " --deploy-benchmarks Deploy workloads + locust load test stack" @@ -346,6 +356,14 @@ for ((i = 0; i < ${#prescan_args[@]}; i++)); do fi GLOBAL_FLAGS+=("--otlp-endpoint=${prescan_args[$((i + 1))]}") ;; + --ateapi-log-level=*) GLOBAL_FLAGS+=("${prescan_args[i]}") ;; + --ateapi-log-level) + if (( i + 1 >= ${#prescan_args[@]} )); then + echo "Error: --ateapi-log-level requires debug, info, warn, or error" >&2 + exit 1 + fi + GLOBAL_FLAGS+=("--ateapi-log-level=${prescan_args[$((i + 1))]}") + ;; --benchmark-worker-count=*) BENCHMARK_FLAGS+=("--worker-count=${prescan_args[i]#*=}") ;; --benchmark-worker-count) BENCHMARK_FLAGS+=("--worker-count=${prescan_args[i+1]:-1}") @@ -389,12 +407,14 @@ while [[ "$#" -gt 0 ]]; do --cluster-size|--ingress-auth-mode) shift ;; --experimental-additional-egress-extproc-service) shift ;; --credential-provider) shift ;; + --ateapi-log-level) shift ;; --benchmark-worker-count|--benchmark-sandbox-class|--benchmark-actor-memory) shift ;; --atenet-dataplane=*|--podcert-workers-per-signer=*|--rollout-timeout=*|--otlp-endpoint=*) ;; --fake-data-plane|--fake-data-plane=*) ;; --cluster-size=*|--cordon-control-plane|--cordon-control-plane=*|--ingress-auth-mode=*) ;; --experimental-additional-egress-extproc-service=*) ;; --credential-provider=*) ;; + --ateapi-log-level=*) ;; --benchmark-worker-count=*|--benchmark-sandbox-class=*|--benchmark-actor-memory=*) ;; --preview) shift ;; --preview=*) ;; @@ -426,6 +446,7 @@ while [[ "$#" -gt 0 ]]; do --create-podcertificate-controller-cas) ate_setup create podcertificate-controller-cas ;; --create-api-server-env-vars) ate_setup create api-server-env-vars ;; --create-api-authentication-config) ate_setup create api-authentication-config ;; + --create-api-config) ate_setup create api-config ;; # The one demo flag that is not just a demo name. --deploy-demo-counter-with-external-volume) diff --git a/hack/render-manifests.sh b/hack/render-manifests.sh index 27df6e9553..316445397b 100755 --- a/hack/render-manifests.sh +++ b/hack/render-manifests.sh @@ -62,15 +62,15 @@ helm template substrate "${CHART_DIR}" \ --set auth.mode=mtls \ --set postgres.clientCertificates.enabled=true \ --set createNamespace=true \ - --set image.registry=ko://github.com \ - --set image.repository=agent-substrate/substrate/cmd \ + --set image.registry=ko://go.ate.dev \ + --set image.repository=substrate/cmd \ --set image.tag="" \ > "${TMP_DIR}/all.yaml" helm template substrate-podcert "${ROOT}/charts/substrate-podcert" \ --namespace podcertificate-controller-system \ --set createNamespace=true \ - --set image.registry=ko://github.com \ - --set image.repository=agent-substrate/substrate/cmd \ + --set image.registry=ko://go.ate.dev \ + --set image.repository=substrate/cmd \ --set image.tag="" \ >> "${TMP_DIR}/all.yaml" diff --git a/hack/run-root-tests.sh b/hack/run-root-tests.sh index f9b354f7e9..b8629a1654 100755 --- a/hack/run-root-tests.sh +++ b/hack/run-root-tests.sh @@ -24,7 +24,7 @@ set -o errexit -o nounset -o pipefail ROOT="$(git rev-parse --show-toplevel)" cd "${ROOT}" -MARKER="github.com/agent-substrate/substrate/internal/roottest" +MARKER="go.ate.dev/substrate/internal/roottest" # Look for the marker in both in-package (TestImports) and external _test # package (XTestImports) test files. diff --git a/hack/tools/code-generator/go.mod b/hack/tools/code-generator/go.mod index a6fa2a4ae5..a66da5b8ef 100644 --- a/hack/tools/code-generator/go.mod +++ b/hack/tools/code-generator/go.mod @@ -1,4 +1,4 @@ -module github.com/agent-substrate/substrate/hack/tools/code-generator +module go.ate.dev/substrate/hack/tools/code-generator go 1.27.0 diff --git a/hack/tools/controller-gen/go.mod b/hack/tools/controller-gen/go.mod index 977f2766bf..3c6ea3b272 100644 --- a/hack/tools/controller-gen/go.mod +++ b/hack/tools/controller-gen/go.mod @@ -1,4 +1,4 @@ -module github.com/agent-substrate/substrate/hack/tools/controller-gen +module go.ate.dev/substrate/hack/tools/controller-gen go 1.27.0 diff --git a/hack/tools/go-licenses/go.mod b/hack/tools/go-licenses/go.mod index 432b1753f3..7b23d3742d 100644 --- a/hack/tools/go-licenses/go.mod +++ b/hack/tools/go-licenses/go.mod @@ -1,4 +1,4 @@ -module github.com/agent-substrate/substrate/hack/tools/go-licenses +module go.ate.dev/substrate/hack/tools/go-licenses go 1.27.0 diff --git a/hack/tools/golangci-lint/go.mod b/hack/tools/golangci-lint/go.mod index 2593d4f0b0..8f19eb233c 100644 --- a/hack/tools/golangci-lint/go.mod +++ b/hack/tools/golangci-lint/go.mod @@ -1,4 +1,4 @@ -module github.com/agent-substrate/substrate/hack/tools/golangci-lint +module go.ate.dev/substrate/hack/tools/golangci-lint go 1.27.0 diff --git a/hack/tools/gotestsum/go.mod b/hack/tools/gotestsum/go.mod index cf0ca9cfd1..f32423efa8 100644 --- a/hack/tools/gotestsum/go.mod +++ b/hack/tools/gotestsum/go.mod @@ -1,4 +1,4 @@ -module github.com/agent-substrate/substrate/hack/tools/gotestsum +module go.ate.dev/substrate/hack/tools/gotestsum go 1.27.0 diff --git a/hack/tools/kind/go.mod b/hack/tools/kind/go.mod index 9100fa9ad0..5545361cdb 100644 --- a/hack/tools/kind/go.mod +++ b/hack/tools/kind/go.mod @@ -1,4 +1,4 @@ -module github.com/agent-substrate/substrate/hack/tools/kind +module go.ate.dev/substrate/hack/tools/kind go 1.27.0 diff --git a/hack/tools/ko/go.mod b/hack/tools/ko/go.mod index bc6d2f33ec..315dd667e7 100644 --- a/hack/tools/ko/go.mod +++ b/hack/tools/ko/go.mod @@ -1,4 +1,4 @@ -module github.com/agent-substrate/substrate/hack/tools/ko +module go.ate.dev/substrate/hack/tools/ko go 1.27.0 diff --git a/hack/tools/kube-api-linter/go.mod b/hack/tools/kube-api-linter/go.mod index 24e6a02533..e62bf3e968 100644 --- a/hack/tools/kube-api-linter/go.mod +++ b/hack/tools/kube-api-linter/go.mod @@ -1,4 +1,4 @@ -module github.com/agent-substrate/substrate/hack/tools/kube-api-linter +module go.ate.dev/substrate/hack/tools/kube-api-linter go 1.27.0 diff --git a/hack/tools/protoc-gen-go-grpc/go.mod b/hack/tools/protoc-gen-go-grpc/go.mod index 468f14b303..75ec153d5a 100644 --- a/hack/tools/protoc-gen-go-grpc/go.mod +++ b/hack/tools/protoc-gen-go-grpc/go.mod @@ -1,4 +1,4 @@ -module github.com/agent-substrate/substrate/hack/tools/protoc-gen-go-grpc +module go.ate.dev/substrate/hack/tools/protoc-gen-go-grpc go 1.27.0 diff --git a/hack/tools/protoc-gen-go/go.mod b/hack/tools/protoc-gen-go/go.mod index 0953874e21..6a2e087cf7 100644 --- a/hack/tools/protoc-gen-go/go.mod +++ b/hack/tools/protoc-gen-go/go.mod @@ -1,4 +1,4 @@ -module github.com/agent-substrate/substrate/hack/tools/protoc-gen-go +module go.ate.dev/substrate/hack/tools/protoc-gen-go go 1.27.0 diff --git a/hack/tools/setup-envtest/go.mod b/hack/tools/setup-envtest/go.mod index e80f2207cd..22eb248db2 100644 --- a/hack/tools/setup-envtest/go.mod +++ b/hack/tools/setup-envtest/go.mod @@ -1,4 +1,4 @@ -module github.com/agent-substrate/substrate/hack/tools/setup-envtest +module go.ate.dev/substrate/hack/tools/setup-envtest go 1.27.0 diff --git a/hack/update/licenses.sh b/hack/update/licenses.sh index 2f2dbfde6a..d93157d835 100755 --- a/hack/update/licenses.sh +++ b/hack/update/licenses.sh @@ -81,7 +81,7 @@ for target in "${targets[@]}"; do } # Bug in go-licenses? Our repo gets included in a loop - rm -rf "${tmp_out}/github.com/agent-substrate/substrate" + rm -rf "${tmp_out}/go.ate.dev/substrate" # Merge the results into the main OUTDIR if [ "$(ls -A "${tmp_out}")" ]; then diff --git a/hack/verify/benchmarking-python.sh b/hack/verify/benchmarking-python.sh new file mode 100755 index 0000000000..7b615a94f4 --- /dev/null +++ b/hack/verify/benchmarking-python.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Runs the benchmarking Python test suites under pytest. Each suite gets a venv +# from its own requirements.txt plus pytest, and runs with the suite directory +# as the working directory because the tests import their modules by bare name. +# pytest is the runner because the nighthawk-ingress tests are plain functions, +# which unittest does not collect. + +set -o errexit -o nounset -o pipefail + +ROOT="$(git rev-parse --show-toplevel)" +cd "${ROOT}" +source hack/util/venv.sh + +# . A suite without a +# requirements.txt gets pytest alone. +SUITES=( + "benchmarking/analysis ." + "benchmarking/automation ." + "benchmarking/locust unit_tests" + "benchmarking/nighthawk-ingress tests" + "benchmarking/workloads/manifests/sweperf ." +) + +# A test that escapes its mocks must not reach a cluster or a registry with +# the developer's credentials: the commands the suites shell out to are +# stubbed to fail, and the kubernetes client finds no kubeconfig. +stubs="$(mktemp -d)" +trap 'rm -rf "${stubs}"' EXIT +for cmd in docker gcloud kubectl; do + printf '#!/bin/sh\necho "%s: refusing to run %s from a unit test" >&2\nexit 1\n' \ + "$(basename "$0")" "${cmd}" >"${stubs}/${cmd}" + chmod +x "${stubs}/${cmd}" +done +export PATH="${stubs}:${PATH}" KUBECONFIG=/nonexistent + +for entry in "${SUITES[@]}"; do + read -r dir tests <<<"${entry}" + venv="${dir}/venv" + ensure_venv "${venv}" + if [[ -f "${dir}/requirements.txt" ]]; then + venv_sync_requirements "${venv}" "${dir}/requirements.txt" + fi + "${venv}/bin/pip" install --quiet pytest + echo "Running ${dir}/${tests}" + (cd "${dir}" && "${ROOT}/${venv}/bin/python" -m pytest -q "${tests}") +done diff --git a/internal/actorevent/actorevent.go b/internal/actorevent/actorevent.go index b189583807..2a3bebbba4 100644 --- a/internal/actorevent/actorevent.go +++ b/internal/actorevent/actorevent.go @@ -39,12 +39,12 @@ import ( "go.opentelemetry.io/otel/log" "go.opentelemetry.io/otel/log/global" - "github.com/agent-substrate/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/ateattr" ) // ScopeName is the instrumentation scope of every actor event. A consumer // selects a stream by event name. -const ScopeName = "github.com/agent-substrate/substrate/internal/actorevent" +const ScopeName = "go.ate.dev/substrate/internal/actorevent" // Event is one name in the closed vocabulary. Name is the LogRecord's own event // name field, not an attribute. Body and Severity live here rather than at a diff --git a/internal/actorevent/actorevent_test.go b/internal/actorevent/actorevent_test.go index d7ba9be5ac..66fc81ad9d 100644 --- a/internal/actorevent/actorevent_test.go +++ b/internal/actorevent/actorevent_test.go @@ -29,8 +29,8 @@ import ( sdklog "go.opentelemetry.io/otel/sdk/log" sdktrace "go.opentelemetry.io/otel/sdk/trace" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/resources" ) const ( diff --git a/internal/actorlog/logger.go b/internal/actorlog/logger.go index ceddc3d254..7fdbaa77dc 100644 --- a/internal/actorlog/logger.go +++ b/internal/actorlog/logger.go @@ -38,8 +38,8 @@ import ( "go.opentelemetry.io/otel/trace" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/resources" ) // SyncedWriter wraps an io.Writer and synchronizes writes across goroutines. diff --git a/internal/actorlog/logger_test.go b/internal/actorlog/logger_test.go index c9f374f387..c8339dc3cb 100644 --- a/internal/actorlog/logger_test.go +++ b/internal/actorlog/logger_test.go @@ -24,8 +24,8 @@ import ( "go.opentelemetry.io/otel/trace" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/resources" ) const ( diff --git a/internal/ateapiauth/client.go b/internal/ateapiauth/client.go index 02d65f15d1..a6d8206cd6 100644 --- a/internal/ateapiauth/client.go +++ b/internal/ateapiauth/client.go @@ -20,8 +20,8 @@ import ( "errors" "fmt" - "github.com/agent-substrate/substrate/internal/credbundle" - "github.com/agent-substrate/substrate/internal/k8sresolver" + "go.ate.dev/substrate/internal/credbundle" + "go.ate.dev/substrate/internal/k8sresolver" "google.golang.org/grpc" "google.golang.org/grpc/credentials" "k8s.io/client-go/kubernetes" diff --git a/internal/ateattr/ateattr.go b/internal/ateattr/ateattr.go index ce27d09ee8..54e31b26e5 100644 --- a/internal/ateattr/ateattr.go +++ b/internal/ateattr/ateattr.go @@ -24,11 +24,11 @@ import ( "go.opentelemetry.io/otel/attribute" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - "github.com/agent-substrate/substrate/pkg/sandboxclass" + "go.ate.dev/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/sandboxclass" ) // Dotted ate.* matches the metric-instrument naming (atenet.*, atelet.*), not the @@ -276,7 +276,6 @@ const ( OperationCreate = "create" OperationResume = "resume" OperationSuspend = "suspend" - OperationPause = "pause" OperationDelete = "delete" OperationRevert = "revert" OperationUnknown = "unknown" @@ -287,7 +286,6 @@ var AllOperations = []string{ OperationCreate, OperationResume, OperationSuspend, - OperationPause, OperationDelete, OperationRevert, } diff --git a/internal/ateattr/ateattr_test.go b/internal/ateattr/ateattr_test.go index b57f8275e1..64773d9f1f 100644 --- a/internal/ateattr/ateattr_test.go +++ b/internal/ateattr/ateattr_test.go @@ -21,11 +21,11 @@ import ( "go.opentelemetry.io/otel/attribute" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) func toMap(kvs []attribute.KeyValue) map[attribute.Key]attribute.Value { @@ -377,8 +377,8 @@ func TestMetricLabelValues(t *testing.T) { {OperationCreate, "create"}, {OperationResume, "resume"}, {OperationSuspend, "suspend"}, - {OperationPause, "pause"}, {OperationDelete, "delete"}, + {OperationRevert, "revert"}, {OperationUnknown, "unknown"}, {ImageCacheOutcomeHit, "hit"}, @@ -721,8 +721,9 @@ func TestNormalizeOperationName(t *testing.T) { {OperationCreate, OperationCreate}, {OperationResume, OperationResume}, {OperationSuspend, OperationSuspend}, - {OperationPause, OperationPause}, + {"pause", OperationUnknown}, {OperationDelete, OperationDelete}, + {OperationRevert, OperationRevert}, {"", OperationUnknown}, {"invalid", OperationUnknown}, {"crash", OperationUnknown}, diff --git a/internal/ateclient/builder.go b/internal/ateclient/builder.go index 45268c0dcc..aaba019966 100644 --- a/internal/ateclient/builder.go +++ b/internal/ateclient/builder.go @@ -24,10 +24,10 @@ import ( "strings" "sync" - "github.com/agent-substrate/substrate/internal/clustertrustbundle" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/internal/portforward" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/clustertrustbundle" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/portforward" + "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/propagation" @@ -74,6 +74,13 @@ func clientServiceAccount() string { return installdefaults.ClientServiceAccount } +// FallbackPrincipal is the principal a client authenticates as when it is +// given no token file: the ServiceAccount it mints a token from. Every caller +// relying on the fallback is this same principal. +func FallbackPrincipal() string { + return fmt.Sprintf("system:serviceaccount:%s:%s", systemNamespace(), clientServiceAccount()) +} + // systemNamespace is the namespace the client expects ateapi to be running in. func systemNamespace() string { if ns := os.Getenv(NamespaceEnv); ns != "" { @@ -126,6 +133,13 @@ func (c *Client) Close() { } } +// WorkerService returns a WorkerService client on the same connection. Only +// atelet is permitted to call WorkerService; this exists so tests can check +// that other callers are rejected. +func (c *Client) WorkerService() ateapipb.WorkerServiceClient { + return ateapipb.NewWorkerServiceClient(c.conn) +} + // NewClient creates a new Ate API client. If endpoint is empty, it automatically // port-forwards to the ate-api-server pod in substrate's system namespace. // extraOpts are appended to the options used to dial the server. @@ -302,6 +316,10 @@ func serverTLSConfig(ctx context.Context, clientset kubernetes.Interface) (*tls. // bearerTokenDialOption attaches the configured token, or mints an ate-client // ServiceAccount token when tokenFile is empty. +// +// The minted token authenticates as FallbackPrincipal, not as whoever runs +// the client: anyone Kubernetes RBAC lets create a token for that +// ServiceAccount gets the same identity, and whatever it is granted. func bearerTokenDialOption(ctx context.Context, clientset *kubernetes.Clientset, tokenFile string) (grpc.DialOption, error) { if tokenFile == "-" { creds, err := readBearerToken(os.Stdin) diff --git a/internal/ateclient/builder_test.go b/internal/ateclient/builder_test.go index 97aeaa52b6..e6d4cff25f 100644 --- a/internal/ateclient/builder_test.go +++ b/internal/ateclient/builder_test.go @@ -29,7 +29,7 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/clustertrustbundle" + "go.ate.dev/substrate/internal/clustertrustbundle" certsv1 "k8s.io/api/certificates/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -210,3 +210,24 @@ func TestServerTLSConfigErrors(t *testing.T) { }) } } + +func TestFallbackPrincipal(t *testing.T) { + tests := []struct { + name string + namespace string + sa string + want string + }{ + {"defaults", "", "", "system:serviceaccount:ate-system:ate-client"}, + {"overridden", "custom-ns", "custom-client", "system:serviceaccount:custom-ns:custom-client"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(NamespaceEnv, tc.namespace) + t.Setenv(ClientServiceAccountEnv, tc.sa) + if got := FallbackPrincipal(); got != tc.want { + t.Errorf("FallbackPrincipal() = %q, want %q", got, tc.want) + } + }) + } +} diff --git a/internal/ateinterceptors/ateinterceptors.go b/internal/ateinterceptors/ateinterceptors.go index 679a724bc2..16717f59d5 100644 --- a/internal/ateinterceptors/ateinterceptors.go +++ b/internal/ateinterceptors/ateinterceptors.go @@ -20,8 +20,8 @@ import ( "strconv" "time" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/principal" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/principal" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" diff --git a/internal/ateinterceptors/ateinterceptors_test.go b/internal/ateinterceptors/ateinterceptors_test.go index f19d8bb4c9..c62f2cd470 100644 --- a/internal/ateinterceptors/ateinterceptors_test.go +++ b/internal/ateinterceptors/ateinterceptors_test.go @@ -24,12 +24,12 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/protoredact" - "github.com/agent-substrate/substrate/internal/serverboot" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - "github.com/agent-substrate/substrate/pkg/proto/credproviderpb" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/protoredact" + "go.ate.dev/substrate/internal/serverboot" + "go.ate.dev/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/credproviderpb" epb "google.golang.org/genproto/googleapis/rpc/errdetails" "google.golang.org/grpc" "google.golang.org/grpc/codes" diff --git a/internal/ateinterceptors/rejectunknown.go b/internal/ateinterceptors/rejectunknown.go index e930894cff..4a6fd94789 100644 --- a/internal/ateinterceptors/rejectunknown.go +++ b/internal/ateinterceptors/rejectunknown.go @@ -18,7 +18,7 @@ import ( "context" "strconv" - "github.com/agent-substrate/substrate/internal/apierror" + "go.ate.dev/substrate/internal/apierror" "google.golang.org/grpc" "google.golang.org/protobuf/encoding/protowire" "google.golang.org/protobuf/proto" diff --git a/internal/ateinterceptors/rejectunknown_test.go b/internal/ateinterceptors/rejectunknown_test.go index 30605dbf7b..87d1dcdb7e 100644 --- a/internal/ateinterceptors/rejectunknown_test.go +++ b/internal/ateinterceptors/rejectunknown_test.go @@ -24,9 +24,9 @@ import ( "google.golang.org/protobuf/proto" "k8s.io/apimachinery/pkg/util/validation/field" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // unknownField encodes a varint field that no descriptor in this binary diff --git a/internal/ateletdial/ateletdial.go b/internal/ateletdial/ateletdial.go index 7c79259342..4972b4447f 100644 --- a/internal/ateletdial/ateletdial.go +++ b/internal/ateletdial/ateletdial.go @@ -26,8 +26,8 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials" - "github.com/agent-substrate/substrate/internal/credbundle" - "github.com/agent-substrate/substrate/internal/substratex509" + "go.ate.dev/substrate/internal/credbundle" + "go.ate.dev/substrate/internal/substratex509" ) // TLSConfig authenticates this worker to atelet with its Pod certificate, and diff --git a/internal/ateletdial/ateletdial_test.go b/internal/ateletdial/ateletdial_test.go index 2be4549988..36f2dd74e5 100644 --- a/internal/ateletdial/ateletdial_test.go +++ b/internal/ateletdial/ateletdial_test.go @@ -30,7 +30,7 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/substratex509" + "go.ate.dev/substrate/internal/substratex509" ) const ateletSPIFFEID = "spiffe://cluster.local/ns/ate-system/sa/atelet" diff --git a/internal/atenet/headers.go b/internal/atenet/headers.go index ee1d640876..d43cc4d69f 100644 --- a/internal/atenet/headers.go +++ b/internal/atenet/headers.go @@ -19,7 +19,7 @@ import ( "fmt" "strings" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/resources" ) const ( diff --git a/internal/ateworker/actornet/dns/resolvconf.go b/internal/ateworker/actornet/resolvconf.go similarity index 66% rename from internal/ateworker/actornet/dns/resolvconf.go rename to internal/ateworker/actornet/resolvconf.go index 26f6fd805c..32959d3171 100644 --- a/internal/ateworker/actornet/dns/resolvconf.go +++ b/internal/ateworker/actornet/resolvconf.go @@ -12,55 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package dns answers an actor's DNS from its sandbox's gateway namespace and -// writes the resolv.conf that points the actor at it. -package dns +package actornet import ( - "bufio" "errors" "fmt" "io/fs" - "net" "os" "path/filepath" "strings" ) -// resolvConfNameservers reads nameservers from resolv.conf as "host:53". -func resolvConfNameservers(path string) ([]string, error) { - f, err := os.Open(path) - if err != nil { - return nil, fmt.Errorf("dns: reading resolv.conf: %w", err) - } - defer f.Close() - - var out []string - scanner := bufio.NewScanner(f) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if i := strings.IndexAny(line, "#;"); i >= 0 { - line = strings.TrimSpace(line[:i]) - } - rest, ok := strings.CutPrefix(line, "nameserver") - if !ok { - continue - } - address := strings.TrimSpace(rest) - if address == "" || net.ParseIP(address) == nil { - continue - } - out = append(out, net.JoinHostPort(address, "53")) - } - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("dns: reading resolv.conf: %w", err) - } - if len(out) == 0 { - return nil, fmt.Errorf("dns: %s names no usable nameserver", path) - } - return out, nil -} - // SandboxResolvConf replaces the pod's nameservers with nameserver, the // address the sandbox's DNS is served on, while preserving the pod's search // domains and options for Kubernetes DNS. @@ -84,7 +46,7 @@ func SandboxResolvConf(nameserver string, podResolvConf []byte) []byte { // os.Root confines path traversal; unlinking prevents writes through existing links. func WriteRootfsResolvConf(rootfs string, content []byte) error { if len(content) == 0 { - return fmt.Errorf("dns: refusing to write an empty resolv.conf") + return fmt.Errorf("refusing to write an empty resolv.conf") } root, err := os.OpenRoot(rootfs) if err != nil { diff --git a/internal/ateworker/actornet/dns/resolvconf_test.go b/internal/ateworker/actornet/resolvconf_test.go similarity index 73% rename from internal/ateworker/actornet/dns/resolvconf_test.go rename to internal/ateworker/actornet/resolvconf_test.go index edafe46c5c..6d664e41ca 100644 --- a/internal/ateworker/actornet/dns/resolvconf_test.go +++ b/internal/ateworker/actornet/resolvconf_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package dns +package actornet import ( "os" @@ -23,53 +23,6 @@ import ( "github.com/google/go-cmp/cmp" ) -func TestResolvConfNameservers(t *testing.T) { - for _, tc := range []struct { - name string - content string - want []string - wantErr bool - }{ - { - name: "cluster resolv.conf", - content: "search ate-system.svc.cluster.local svc.cluster.local\nnameserver 10.96.0.10\noptions ndots:5\n", - want: []string{"10.96.0.10:53"}, - }, - { - name: "several, in order", - content: "nameserver 10.96.0.10\nnameserver 8.8.8.8\n", - want: []string{"10.96.0.10:53", "8.8.8.8:53"}, - }, - { - name: "comments and blanks", - content: "# generated\n\n nameserver 10.96.0.10 # cluster\n;nameserver 1.1.1.1\n", - want: []string{"10.96.0.10:53"}, - }, - {name: "no nameserver", content: "search cluster.local\n", wantErr: true}, - {name: "unparsable address", content: "nameserver not-an-ip\n", wantErr: true}, - } { - t.Run(tc.name, func(t *testing.T) { - path := filepath.Join(t.TempDir(), "resolv.conf") - if err := os.WriteFile(path, []byte(tc.content), 0o644); err != nil { - t.Fatal(err) - } - got, err := resolvConfNameservers(path) - if tc.wantErr { - if err == nil { - t.Fatalf("resolvConfNameservers() = %v, want an error", got) - } - return - } - if err != nil { - t.Fatalf("resolvConfNameservers: %v", err) - } - if diff := cmp.Diff(tc.want, got); diff != "" { - t.Errorf("nameservers mismatch (-want +got):\n%s", diff) - } - }) - } -} - func TestSandboxResolvConf(t *testing.T) { // The shape kubelet writes into a pod. pod := "nameserver 10.96.0.10\n" + diff --git a/internal/ateworker/actornet/sandbox.go b/internal/ateworker/actornet/sandbox.go index 3e08da95bd..43161316c0 100644 --- a/internal/ateworker/actornet/sandbox.go +++ b/internal/ateworker/actornet/sandbox.go @@ -26,9 +26,9 @@ import ( "net/netip" "sync" - "github.com/agent-substrate/substrate/internal/ateworker/actornet/dns" - "github.com/agent-substrate/substrate/internal/netns" - "github.com/agent-substrate/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/atunnel/dns" + "go.ate.dev/substrate/internal/netns" + "go.ate.dev/substrate/internal/nodepath" "github.com/google/nftables" "github.com/google/nftables/binaryutil" @@ -364,7 +364,7 @@ func CleanupSandboxNetwork(network *SandboxNetwork) error { } // egressServer serves one actor's captured connections. Satisfied by -// atunnel.Egress; an interface so this package does not depend on it. +// egress.Proxy; an interface so this package does not depend on it. type egressServer interface { Bind(actorUID string) (func(context.Context, net.Listener) error, error) } diff --git a/internal/ateworker/actornet/sandbox_atunnel_linux_test.go b/internal/ateworker/actornet/sandbox_atunnel_linux_test.go index 3109a8d6da..ef0c94bb14 100644 --- a/internal/ateworker/actornet/sandbox_atunnel_linux_test.go +++ b/internal/ateworker/actornet/sandbox_atunnel_linux_test.go @@ -25,10 +25,10 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/ateworker/actornet" - "github.com/agent-substrate/substrate/internal/atunnel" - "github.com/agent-substrate/substrate/internal/netns" - "github.com/agent-substrate/substrate/internal/roottest" + "go.ate.dev/substrate/internal/ateworker/actornet" + "go.ate.dev/substrate/internal/linuxsys" + "go.ate.dev/substrate/internal/netns" + "go.ate.dev/substrate/internal/roottest" ) // Verify redirection preserves the destination, including unconfigured ports. @@ -59,7 +59,7 @@ func TestSandboxEgressReachesAtunnelOnAnyPort(t *testing.T) { return } defer c.Close() - dst, err := atunnel.TCPOriginalDestination(c) + dst, err := linuxsys.TCPOriginalDestination(c) if err != nil { t.Errorf("TCPOriginalDestination: %v", err) return @@ -164,7 +164,7 @@ func TestSandboxLinkLocalDoesNotReachAtunnel(t *testing.T) { t.Fatalf("accept allowed connection: %v", err) } defer conn.Close() - destination, err := atunnel.TCPOriginalDestination(conn) + destination, err := linuxsys.TCPOriginalDestination(conn) if err != nil { t.Fatalf("TCPOriginalDestination: %v", err) } diff --git a/internal/ateworker/actornet/sandbox_linux_test.go b/internal/ateworker/actornet/sandbox_linux_test.go index 6308c39b5a..46283a2fe9 100644 --- a/internal/ateworker/actornet/sandbox_linux_test.go +++ b/internal/ateworker/actornet/sandbox_linux_test.go @@ -29,11 +29,11 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/nodepath" "github.com/vishvananda/netlink" + "go.ate.dev/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/netns" - "github.com/agent-substrate/substrate/internal/roottest" + "go.ate.dev/substrate/internal/netns" + "go.ate.dev/substrate/internal/roottest" ) const testEgressPort = 15001 diff --git a/internal/ateworker/cgroup/actor_linux_test.go b/internal/ateworker/cgroup/actor_linux_test.go index 991470cf7e..a38ce75144 100644 --- a/internal/ateworker/cgroup/actor_linux_test.go +++ b/internal/ateworker/cgroup/actor_linux_test.go @@ -26,7 +26,7 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/roottest" + "go.ate.dev/substrate/internal/roottest" ) // A process started with the leaf's SysProcAttr is born inside the leaf. diff --git a/internal/ateworker/phaselog/phaselog.go b/internal/ateworker/phaselog/phaselog.go index 85fe959047..aa520c354f 100644 --- a/internal/ateworker/phaselog/phaselog.go +++ b/internal/ateworker/phaselog/phaselog.go @@ -24,10 +24,10 @@ import ( "log/slog" "time" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/resources" "google.golang.org/grpc/status" ) diff --git a/internal/ateworker/phaselog/phaselog_test.go b/internal/ateworker/phaselog/phaselog_test.go index 157fb821bd..d78c4dd54b 100644 --- a/internal/ateworker/phaselog/phaselog_test.go +++ b/internal/ateworker/phaselog/phaselog_test.go @@ -23,9 +23,9 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/resources" ) func phaseLogAttribution() resources.ActorAttribution { diff --git a/internal/ateworker/register/register.go b/internal/ateworker/register/register.go index 3eb8cd0a7f..582a0bac57 100644 --- a/internal/ateworker/register/register.go +++ b/internal/ateworker/register/register.go @@ -13,8 +13,8 @@ // limitations under the License. // Package register registers a worker with the control plane through the -// node-local atelet, reporting its compute capacity and the sandbox runtime -// its actors start under. +// node-local atelet, reporting its compute capacity and the sandbox runtimes +// of its pool, and provides file-watching helpers shared by worker binaries. package register import ( @@ -22,26 +22,33 @@ import ( "crypto/tls" "errors" "fmt" + "io/fs" "log/slog" + "maps" "os" "path/filepath" + "slices" "strconv" "strings" "time" - "github.com/agent-substrate/substrate/internal/ateletdial" - "github.com/agent-substrate/substrate/internal/hardware" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/workercapacity" + "go.ate.dev/substrate/internal/ateletdial" + "go.ate.dev/substrate/internal/hardware" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/sandboxruntimes" + "go.ate.dev/substrate/internal/workercapacity" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" "k8s.io/apimachinery/pkg/api/resource" - - "github.com/agent-substrate/substrate/internal/proto/ateletpb" ) const ( reportTimeout = 10 * time.Second initialReportBackoff = 500 * time.Millisecond maxReportBackoff = 30 * time.Second + resyncInterval = 5 * time.Minute ) // FromFiles combines the downward API compute limits with the worker's own @@ -68,17 +75,36 @@ func fromDir(dir string, actors int) *ateletpb.WorkerResources { } } -// defaultRuntime is the runtime actors hosted by this worker start under: its -// sandbox class on this host's hardware. -// -// TODO: Report the pool's SandboxConfig so atelet can complete the identity -// with the digests of the default version's assets and list its other -// enabled versions as restorable_runtimes. -func defaultRuntime(sandboxClass string) *ateletpb.SandboxRuntime { +// buildRuntime converts the compatibility attributes atelet wrote for one +// SandboxConfig version into a SandboxRuntime of sandboxClass. +func buildRuntime(sandboxClass string, attrs sandboxruntimes.Attributes) *ateletpb.SandboxRuntime { + entries := make([]*ateletpb.AttributeEntry, 0, len(attrs)) + for _, k := range slices.Sorted(maps.Keys(attrs)) { + entries = append(entries, &ateletpb.AttributeEntry{ + Key: k, + Value: attrs[k], + }) + } return &ateletpb.SandboxRuntime{ SandboxClass: sandboxClass, - Version: hardware.ProbeHost(), + Version: &ateletpb.VersionedSandboxCompat{ + SchemaVersion: hardware.SchemaVersionV1, + Attributes: entries, + }, + } +} + +// buildRequest builds the RegisterWorkerRequest for capacity and the runtimes +// in f under sandboxClass. +func buildRequest(capacity *ateletpb.WorkerResources, sandboxClass string, f *sandboxruntimes.File) *ateletpb.RegisterWorkerRequest { + req := &ateletpb.RegisterWorkerRequest{ + Capacity: capacity, + DefaultRuntime: buildRuntime(sandboxClass, f.DefaultRuntime), + } + for _, attrs := range f.RestorableRuntimes { + req.RestorableRuntimes = append(req.RestorableRuntimes, buildRuntime(sandboxClass, attrs)) } + return req } // cpuMemory is the Resources for those two dimensions. A zero dimension is @@ -113,7 +139,7 @@ func readLimit(path string) int64 { } // ReportConfig configures how a worker reaches the node-local atelet and what -// capacity and sandbox runtime class it registers with. +// it registers with. type ReportConfig struct { SocketPath string CredentialBundlePath string @@ -126,60 +152,146 @@ type ReportConfig struct { Actors int // SandboxClass is the class this worker implements ("gvisor", "microvm"). SandboxClass string + // RuntimesFile is the sandbox runtimes file atelet keeps for this worker + // (nodepath.WorkerSandboxRuntimesPath). + RuntimesFile string } -// Report tells the node-local atelet what this worker can supply and the -// runtime its actors start under, retrying until it is accepted or ctx ends. +// Report keeps the worker registered with the node-local atelet until ctx +// ends: its capacity, and the runtimes in its sandbox runtimes file, which it +// reads at startup and again whenever atelet rewrites it. It returns early +// only on a configuration error, which no retry would fix. // -// Retrying is what makes a single report durable: atelet only accepts once the +// Retrying is what makes a report durable: atelet only accepts once the // control plane has recorded it, and the Worker record may not exist yet when // a worker first comes up. Nothing else reports this, so giving up would leave // the Worker holding no capacity and hosting nothing. func Report(ctx context.Context, cfg ReportConfig) error { if cfg.SandboxClass == "" { - // The control plane refuses a runtime without a class, and retrying - // would not change that. - return errors.New("capacity report: SandboxClass is required") + // The control plane refuses a runtime without a class. + return errors.New("register worker: SandboxClass is required") + } + if cfg.RuntimesFile == "" { + return errors.New("register worker: RuntimesFile is required") } tlsConfig, err := ateletdial.TLSConfig(cfg.CredentialBundlePath, cfg.TrustBundlePath, cfg.AteletSPIFFEID) if err != nil { - return fmt.Errorf("capacity report: %w", err) - } - req := &ateletpb.RegisterWorkerRequest{ - Capacity: FromFiles(cfg.Actors), - DefaultRuntime: defaultRuntime(cfg.SandboxClass), + return fmt.Errorf("register worker: %w", err) } - err = retryReport(ctx, func() error { - return reportOnce(ctx, cfg.SocketPath, tlsConfig, req) - }, initialReportBackoff) - if err != nil { - return err + changes := WatchFilePath(ctx, cfg.RuntimesFile) + r := newReconciler( + cfg, + FromFiles(cfg.Actors), + func() (*sandboxruntimes.File, error) { return sandboxruntimes.Read(cfg.RuntimesFile) }, + func(ctx context.Context, req *ateletpb.RegisterWorkerRequest) error { + return reportOnce(ctx, cfg.SocketPath, tlsConfig, req) + }, + ) + return r.run(ctx, changes) +} + +// reconciler registers the worker with the runtimes in its sandbox runtimes +// file whenever they differ from the ones it last registered. +type reconciler struct { + cfg ReportConfig + capacity *ateletpb.WorkerResources + read func() (*sandboxruntimes.File, error) + register func(context.Context, *ateletpb.RegisterWorkerRequest) error + initialBackoff time.Duration + maxBackoff time.Duration + resync time.Duration + after func(time.Duration) <-chan time.Time + // registered is the last request accepted by atelet, nil until the first + // registration succeeds. + registered *ateletpb.RegisterWorkerRequest +} + +func newReconciler( + cfg ReportConfig, + capacity *ateletpb.WorkerResources, + read func() (*sandboxruntimes.File, error), + register func(context.Context, *ateletpb.RegisterWorkerRequest) error, +) *reconciler { + return &reconciler{ + cfg: cfg, + capacity: capacity, + read: read, + register: register, + initialBackoff: initialReportBackoff, + maxBackoff: maxReportBackoff, + resync: resyncInterval, + after: time.After, } - slog.InfoContext(ctx, "Registered worker capacity and sandbox runtime", - slog.Any("capacity", req.GetCapacity()), - slog.Any("default_runtime", req.GetDefaultRuntime())) - return nil } -// retryReport calls send until it succeeds or ctx ends, backing off between -// attempts. send is a parameter so the loop can be exercised without a socket -// or certificates. -func retryReport(ctx context.Context, send func() error, backoff time.Duration) error { +// run syncs at startup, on every change to the file, with backoff after a +// transient failure, and periodically at resyncInterval, until ctx ends. +// Change signals are ignored while backing off so periodic polling or rapid +// writes cannot bypass the backoff delay when atelet or ateapi is down. +func (r *reconciler) run(ctx context.Context, changes <-chan struct{}) error { + backoff := r.initialBackoff for { - err := send() - if err == nil { - return nil + wait := r.resync + onChange := changes + if r.sync(ctx) { + wait = backoff + backoff = min(backoff*2, r.maxBackoff) + onChange = nil + } else { + backoff = r.initialBackoff } - slog.WarnContext(ctx, "Retrying worker capacity report", slog.Duration("in", backoff), slog.Any("err", err)) select { case <-ctx.Done(): return ctx.Err() - case <-time.After(backoff): + case _, ok := <-onChange: + if !ok { + changes = nil + } + case <-r.after(wait): + } + } +} + +// sync reads the file and registers if needed. It reports whether the caller +// should retry with backoff. +func (r *reconciler) sync(ctx context.Context) (retry bool) { + desired, err := r.read() + if errors.Is(err, fs.ErrNotExist) { + if r.registered == nil { + slog.InfoContext(ctx, "Waiting for the sandbox runtimes file, which atelet writes; see its logs if this persists", + slog.String("path", r.cfg.RuntimesFile)) } - backoff = min(backoff*2, maxReportBackoff) + return false } + if err != nil { + slog.WarnContext(ctx, "Ignoring an unreadable sandbox runtimes file", slog.String("path", r.cfg.RuntimesFile), slog.Any("err", err)) + return false + } + if len(desired.DefaultRuntime) == 0 { + slog.WarnContext(ctx, "Ignoring a sandbox runtimes file with no default runtime", slog.String("path", r.cfg.RuntimesFile)) + return false + } + req := buildRequest(r.capacity, r.cfg.SandboxClass, desired) + if proto.Equal(req, r.registered) { + return false + } + if err := r.register(ctx, req); err != nil { + if status.Code(err) == codes.InvalidArgument { + slog.ErrorContext(ctx, "Worker registration rejected as invalid; waiting for sandbox runtimes file to change", slog.Any("err", err)) + return false + } + slog.WarnContext(ctx, "Could not register worker; will retry", slog.Any("err", err)) + return true + } + r.registered = req + slog.InfoContext(ctx, "Registered worker capacity and sandbox runtimes", + slog.Any("capacity", req.GetCapacity()), + slog.Any("default_runtime", req.GetDefaultRuntime()), + slog.Any("restorable_runtimes", req.GetRestorableRuntimes())) + return false } +// reportOnce dials atelet and sends one RegisterWorker. func reportOnce(ctx context.Context, socketPath string, tlsConfig *tls.Config, req *ateletpb.RegisterWorkerRequest) error { conn, err := ateletdial.Dial(socketPath, tlsConfig) if err != nil { diff --git a/internal/ateworker/register/register_test.go b/internal/ateworker/register/register_test.go index 43cb2690f8..ab63b289cb 100644 --- a/internal/ateworker/register/register_test.go +++ b/internal/ateworker/register/register_test.go @@ -23,12 +23,17 @@ import ( "testing" "time" + "github.com/fsnotify/fsnotify" "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "google.golang.org/protobuf/testing/protocmp" - "github.com/agent-substrate/substrate/internal/hardware" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/workercapacity" + "go.ate.dev/substrate/internal/hardware" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/sandboxruntimes" + "go.ate.dev/substrate/internal/workercapacity" ) // testActors stands in for the worker's own ceiling, which is a flag in @@ -73,53 +78,521 @@ func TestFromFilesMissing(t *testing.T) { } } -func TestDefaultRuntime(t *testing.T) { - got := defaultRuntime("gvisor") - // The class is the worker's own; the compat identity is the host's, under - // the probe's schema version. +func TestBuildRuntime(t *testing.T) { + got := buildRuntime("gvisor", sandboxruntimes.Attributes{ + hardware.AttrSandboxRuntimeHash: "sha256:abc", + hardware.AttrCPUFeatures: "3f", + hardware.AttrArchitecture: runtime.GOARCH, + }) want := &ateletpb.SandboxRuntime{ SandboxClass: "gvisor", Version: &ateletpb.VersionedSandboxCompat{ SchemaVersion: hardware.SchemaVersionV1, - Attributes: []*ateletpb.AttributeEntry{{Key: hardware.AttrArchitecture, Value: runtime.GOARCH}}, + Attributes: []*ateletpb.AttributeEntry{ + {Key: hardware.AttrArchitecture, Value: runtime.GOARCH}, + {Key: hardware.AttrCPUFeatures, Value: "3f"}, + {Key: hardware.AttrSandboxRuntimeHash, Value: "sha256:abc"}, + }, }, } if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { - t.Errorf("defaultRuntime() mismatch (-want +got):\n%s", diff) + t.Errorf("buildRuntime() mismatch (-want +got):\n%s", diff) } } -// reportSeam swaps the one-shot call out so the retry loop can be exercised -// without a socket or certificates. -func TestReportRetriesUntilAccepted(t *testing.T) { - attempts := 0 - send := func() error { - attempts++ - if attempts < 3 { - return errors.New("worker record does not exist yet") +func TestReconcilerSync(t *testing.T) { + capacity := &ateletpb.WorkerResources{Actors: testActors} + v1 := &sandboxruntimes.File{ + DefaultRuntime: sandboxruntimes.Attributes{ + hardware.AttrArchitecture: runtime.GOARCH, + hardware.AttrCPUFeatures: "3", + hardware.AttrSandboxRuntimeHash: "sha256:v1", + }, + } + v2 := &sandboxruntimes.File{ + DefaultRuntime: sandboxruntimes.Attributes{ + hardware.AttrArchitecture: runtime.GOARCH, + hardware.AttrCPUFeatures: "7", + hardware.AttrSandboxRuntimeHash: "sha256:v2", + }, + RestorableRuntimes: []sandboxruntimes.Attributes{v1.DefaultRuntime}, + } + reqV1 := buildRequest(capacity, "gvisor", v1) + reqV2 := buildRequest(capacity, "gvisor", v2) + + tests := []struct { + name string + setupFile func(t *testing.T, path string) + registered *ateletpb.RegisterWorkerRequest + registerErr error + wantRetry bool + wantCalls []*ateletpb.RegisterWorkerRequest + wantRegistered *ateletpb.RegisterWorkerRequest + }{ + { + // File not written yet: waits for fsnotify rather than spinning in backoff. + name: "file missing", + setupFile: func(*testing.T, string) {}, + }, + { + name: "corrupt JSON", + setupFile: func(t *testing.T, path string) { + if err := os.WriteFile(path, []byte("{not json"), 0o600); err != nil { + t.Fatal(err) + } + }, + }, + { + name: "empty default runtime", + setupFile: func(t *testing.T, path string) { + if _, err := sandboxruntimes.Write(path, &sandboxruntimes.File{}); err != nil { + t.Fatal(err) + } + }, + }, + { + name: "register error asks for retry and keeps prior registered state", + setupFile: func(t *testing.T, path string) { + if _, err := sandboxruntimes.Write(path, v1); err != nil { + t.Fatal(err) + } + }, + registerErr: errors.New("worker record does not exist yet"), + wantRetry: true, + }, + { + name: "InvalidArgument from register is not retried", + setupFile: func(t *testing.T, path string) { + if _, err := sandboxruntimes.Write(path, v1); err != nil { + t.Fatal(err) + } + }, + registerErr: status.Error(codes.InvalidArgument, "invalid attribute key"), + wantRetry: false, + }, + { + name: "first success registers and records request", + setupFile: func(t *testing.T, path string) { + if _, err := sandboxruntimes.Write(path, v1); err != nil { + t.Fatal(err) + } + }, + wantCalls: []*ateletpb.RegisterWorkerRequest{reqV1}, + wantRegistered: reqV1, + }, + { + name: "unchanged file is a no-op", + setupFile: func(t *testing.T, path string) { + if _, err := sandboxruntimes.Write(path, v1); err != nil { + t.Fatal(err) + } + }, + registered: reqV1, + wantRegistered: reqV1, + }, + { + name: "updated file registers new default and restorable runtimes", + setupFile: func(t *testing.T, path string) { + if _, err := sandboxruntimes.Write(path, v2); err != nil { + t.Fatal(err) + } + }, + registered: reqV1, + wantCalls: []*ateletpb.RegisterWorkerRequest{reqV2}, + wantRegistered: reqV2, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), nodepath.SandboxRuntimesFileName) + tc.setupFile(t, path) + + var calls []*ateletpb.RegisterWorkerRequest + r := newReconciler( + ReportConfig{SandboxClass: "gvisor", RuntimesFile: path}, + capacity, + func() (*sandboxruntimes.File, error) { return sandboxruntimes.Read(path) }, + func(_ context.Context, req *ateletpb.RegisterWorkerRequest) error { + if tc.registerErr != nil { + return tc.registerErr + } + calls = append(calls, req) + return nil + }, + ) + r.registered = tc.registered + + gotRetry := r.sync(context.Background()) + if gotRetry != tc.wantRetry { + t.Errorf("sync() retry = %v, want %v", gotRetry, tc.wantRetry) + } + if diff := cmp.Diff(tc.wantCalls, calls, protocmp.Transform()); diff != "" { + t.Errorf("register calls mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(tc.wantRegistered, r.registered, protocmp.Transform()); diff != "" { + t.Errorf("registered state mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestReconcilerRunBackoff(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + file := &sandboxruntimes.File{ + DefaultRuntime: sandboxruntimes.Attributes{hardware.AttrSandboxRuntimeHash: "sha256:v1"}, + } + changes := make(chan struct{}, 1) + + const ( + initial = 10 * time.Millisecond + maxB = 35 * time.Millisecond + resync = 5 * time.Minute + ) + + var ( + waits []time.Duration + attempts int + ) + r := newReconciler( + ReportConfig{SandboxClass: "gvisor", RuntimesFile: "sandbox-runtimes.json"}, + &ateletpb.WorkerResources{Actors: testActors}, + func() (*sandboxruntimes.File, error) { return file, nil }, + func(_ context.Context, _ *ateletpb.RegisterWorkerRequest) error { + attempts++ + switch attempts { + case 1, 2, 3: + // Fail three times: waits should be 10ms, 20ms, 35ms (capped at maxBackoff). + return errors.New("transient failure") + case 4: + // Succeed on attempt 4, then trigger a file update on changes. + file = &sandboxruntimes.File{ + DefaultRuntime: sandboxruntimes.Attributes{hardware.AttrSandboxRuntimeHash: "sha256:v2"}, + } + changes <- struct{}{} + return nil + case 5: + // Fail once on the updated file: backoff must have reset to initial (10ms). + return errors.New("transient failure after reset") + default: + cancel() + return nil + } + }, + ) + r.initialBackoff = initial + r.maxBackoff = maxB + r.resync = resync + r.after = func(d time.Duration) <-chan time.Time { + waits = append(waits, d) + if d == resync { + // Resync timer is armed on success; let changes wake the select instead. + return nil } - return nil + ch := make(chan time.Time, 1) + ch <- time.Now() + return ch } - if err := retryReport(context.Background(), send, time.Millisecond); err != nil { - t.Fatalf("retryReport() failed: %v", err) + + if err := r.run(ctx, changes); !errors.Is(err, context.Canceled) { + t.Fatalf("run() = %v, want context.Canceled", err) + } + wantWaits := []time.Duration{ + initial, // after attempt 1 + 2 * initial, // after attempt 2 (doubled) + maxB, // after attempt 3 (capped at maxBackoff) + resync, // after attempt 4 (succeeded; armed resync timer while changes fired) + initial, // after attempt 5 (reset following attempt 4's success) + resync, // after attempt 6 (succeeded before cancel) } - if attempts != 3 { - t.Errorf("gave up after %d attempts, want 3", attempts) + if diff := cmp.Diff(wantWaits, waits); diff != "" { + t.Errorf("timer durations mismatch (-want +got):\n%s", diff) } } -func TestReportStopsWhenContextEnds(t *testing.T) { +// A change signal that lands while run is backing off after a failed +// registration must not cut the backoff short: with ateapi down, every +// worker on the node would otherwise retry on every signal in lockstep. The +// signal is still acted on once the backoff has elapsed. +func TestReconcilerIgnoresChangesDuringBackoff(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + file := &sandboxruntimes.File{ + DefaultRuntime: sandboxruntimes.Attributes{hardware.AttrSandboxRuntimeHash: "sha256:v1"}, + } + changes := make(chan struct{}, 1) + // started reports each registration attempt as it begins; release lets + // the second one return. + started := make(chan int, 4) + release := make(chan struct{}) attempts := 0 - send := func() error { - attempts++ - if attempts == 2 { - cancel() + r := newReconciler( + ReportConfig{SandboxClass: "gvisor", RuntimesFile: "sandbox-runtimes.json"}, + &ateletpb.WorkerResources{Actors: testActors}, + func() (*sandboxruntimes.File, error) { return file, nil }, + func(context.Context, *ateletpb.RegisterWorkerRequest) error { + attempts++ + started <- attempts + if attempts == 1 { + // The change is pending before run reaches its select. + changes <- struct{}{} + return errors.New("transient failure") + } + <-release + return nil + }, + ) + // Unbuffered, so a send succeeds only while run is waiting on the timer. + backoffTimer := make(chan time.Time) + armed := make(chan time.Duration, 4) + r.after = func(d time.Duration) <-chan time.Time { + armed <- d + if d == r.initialBackoff { + return backoffTimer + } + return make(chan time.Time) // the resync timer never fires here + } + + done := make(chan error, 1) + go func() { + done <- r.run(ctx, changes) + }() + + if n := <-started; n != 1 { + t.Fatalf("first attempt = %d, want 1", n) + } + if d := <-armed; d != r.initialBackoff { + t.Fatalf("armed %v after the failed attempt, want the initial backoff %v", d, r.initialBackoff) + } + // Were run listening to changes during the backoff, it would be inside + // the second attempt now rather than receiving the timer. + select { + case n := <-started: + t.Fatalf("attempt %d started before the backoff elapsed: the change signal bypassed the backoff", n) + case backoffTimer <- time.Now(): + } + if n := <-started; n != 2 { + t.Fatalf("attempt after the backoff = %d, want 2", n) + } + close(release) + + // The pending change is consumed once the retry succeeds; the file it + // announces is what was just registered, so no third attempt follows. + for _, want := range []time.Duration{r.resync, r.resync} { + if d := <-armed; d != want { + t.Fatalf("armed %v after a successful sync, want the resync interval %v", d, want) } - return errors.New("still failing") } - if err := retryReport(ctx, send, time.Millisecond); !errors.Is(err, context.Canceled) { - t.Errorf("retryReport() = %v, want context.Canceled", err) + cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatalf("run() = %v, want context.Canceled", err) + } + if attempts != 2 { + t.Errorf("attempts = %d, want 2", attempts) + } +} + +func TestReconcilerRunResync(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var reads int + r := newReconciler( + ReportConfig{SandboxClass: "gvisor", RuntimesFile: "sandbox-runtimes.json"}, + &ateletpb.WorkerResources{Actors: testActors}, + func() (*sandboxruntimes.File, error) { + reads++ + if reads >= 2 { + cancel() + } + return &sandboxruntimes.File{ + DefaultRuntime: sandboxruntimes.Attributes{hardware.AttrSandboxRuntimeHash: "sha256:v1"}, + }, nil + }, + func(context.Context, *ateletpb.RegisterWorkerRequest) error { return nil }, + ) + var armed []time.Duration + r.after = func(d time.Duration) <-chan time.Time { + armed = append(armed, d) + if ctx.Err() != nil { + return nil // once canceled, let ctx.Done win the select + } + ch := make(chan time.Time, 1) + ch <- time.Now() + return ch + } + + // Pass a nil changes channel: only the resync timer can trigger the second read. + if err := r.run(ctx, nil); !errors.Is(err, context.Canceled) { + t.Fatalf("run() = %v, want context.Canceled", err) + } + if reads != 2 { + t.Errorf("read calls = %d, want 2 (initial sync + resync tick)", reads) + } + if len(armed) == 0 || armed[0] != resyncInterval { + t.Errorf("armed timers = %v, want first timer to be resyncInterval (%v)", armed, resyncInterval) + } +} + +func TestWatchFileFallbackToPolling(t *testing.T) { + origInterval := watchPollInterval + watchPollInterval = 5 * time.Millisecond + t.Cleanup(func() { watchPollInterval = origInterval }) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + // Point WatchFilePath at a path whose parent directory does not exist so w.Add + // fails naturally and triggers the polling fallback. + missingPath := filepath.Join(t.TempDir(), "missing-parent", nodepath.SandboxRuntimesFileName) + changes := WatchFilePath(ctx, missingPath) + + select { + case _, ok := <-changes: + if !ok { + t.Fatal("changes channel closed before emitting a polling signal") + } + case <-ctx.Done(): + t.Fatal("timed out waiting for polling fallback signal") + } + + cancel() + for range changes { + } +} + +func TestWatchFileFilterAndErrors(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dir := nodepath.WorkerPath("pod-uid") + target := nodepath.WorkerSandboxRuntimesPath("pod-uid") + events := make(chan fsnotify.Event) + errs := make(chan error) + changes := make(chan struct{}, 1) + + done := make(chan struct{}) + go func() { + defer close(done) + watchFileLoop(ctx, target, events, errs, changes) + }() + + assertNoSignal := func(t *testing.T, desc string) { + t.Helper() + select { + case <-changes: + t.Fatalf("unexpected signal after %s", desc) + case <-time.After(10 * time.Millisecond): + } + } + assertSignal := func(t *testing.T, desc string) { + t.Helper() + select { + case <-changes: + case <-time.After(2 * time.Second): + t.Fatalf("timed out waiting for signal after %s", desc) + } + } + + // Unrelated files in the same worker directory must be ignored. + events <- fsnotify.Event{Name: nodepath.WorkerSocketPath("pod-uid"), Op: fsnotify.Create} + events <- fsnotify.Event{Name: filepath.Join(dir, ".sandbox-runtimes-123456"), Op: fsnotify.Create | fsnotify.Write} + assertNoSignal(t, "events on worker.sock and temp file") + + // Non-Create/Write operations on the target file must be ignored. + events <- fsnotify.Event{Name: target, Op: fsnotify.Chmod} + events <- fsnotify.Event{Name: target, Op: fsnotify.Remove} + events <- fsnotify.Event{Name: target, Op: fsnotify.Rename} + assertNoSignal(t, "Chmod/Remove/Rename on target file") + + // Create and Write on the target file must signal. + events <- fsnotify.Event{Name: target, Op: fsnotify.Create} + assertSignal(t, "Create on target file") + events <- fsnotify.Event{Name: target, Op: fsnotify.Write} + assertSignal(t, "Write on target file") + + // An error on the watcher's Errors channel must also signal so the caller re-reads. + errs <- errors.New("fsnotify queue overflow") + assertSignal(t, "watcher error") + + cancel() + <-done +} + +func TestReconcilerRunWatchesFile(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + path := filepath.Join(t.TempDir(), nodepath.SandboxRuntimesFileName) + got := make(chan *ateletpb.RegisterWorkerRequest, 4) + r := newReconciler( + ReportConfig{SandboxClass: "gvisor", RuntimesFile: path}, + &ateletpb.WorkerResources{Actors: testActors}, + func() (*sandboxruntimes.File, error) { return sandboxruntimes.Read(path) }, + func(_ context.Context, req *ateletpb.RegisterWorkerRequest) error { + got <- req + return nil + }, + ) + + changes := WatchFilePath(ctx, path) + done := make(chan error, 1) + go func() { + done <- r.run(ctx, changes) + }() + + // Write the file after WatchFilePath has started, as happens when a worker comes up + // before atelet has reconciled the pod. + v1 := &sandboxruntimes.File{ + DefaultRuntime: sandboxruntimes.Attributes{ + hardware.AttrArchitecture: runtime.GOARCH, + hardware.AttrCPUFeatures: "3", + hardware.AttrSandboxRuntimeHash: "sha256:v1", + }, + } + if _, err := sandboxruntimes.Write(path, v1); err != nil { + t.Fatalf("writing v1: %v", err) + } + select { + case req := <-got: + if diff := cmp.Diff(buildRuntime("gvisor", v1.DefaultRuntime), req.GetDefaultRuntime(), protocmp.Transform()); diff != "" { + t.Errorf("initial registration default_runtime mismatch (-want +got):\n%s", diff) + } + case <-ctx.Done(): + t.Fatal("timed out waiting for initial registration after file creation") + } + + // Rewrite the file via atomic rename and verify the watcher triggers a + // second registration. + v2 := &sandboxruntimes.File{ + DefaultRuntime: sandboxruntimes.Attributes{ + hardware.AttrArchitecture: runtime.GOARCH, + hardware.AttrCPUFeatures: "7", + hardware.AttrSandboxRuntimeHash: "sha256:v2", + }, + RestorableRuntimes: []sandboxruntimes.Attributes{v1.DefaultRuntime}, + } + if _, err := sandboxruntimes.Write(path, v2); err != nil { + t.Fatalf("writing v2: %v", err) + } + select { + case req := <-got: + if diff := cmp.Diff(buildRuntime("gvisor", v2.DefaultRuntime), req.GetDefaultRuntime(), protocmp.Transform()); diff != "" { + t.Errorf("updated registration default_runtime mismatch (-want +got):\n%s", diff) + } + if len(req.GetRestorableRuntimes()) != 1 { + t.Fatalf("got %d restorable runtimes, want 1", len(req.GetRestorableRuntimes())) + } + case <-ctx.Done(): + t.Fatal("timed out waiting for registration after file update") + } + + cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Errorf("run() = %v, want context.Canceled", err) } } @@ -135,6 +608,7 @@ func TestReportFailsFastOnBadCredentials(t *testing.T) { CredentialBundlePath: filepath.Join(t.TempDir(), "does-not-exist.pem"), TrustBundlePath: filepath.Join(t.TempDir(), "also-missing.pem"), SandboxClass: "gvisor", + RuntimesFile: filepath.Join(t.TempDir(), nodepath.SandboxRuntimesFileName), }) if err == nil { t.Fatal("Report() with unreadable credentials succeeded, want an error the caller can exit on") @@ -144,17 +618,39 @@ func TestReportFailsFastOnBadCredentials(t *testing.T) { } } -// A runtime without a class is refused by the control plane on every attempt, -// so a worker that names none must exit rather than retry. -func TestReportFailsFastWithoutSandboxClass(t *testing.T) { +// A runtime without a class or runtimes file path is a configuration bug, so +// Report must fail immediately rather than spin. +func TestReportFailsFastOnMissingConfig(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - err := Report(ctx, ReportConfig{SocketPath: filepath.Join(t.TempDir(), "atelet.sock")}) - if err == nil { - t.Fatal("Report() without a SandboxClass succeeded, want an error the caller can exit on") - } - if errors.Is(err, context.DeadlineExceeded) { - t.Errorf("Report() retried a permanent failure until the deadline: %v", err) + for _, tc := range []struct { + name string + cfg ReportConfig + }{ + { + name: "missing SandboxClass", + cfg: ReportConfig{ + SocketPath: filepath.Join(t.TempDir(), "atelet.sock"), + RuntimesFile: filepath.Join(t.TempDir(), nodepath.SandboxRuntimesFileName), + }, + }, + { + name: "missing RuntimesFile", + cfg: ReportConfig{ + SocketPath: filepath.Join(t.TempDir(), "atelet.sock"), + SandboxClass: "gvisor", + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + err := Report(ctx, tc.cfg) + if err == nil { + t.Fatal("Report() succeeded, want an error the caller can exit on") + } + if errors.Is(err, context.DeadlineExceeded) { + t.Errorf("Report() retried a permanent failure until the deadline: %v", err) + } + }) } } diff --git a/internal/ateworker/register/watch.go b/internal/ateworker/register/watch.go new file mode 100644 index 0000000000..31d8df67ca --- /dev/null +++ b/internal/ateworker/register/watch.go @@ -0,0 +1,123 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package register + +import ( + "context" + "log/slog" + "path/filepath" + "time" + + "github.com/fsnotify/fsnotify" +) + +// watchPollInterval is how often WatchFilePath signals when fsnotify is +// unavailable (for example, when the node's inotify instance budget is +// exhausted). A var so tests can shorten it. +var watchPollInterval = 5 * time.Second + +// WatchFilePath signals on the returned channel whenever the file at path is +// created, written, or replaced, or when the underlying watcher reports an +// error (such as an event queue overflow) so the caller can re-read the file. +// It watches path's parent directory, which must exist, so atomic file +// replacements via rename are detected even when the file itself does not +// exist yet at startup. +// +// If fsnotify cannot watch the parent directory (for example, when the host +// UID's fs.inotify.max_user_instances budget is exhausted), WatchFilePath logs a +// warning and falls back to signaling periodically at watchPollInterval until +// ctx ends. The returned channel is closed when ctx ends. +func WatchFilePath(ctx context.Context, path string) <-chan struct{} { + cleanPath := filepath.Clean(path) + changes := make(chan struct{}, 1) + w, err := fsnotify.NewWatcher() + if err != nil { + slog.WarnContext(ctx, "Could not create fsnotify watcher; falling back to polling", + slog.String("path", cleanPath), slog.Duration("interval", watchPollInterval), slog.Any("err", err)) + go pollFileLoop(ctx, watchPollInterval, changes) + return changes + } + dir := filepath.Dir(cleanPath) + if err := w.Add(dir); err != nil { + _ = w.Close() + slog.WarnContext(ctx, "Could not watch directory with fsnotify; falling back to polling", + slog.String("dir", dir), slog.String("path", cleanPath), slog.Duration("interval", watchPollInterval), slog.Any("err", err)) + go pollFileLoop(ctx, watchPollInterval, changes) + return changes + } + go func() { + defer close(changes) + defer func() { + if err := w.Close(); err != nil { + slog.WarnContext(ctx, "Failed to close file watcher", slog.String("path", cleanPath), slog.Any("err", err)) + } + }() + watchFileLoop(ctx, cleanPath, w.Events, w.Errors, changes) + }() + return changes +} + +func pollFileLoop(ctx context.Context, interval time.Duration, changes chan<- struct{}) { + defer close(changes) + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + select { + case changes <- struct{}{}: + default: + } + } + } +} + +func watchFileLoop(ctx context.Context, cleanPath string, events <-chan fsnotify.Event, errs <-chan error, changes chan<- struct{}) { + for { + select { + case <-ctx.Done(): + return + case ev, ok := <-events: + if !ok { + return + } + if !isFileModifyEvent(cleanPath, ev) { + continue + } + select { + case changes <- struct{}{}: + default: // a signal is already pending + } + case err, ok := <-errs: + if !ok { + return + } + slog.WarnContext(ctx, "File watch error; signaling re-read", slog.String("path", cleanPath), slog.Any("err", err)) + select { + case changes <- struct{}{}: + default: + } + } + } +} + +func isFileModifyEvent(cleanPath string, ev fsnotify.Event) bool { + if filepath.Clean(ev.Name) != cleanPath { + return false + } + return ev.Has(fsnotify.Create) || ev.Has(fsnotify.Write) +} diff --git a/internal/ateworker/suspend/suspend.go b/internal/ateworker/suspend/suspend.go index f40bfd5960..08ac36f2c6 100644 --- a/internal/ateworker/suspend/suspend.go +++ b/internal/ateworker/suspend/suspend.go @@ -33,8 +33,8 @@ import ( "google.golang.org/grpc/status" "k8s.io/apimachinery/pkg/util/wait" - "github.com/agent-substrate/substrate/internal/ateletdial" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/ateletdial" + "go.ate.dev/substrate/internal/proto/ateletpb" ) // requestTimeout bounds one attempt. It is generous because the control plane diff --git a/internal/ateworker/suspend/suspend_test.go b/internal/ateworker/suspend/suspend_test.go index 9e5b9a7fbf..a8586a8eb5 100644 --- a/internal/ateworker/suspend/suspend_test.go +++ b/internal/ateworker/suspend/suspend_test.go @@ -32,7 +32,7 @@ import ( "google.golang.org/protobuf/testing/protocmp" "k8s.io/apimachinery/pkg/util/wait" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/proto/ateletpb" ) // capturingAtelet stands in for the node-local atelet. It records every diff --git a/internal/ateworker/tunnel/egresspolicy.go b/internal/ateworker/tunnel/egresspolicy.go index 0e5ff7e898..10d4c3ca9e 100644 --- a/internal/ateworker/tunnel/egresspolicy.go +++ b/internal/ateworker/tunnel/egresspolicy.go @@ -25,12 +25,12 @@ import ( "golang.org/x/sync/singleflight" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/ateletdial" - "github.com/agent-substrate/substrate/internal/ateworker/actornet/dns" - "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/ateletdial" + "go.ate.dev/substrate/internal/atunnel/dns" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" ) const ( diff --git a/internal/ateworker/tunnel/egresspolicy_test.go b/internal/ateworker/tunnel/egresspolicy_test.go index 33b832989f..906e62c964 100644 --- a/internal/ateworker/tunnel/egresspolicy_test.go +++ b/internal/ateworker/tunnel/egresspolicy_test.go @@ -43,10 +43,10 @@ import ( "google.golang.org/grpc/status" "google.golang.org/protobuf/testing/protocmp" - "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/substratex509" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/substratex509" ) var testActor = resources.ActorAttribution{Ref: resources.ActorRef{Atespace: "space", Name: "actor"}, UID: "uid"} diff --git a/internal/ateworker/tunnel/tunnel.go b/internal/ateworker/tunnel/tunnel.go index ecb2ceb617..142f044067 100644 --- a/internal/ateworker/tunnel/tunnel.go +++ b/internal/ateworker/tunnel/tunnel.go @@ -29,15 +29,17 @@ import ( "net/url" "time" - "github.com/agent-substrate/substrate/internal/ateletdial" - "github.com/agent-substrate/substrate/internal/ateworker/actornet/dns" - "github.com/agent-substrate/substrate/internal/atunnel" - "github.com/agent-substrate/substrate/internal/egresspolicy" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/serverboot" + "go.ate.dev/substrate/internal/ateletdial" + "go.ate.dev/substrate/internal/atunnel/dns" + "go.ate.dev/substrate/internal/atunnel/egress" + "go.ate.dev/substrate/internal/atunnel/ingress" + "go.ate.dev/substrate/internal/egresspolicy" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/linuxsys" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/serverboot" ) // resolvConfPath holds the worker pod's resolvers. Tests override it. @@ -94,9 +96,9 @@ func RegisterFlags(fs FlagSet) *Config { // Tunnel is the atunnel state one worker shares across its actors. type Tunnel struct { // Ingress proxies router traffic to the active actors. - Ingress *atunnel.Server + Ingress *ingress.Server // Egress tunnels the active actors' outbound TCP to the egress gateway. - Egress *atunnel.Egress + Egress *egress.Proxy // EgressPort is the local atunnel listener used as the target of the // actor network's transparent TCP redirect. EgressPort uint16 @@ -130,7 +132,7 @@ func Start(ctx context.Context, cfg Config, upstream string) (*Tunnel, error) { } // Ingress - ingress, err := atunnel.NewServer(atunnel.Config{ + ingressServer, err := ingress.NewServer(ingress.Config{ CredentialBundlePath: cfg.CredentialBundle, TrustBundlePath: cfg.TrustBundle, AllowedClientID: cfg.ClientIdentity, @@ -139,26 +141,26 @@ func Start(ctx context.Context, cfg Config, upstream string) (*Tunnel, error) { if err != nil { return nil, fmt.Errorf("configuring ingress: %w", err) } - if err := serve(ctx, "atunnel", cfg.ListenAddress, ingress.Serve); err != nil { + if err := serve(ctx, "atunnel", cfg.ListenAddress, ingressServer.Serve); err != nil { return nil, err } - if err := serve(ctx, "atunnel CONNECT", cfg.ConnectListenAddress, ingress.ServeConnect); err != nil { + if err := serve(ctx, "atunnel CONNECT", cfg.ConnectListenAddress, ingressServer.ServeConnect); err != nil { return nil, err } // Egress - egress, err := atunnel.NewEgress(atunnel.TCPOriginalDestination) + egressProxy, err := egress.New(linuxsys.TCPOriginalDestination) if err != nil { return nil, fmt.Errorf("configuring egress: %w", err) } - egressPort, err := atunnel.EgressPort(cfg.EgressListenAddress) + egressPort, err := egress.PortFromListenAddr(cfg.EgressListenAddress) if err != nil { return nil, err } return &Tunnel{ - Ingress: ingress, - Egress: egress, + Ingress: ingressServer, + Egress: egressProxy, EgressPort: egressPort, DNSRelay: dnsRelay, cfg: cfg, @@ -185,9 +187,9 @@ func serve(ctx context.Context, name, address string, serveFn func(context.Conte // ActorEgress is an actor's tunneled egress, ready to activate. type ActorEgress struct { // client presents the actor certificate to the remote egress gateway. - client *atunnel.Client + client *egress.Client // certificateSource owns the actor key and renews its certificate via atelet. - certificateSource *atunnel.BrokerCertificateSource + certificateSource *egress.BrokerCertificateSource expiresAt time.Time } @@ -204,7 +206,7 @@ func (t *Tunnel) PrepareEgress(ctx context.Context, actor resources.ActorAttribu if err != nil { return nil, fmt.Errorf("invalid egress gateway address %q: %w", gateway.GetAddress(), err) } - certificateSource, err := atunnel.NewBrokerCertificateSource(atunnel.BrokerConfig{ + certificateSource, err := egress.NewBrokerCertificateSource(egress.BrokerConfig{ SocketPath: nodepath.WorkerSupportSocket, CredentialBundlePath: t.cfg.CredentialBundle, TrustBundlePath: t.cfg.TrustBundle, @@ -222,7 +224,7 @@ func (t *Tunnel) PrepareEgress(ctx context.Context, actor resources.ActorAttribu if err != nil { return nil, fmt.Errorf("while obtaining actor certificate: %w", err) } - gatewayClient, err := atunnel.NewClient(atunnel.ClientConfig{ + gatewayClient, err := egress.NewClient(egress.ClientConfig{ GatewayAddress: gateway.GetAddress(), ServerName: serverName, GetClientCertificate: certificateSource.GetClientCertificate, @@ -236,14 +238,14 @@ func (t *Tunnel) PrepareEgress(ctx context.Context, actor resources.ActorAttribu // Activate starts admitting the actor's traffic. Ingress reaches the actor // through dial; egress is activated only when it was prepared. -func (t *Tunnel) Activate(actor resources.ActorAttribution, dial atunnel.DialFunc, egress *ActorEgress) error { +func (t *Tunnel) Activate(actor resources.ActorAttribution, dial ingress.DialFunc, actorEgress *ActorEgress) error { if err := t.Ingress.Activate(actor.Ref.Atespace, actor.Ref.Name, actor.UID, dial); err != nil { return fmt.Errorf("while activating actor ingress: %w", err) } - if egress == nil { + if actorEgress == nil { return nil } - if err := t.Egress.Activate(actor.UID, egress.client, egress.certificateSource, egress.expiresAt); err != nil { + if err := t.Egress.Activate(actor.UID, actorEgress.client, actorEgress.certificateSource, actorEgress.expiresAt); err != nil { return fmt.Errorf("while activating actor egress: %w", err) } return nil diff --git a/internal/ateworker/tunnel/tunnel_test.go b/internal/ateworker/tunnel/tunnel_test.go index 4fbbe75f65..022ee9dc0e 100644 --- a/internal/ateworker/tunnel/tunnel_test.go +++ b/internal/ateworker/tunnel/tunnel_test.go @@ -33,11 +33,11 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/egresspolicy" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/substratex509" "github.com/spf13/pflag" + "go.ate.dev/substrate/internal/egresspolicy" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/substratex509" ) func TestRegisterFlags(t *testing.T) { diff --git a/internal/ateworker/actornet/dns/dnshandler.go b/internal/atunnel/dns/dnshandler.go similarity index 99% rename from internal/ateworker/actornet/dns/dnshandler.go rename to internal/atunnel/dns/dnshandler.go index 665e5a30b0..c0d6178b50 100644 --- a/internal/ateworker/actornet/dns/dnshandler.go +++ b/internal/atunnel/dns/dnshandler.go @@ -21,7 +21,7 @@ import ( "net" "time" - "github.com/agent-substrate/substrate/internal/ateworker/actornet/dns/protocol" + "go.ate.dev/substrate/internal/atunnel/dns/protocol" "golang.org/x/net/dns/dnsmessage" ) diff --git a/internal/ateworker/actornet/dns/dnshandler_test.go b/internal/atunnel/dns/dnshandler_test.go similarity index 99% rename from internal/ateworker/actornet/dns/dnshandler_test.go rename to internal/atunnel/dns/dnshandler_test.go index 1551212e2a..028d4010d6 100644 --- a/internal/ateworker/actornet/dns/dnshandler_test.go +++ b/internal/atunnel/dns/dnshandler_test.go @@ -20,7 +20,7 @@ import ( "net" "testing" - "github.com/agent-substrate/substrate/internal/ateworker/actornet/dns/protocol" + "go.ate.dev/substrate/internal/atunnel/dns/protocol" "golang.org/x/net/dns/dnsmessage" ) diff --git a/internal/ateworker/actornet/dns/evaluator.go b/internal/atunnel/dns/evaluator.go similarity index 94% rename from internal/ateworker/actornet/dns/evaluator.go rename to internal/atunnel/dns/evaluator.go index e85acf5319..cd79a428ea 100644 --- a/internal/ateworker/actornet/dns/evaluator.go +++ b/internal/atunnel/dns/evaluator.go @@ -15,8 +15,8 @@ package dns import ( - "github.com/agent-substrate/substrate/internal/egresspolicy" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/egresspolicy" + "go.ate.dev/substrate/internal/proto/ateletpb" ) // evaluator decides whether a DNS hostname is allowed by an EgressPolicy's diff --git a/internal/ateworker/actornet/dns/evaluator_test.go b/internal/atunnel/dns/evaluator_test.go similarity index 98% rename from internal/ateworker/actornet/dns/evaluator_test.go rename to internal/atunnel/dns/evaluator_test.go index d1e600ea3c..0b1604e2ea 100644 --- a/internal/ateworker/actornet/dns/evaluator_test.go +++ b/internal/atunnel/dns/evaluator_test.go @@ -17,7 +17,7 @@ package dns import ( "testing" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/proto/ateletpb" ) func TestEvaluator(t *testing.T) { diff --git a/internal/ateworker/actornet/dns/freemap/doc.go b/internal/atunnel/dns/freemap/doc.go similarity index 100% rename from internal/ateworker/actornet/dns/freemap/doc.go rename to internal/atunnel/dns/freemap/doc.go diff --git a/internal/ateworker/actornet/dns/freemap/freemap64k.go b/internal/atunnel/dns/freemap/freemap64k.go similarity index 100% rename from internal/ateworker/actornet/dns/freemap/freemap64k.go rename to internal/atunnel/dns/freemap/freemap64k.go diff --git a/internal/ateworker/actornet/dns/freemap/freemap64k_test.go b/internal/atunnel/dns/freemap/freemap64k_test.go similarity index 100% rename from internal/ateworker/actornet/dns/freemap/freemap64k_test.go rename to internal/atunnel/dns/freemap/freemap64k_test.go diff --git a/internal/ateworker/actornet/dns/limiter.go b/internal/atunnel/dns/limiter.go similarity index 100% rename from internal/ateworker/actornet/dns/limiter.go rename to internal/atunnel/dns/limiter.go diff --git a/internal/ateworker/actornet/dns/limiter_test.go b/internal/atunnel/dns/limiter_test.go similarity index 100% rename from internal/ateworker/actornet/dns/limiter_test.go rename to internal/atunnel/dns/limiter_test.go diff --git a/internal/ateworker/actornet/dns/pendingreq.go b/internal/atunnel/dns/pendingreq.go similarity index 97% rename from internal/ateworker/actornet/dns/pendingreq.go rename to internal/atunnel/dns/pendingreq.go index 65a04e7c5e..d120bc348d 100644 --- a/internal/ateworker/actornet/dns/pendingreq.go +++ b/internal/atunnel/dns/pendingreq.go @@ -21,8 +21,8 @@ import ( "sync" "time" - "github.com/agent-substrate/substrate/internal/ateworker/actornet/dns/freemap" - "github.com/agent-substrate/substrate/internal/ateworker/actornet/dns/protocol" + "go.ate.dev/substrate/internal/atunnel/dns/freemap" + "go.ate.dev/substrate/internal/atunnel/dns/protocol" "golang.org/x/net/dns/dnsmessage" ) diff --git a/internal/ateworker/actornet/dns/pendingreq_test.go b/internal/atunnel/dns/pendingreq_test.go similarity index 100% rename from internal/ateworker/actornet/dns/pendingreq_test.go rename to internal/atunnel/dns/pendingreq_test.go diff --git a/internal/ateworker/actornet/dns/policy.go b/internal/atunnel/dns/policy.go similarity index 96% rename from internal/ateworker/actornet/dns/policy.go rename to internal/atunnel/dns/policy.go index 39f2e3567b..6be8a9e643 100644 --- a/internal/ateworker/actornet/dns/policy.go +++ b/internal/atunnel/dns/policy.go @@ -21,7 +21,7 @@ import ( "golang.org/x/net/dns/dnsmessage" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/proto/ateletpb" ) // PolicySource supplies the egress policy an actor's DNS queries are checked diff --git a/internal/ateworker/actornet/dns/policy_test.go b/internal/atunnel/dns/policy_test.go similarity index 98% rename from internal/ateworker/actornet/dns/policy_test.go rename to internal/atunnel/dns/policy_test.go index aec5aabb8e..93164c2c13 100644 --- a/internal/ateworker/actornet/dns/policy_test.go +++ b/internal/atunnel/dns/policy_test.go @@ -25,8 +25,8 @@ import ( "github.com/google/go-cmp/cmp" "golang.org/x/net/dns/dnsmessage" - "github.com/agent-substrate/substrate/internal/ateworker/actornet/dns/protocol" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/atunnel/dns/protocol" + "go.ate.dev/substrate/internal/proto/ateletpb" ) // staticPolicy supplies policy, or fails with err. diff --git a/internal/ateworker/actornet/dns/protocol/doc.go b/internal/atunnel/dns/protocol/doc.go similarity index 100% rename from internal/ateworker/actornet/dns/protocol/doc.go rename to internal/atunnel/dns/protocol/doc.go diff --git a/internal/ateworker/actornet/dns/protocol/protocol.go b/internal/atunnel/dns/protocol/protocol.go similarity index 100% rename from internal/ateworker/actornet/dns/protocol/protocol.go rename to internal/atunnel/dns/protocol/protocol.go diff --git a/internal/ateworker/actornet/dns/protocol/protocol_test.go b/internal/atunnel/dns/protocol/protocol_test.go similarity index 100% rename from internal/ateworker/actornet/dns/protocol/protocol_test.go rename to internal/atunnel/dns/protocol/protocol_test.go diff --git a/internal/ateworker/actornet/dns/protocol/tcp.go b/internal/atunnel/dns/protocol/tcp.go similarity index 100% rename from internal/ateworker/actornet/dns/protocol/tcp.go rename to internal/atunnel/dns/protocol/tcp.go diff --git a/internal/ateworker/actornet/dns/protocol/tcp_test.go b/internal/atunnel/dns/protocol/tcp_test.go similarity index 100% rename from internal/ateworker/actornet/dns/protocol/tcp_test.go rename to internal/atunnel/dns/protocol/tcp_test.go diff --git a/internal/ateworker/actornet/dns/readme.md b/internal/atunnel/dns/readme.md similarity index 100% rename from internal/ateworker/actornet/dns/readme.md rename to internal/atunnel/dns/readme.md diff --git a/internal/ateworker/actornet/dns/relay.go b/internal/atunnel/dns/relay.go similarity index 100% rename from internal/ateworker/actornet/dns/relay.go rename to internal/atunnel/dns/relay.go diff --git a/internal/ateworker/actornet/dns/relay_linux.go b/internal/atunnel/dns/relay_linux.go similarity index 96% rename from internal/ateworker/actornet/dns/relay_linux.go rename to internal/atunnel/dns/relay_linux.go index 394bf88645..a85140d039 100644 --- a/internal/ateworker/actornet/dns/relay_linux.go +++ b/internal/atunnel/dns/relay_linux.go @@ -22,7 +22,7 @@ import ( "net" "strconv" - "github.com/agent-substrate/substrate/internal/netns" + "go.ate.dev/substrate/internal/netns" ) // Serve serves UDP and TCP DNS in the sandbox's local gateway namespace. diff --git a/internal/ateworker/actornet/dns/relay_test.go b/internal/atunnel/dns/relay_test.go similarity index 100% rename from internal/ateworker/actornet/dns/relay_test.go rename to internal/atunnel/dns/relay_test.go diff --git a/internal/atunnel/dns/resolvconf.go b/internal/atunnel/dns/resolvconf.go new file mode 100644 index 0000000000..dbdd9d87be --- /dev/null +++ b/internal/atunnel/dns/resolvconf.go @@ -0,0 +1,58 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package dns answers an actor's DNS from its sandbox's gateway namespace. +package dns + +import ( + "bufio" + "fmt" + "net" + "os" + "strings" +) + +// resolvConfNameservers reads nameservers from resolv.conf as "host:53". +func resolvConfNameservers(path string) ([]string, error) { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("dns: reading resolv.conf: %w", err) + } + defer f.Close() + + var out []string + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if i := strings.IndexAny(line, "#;"); i >= 0 { + line = strings.TrimSpace(line[:i]) + } + rest, ok := strings.CutPrefix(line, "nameserver") + if !ok { + continue + } + address := strings.TrimSpace(rest) + if address == "" || net.ParseIP(address) == nil { + continue + } + out = append(out, net.JoinHostPort(address, "53")) + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("dns: reading resolv.conf: %w", err) + } + if len(out) == 0 { + return nil, fmt.Errorf("dns: %s names no usable nameserver", path) + } + return out, nil +} diff --git a/internal/atunnel/dns/resolvconf_test.go b/internal/atunnel/dns/resolvconf_test.go new file mode 100644 index 0000000000..a9a789cbf8 --- /dev/null +++ b/internal/atunnel/dns/resolvconf_test.go @@ -0,0 +1,70 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dns + +import ( + "os" + "path/filepath" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestResolvConfNameservers(t *testing.T) { + for _, tc := range []struct { + name string + content string + want []string + wantErr bool + }{ + { + name: "cluster resolv.conf", + content: "search ate-system.svc.cluster.local svc.cluster.local\nnameserver 10.96.0.10\noptions ndots:5\n", + want: []string{"10.96.0.10:53"}, + }, + { + name: "several, in order", + content: "nameserver 10.96.0.10\nnameserver 8.8.8.8\n", + want: []string{"10.96.0.10:53", "8.8.8.8:53"}, + }, + { + name: "comments and blanks", + content: "# generated\n\n nameserver 10.96.0.10 # cluster\n;nameserver 1.1.1.1\n", + want: []string{"10.96.0.10:53"}, + }, + {name: "no nameserver", content: "search cluster.local\n", wantErr: true}, + {name: "unparsable address", content: "nameserver not-an-ip\n", wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "resolv.conf") + if err := os.WriteFile(path, []byte(tc.content), 0o644); err != nil { + t.Fatal(err) + } + got, err := resolvConfNameservers(path) + if tc.wantErr { + if err == nil { + t.Fatalf("resolvConfNameservers() = %v, want an error", got) + } + return + } + if err != nil { + t.Fatalf("resolvConfNameservers: %v", err) + } + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("nameservers mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/internal/ateworker/actornet/dns/serve_linux_test.go b/internal/atunnel/dns/serve_linux_test.go similarity index 92% rename from internal/ateworker/actornet/dns/serve_linux_test.go rename to internal/atunnel/dns/serve_linux_test.go index 995e3e3fbc..64958d055e 100644 --- a/internal/ateworker/actornet/dns/serve_linux_test.go +++ b/internal/atunnel/dns/serve_linux_test.go @@ -21,8 +21,8 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/netns" - "github.com/agent-substrate/substrate/internal/roottest" + "go.ate.dev/substrate/internal/netns" + "go.ate.dev/substrate/internal/roottest" ) func TestClosingSandboxDNSStopsServing(t *testing.T) { diff --git a/internal/ateworker/actornet/dns/server.go b/internal/atunnel/dns/server.go similarity index 100% rename from internal/ateworker/actornet/dns/server.go rename to internal/atunnel/dns/server.go diff --git a/internal/ateworker/actornet/dns/tcphandler.go b/internal/atunnel/dns/tcphandler.go similarity index 98% rename from internal/ateworker/actornet/dns/tcphandler.go rename to internal/atunnel/dns/tcphandler.go index e3eab1c011..b7434b9b09 100644 --- a/internal/ateworker/actornet/dns/tcphandler.go +++ b/internal/atunnel/dns/tcphandler.go @@ -24,7 +24,7 @@ import ( "sync" "time" - "github.com/agent-substrate/substrate/internal/ateworker/actornet/dns/protocol" + "go.ate.dev/substrate/internal/atunnel/dns/protocol" ) const ( diff --git a/internal/ateworker/actornet/dns/tcphandler_test.go b/internal/atunnel/dns/tcphandler_test.go similarity index 98% rename from internal/ateworker/actornet/dns/tcphandler_test.go rename to internal/atunnel/dns/tcphandler_test.go index d79bd266f9..5041831219 100644 --- a/internal/ateworker/actornet/dns/tcphandler_test.go +++ b/internal/atunnel/dns/tcphandler_test.go @@ -22,7 +22,7 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/ateworker/actornet/dns/protocol" + "go.ate.dev/substrate/internal/atunnel/dns/protocol" "golang.org/x/net/dns/dnsmessage" ) diff --git a/internal/ateworker/actornet/dns/udphandler.go b/internal/atunnel/dns/udphandler.go similarity index 100% rename from internal/ateworker/actornet/dns/udphandler.go rename to internal/atunnel/dns/udphandler.go diff --git a/internal/ateworker/actornet/dns/udphandler_test.go b/internal/atunnel/dns/udphandler_test.go similarity index 100% rename from internal/ateworker/actornet/dns/udphandler_test.go rename to internal/atunnel/dns/udphandler_test.go diff --git a/internal/atunnel/client.go b/internal/atunnel/egress/client.go similarity index 90% rename from internal/atunnel/client.go rename to internal/atunnel/egress/client.go index cc6077c839..5c844c1f1a 100644 --- a/internal/atunnel/client.go +++ b/internal/atunnel/egress/client.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package atunnel +package egress import ( "bufio" @@ -43,13 +43,13 @@ type ClientConfig struct { TrustBundlePath string } -// DialFunc dials a network address. It matches net.Dialer.DialContext. -type DialFunc func(ctx context.Context, network, address string) (net.Conn, error) +// dialFunc dials a network address. It matches net.Dialer.DialContext. +type dialFunc func(ctx context.Context, network, address string) (net.Conn, error) // ErrGatewayHandshake reports that the gateway's front door refused the // connection at TLS: it rejected the client certificate, or its own // certificate did not verify. -var ErrGatewayHandshake = errors.New("atunnel: egress gateway TLS handshake") +var ErrGatewayHandshake = errors.New("egress: gateway TLS handshake") // ConnectRejectedError reports a CONNECT the gateway answered with a non-2xx // status. The caller authenticated successfully and the request was declined @@ -58,23 +58,23 @@ var ErrGatewayHandshake = errors.New("atunnel: egress gateway TLS handshake") // caller can act on. type ConnectRejectedError struct { StatusCode int - // Status is the full status line, e.g. "403 Forbidden". - Status string - // Message is the response body, or the status text when the body is empty. - Message string + // status is the full status line, e.g. "403 Forbidden". + status string + // message is the response body, or the status text when the body is empty. + message string } func (e *ConnectRejectedError) Error() string { - return fmt.Sprintf("atunnel: egress gateway rejected CONNECT with %s: %s", e.Status, e.Message) + return fmt.Sprintf("egress: gateway rejected CONNECT with %s: %s", e.status, e.message) } -// ClientOption customizes a Client beyond its configuration. Production +// clientOption customizes a Client beyond its configuration. Production // callers need none of these. -type ClientOption func(*Client) +type clientOption func(*Client) -// WithDialer overrides how the client reaches the egress gateway. It exists so +// withDialer overrides how the client reaches the egress gateway. It exists so // tests can substitute a transport; by default a net.Dialer is used. -func WithDialer(dial DialFunc) ClientOption { +func withDialer(dial dialFunc) clientOption { return func(c *Client) { if dial != nil { c.dialContext = dial @@ -86,7 +86,7 @@ func WithDialer(dial DialFunc) ClientOption { type Client struct { gatewayAddress string tlsConfig *tls.Config - dialContext DialFunc + dialContext dialFunc mu sync.Mutex closed bool @@ -94,30 +94,30 @@ type Client struct { dialing *dialCall } -// Client implements egressDialer. -var _ egressDialer = (*Client)(nil) +// Client implements tunnelDialer. +var _ tunnelDialer = (*Client)(nil) // NewClient creates an egress CONNECT client and validates its TLS material. -func NewClient(cfg ClientConfig, opts ...ClientOption) (*Client, error) { +func NewClient(cfg ClientConfig, opts ...clientOption) (*Client, error) { if _, _, err := net.SplitHostPort(cfg.GatewayAddress); err != nil { - return nil, fmt.Errorf("atunnel: invalid egress gateway address %q: %w", cfg.GatewayAddress, err) + return nil, fmt.Errorf("egress: invalid egress gateway address %q: %w", cfg.GatewayAddress, err) } if cfg.ServerName == "" { - return nil, fmt.Errorf("atunnel: egress gateway server name is required") + return nil, fmt.Errorf("egress: gateway server name is required") } if cfg.GetClientCertificate == nil { - return nil, fmt.Errorf("atunnel: client certificate source is required") + return nil, fmt.Errorf("egress: client certificate source is required") } if cfg.TrustBundlePath == "" { - return nil, fmt.Errorf("atunnel: trust bundle path is required") + return nil, fmt.Errorf("egress: trust bundle path is required") } trustPEM, err := os.ReadFile(cfg.TrustBundlePath) if err != nil { - return nil, fmt.Errorf("atunnel: reading trust bundle: %w", err) + return nil, fmt.Errorf("egress: reading trust bundle: %w", err) } rootCAs := x509.NewCertPool() if !rootCAs.AppendCertsFromPEM(trustPEM) { - return nil, fmt.Errorf("atunnel: trust bundle %q contains no certificates", cfg.TrustBundlePath) + return nil, fmt.Errorf("egress: trust bundle %q contains no certificates", cfg.TrustBundlePath) } client := &Client{ @@ -197,7 +197,7 @@ type dialCall struct { func (c *Client) acquireConn(ctx context.Context) (*h2ClientConn, bool, *tls.Conn, error) { for { if err := ctx.Err(); err != nil { - return nil, false, nil, fmt.Errorf("atunnel: connecting to egress gateway: %w", err) + return nil, false, nil, fmt.Errorf("egress: connecting to egress gateway: %w", err) } // First check if there is an existing H/2 connection with available stream capacity. @@ -205,7 +205,7 @@ func (c *Client) acquireConn(ctx context.Context) (*h2ClientConn, bool, *tls.Con c.mu.Lock() if c.closed { c.mu.Unlock() - return nil, false, nil, fmt.Errorf("atunnel: connecting to egress gateway: %w", net.ErrClosed) + return nil, false, nil, fmt.Errorf("egress: connecting to egress gateway: %w", net.ErrClosed) } keep := c.h2Conns[:0] var reserved *h2ClientConn @@ -275,7 +275,7 @@ func (c *Client) acquireConn(ctx context.Context) (*h2ClientConn, bool, *tls.Con if err != nil { if ctx.Err() != nil { - return nil, false, nil, fmt.Errorf("atunnel: connecting to egress gateway: %w", ctx.Err()) + return nil, false, nil, fmt.Errorf("egress: connecting to egress gateway: %w", ctx.Err()) } return nil, false, nil, err } @@ -309,7 +309,7 @@ func (c *Client) acquireConn(ctx context.Context) (*h2ClientConn, bool, *tls.Con if orphan != nil { _ = orphan.Close() } - return nil, false, nil, fmt.Errorf("atunnel: connecting to egress gateway: %w", ctx.Err()) + return nil, false, nil, fmt.Errorf("egress: connecting to egress gateway: %w", ctx.Err()) } } } @@ -325,7 +325,7 @@ func (c *Client) runDial(dialCtx context.Context, call *dialCall) { ) rawConn, dialErr := c.dialContext(dialCtx, "tcp", c.gatewayAddress) if dialErr != nil { - err = fmt.Errorf("atunnel: connecting to egress gateway: %w", dialErr) + err = fmt.Errorf("egress: connecting to egress gateway: %w", dialErr) } else { tlsConn := tls.Client(rawConn, c.tlsConfig.Clone()) if hsErr := tlsConn.HandshakeContext(dialCtx); hsErr != nil { @@ -345,7 +345,7 @@ func (c *Client) runDial(dialCtx context.Context, call *dialCall) { } if c.closed || call.waiters == 0 { if c.closed && err == nil { - call.err = fmt.Errorf("atunnel: connecting to egress gateway: %w", net.ErrClosed) + call.err = fmt.Errorf("egress: connecting to egress gateway: %w", net.ErrClosed) } else { call.err = err } @@ -509,8 +509,8 @@ func newConnectRejectedError(resp *http.Response, body []byte) *ConnectRejectedE } return &ConnectRejectedError{ StatusCode: resp.StatusCode, - Status: resp.Status, - Message: message, + status: resp.Status, + message: message, } } @@ -997,7 +997,7 @@ func connectExchangeError(op string, err error) error { if gatewayHungUp(err) { return fmt.Errorf("%w: %s: %w", ErrGatewayHandshake, op, err) } - return fmt.Errorf("atunnel: %s: %w", op, err) + return fmt.Errorf("egress: %s: %w", op, err) } // gatewayHungUp reports whether err is the peer tearing the connection down, @@ -1025,16 +1025,16 @@ func gatewayHungUp(err error) bool { func validateDestination(destination string) error { host, port, err := net.SplitHostPort(destination) if err != nil { - return fmt.Errorf("atunnel: invalid egress destination %q: %w", destination, err) + return fmt.Errorf("egress: invalid egress destination %q: %w", destination, err) } if host == "" { - return fmt.Errorf("atunnel: invalid egress destination %q: host is empty", destination) + return fmt.Errorf("egress: invalid egress destination %q: host is empty", destination) } if net.ParseIP(host) == nil { - return fmt.Errorf("atunnel: invalid egress destination %q: host must be an IP address", destination) + return fmt.Errorf("egress: invalid egress destination %q: host must be an IP address", destination) } - if _, ok := ParsePort(port); !ok { - return fmt.Errorf("atunnel: invalid egress destination %q: port must be between 1 and 65535", destination) + if _, ok := parsePort(port); !ok { + return fmt.Errorf("egress: invalid egress destination %q: port must be between 1 and 65535", destination) } return nil } diff --git a/internal/atunnel/client_test.go b/internal/atunnel/egress/client_test.go similarity index 96% rename from internal/atunnel/client_test.go rename to internal/atunnel/egress/client_test.go index c4114ea7ac..8054cc31e0 100644 --- a/internal/atunnel/client_test.go +++ b/internal/atunnel/egress/client_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package atunnel +package egress import ( "bufio" @@ -57,7 +57,7 @@ func TestClientDialContext(t *testing.T) { t.Errorf("tunneled payload = %q, want ping", payload) } }) - client := newTestClient(t, ca, WithDialer(dialFixedAddress(gatewayAddress))) + client := newTestClient(t, ca, withDialer(dialFixedAddress(gatewayAddress))) conn, err := client.DialContext(context.Background(), "192.0.2.10:443") if err != nil { @@ -125,7 +125,7 @@ func TestClientDialContextHTTP2(t *testing.T) { } flusher.Flush() })) - client := newTestClient(t, ca, WithDialer(dialFixedAddress(gatewayAddress))) + client := newTestClient(t, ca, withDialer(dialFixedAddress(gatewayAddress))) dialCtx, cancelDial := context.WithCancel(context.Background()) conn, err := client.DialContext(dialCtx, "192.0.2.10:443") @@ -214,7 +214,7 @@ func TestClientDialContextHTTP2MultiplexesStreams(t *testing.T) { })) var tcpDials atomic.Int32 - client := newTestClient(t, ca, WithDialer(func(ctx context.Context, network, _ string) (net.Conn, error) { + client := newTestClient(t, ca, withDialer(func(ctx context.Context, network, _ string) (net.Conn, error) { tcpDials.Add(1) return (&net.Dialer{}).DialContext(ctx, network, gatewayAddress) })) @@ -272,7 +272,7 @@ func TestClientDialContextHTTP2ReconnectsAfterConnectionClose(t *testing.T) { })) var tcpDials atomic.Int32 - client := newTestClient(t, ca, WithDialer(func(ctx context.Context, network, _ string) (net.Conn, error) { + client := newTestClient(t, ca, withDialer(func(ctx context.Context, network, _ string) (net.Conn, error) { tcpDials.Add(1) return (&net.Dialer{}).DialContext(ctx, network, gatewayAddress) })) @@ -320,7 +320,7 @@ func TestClientDialContextHTTP2Deadlines(t *testing.T) { <-releaseServerSend _, _ = io.WriteString(w, "delayed") })) - client := newTestClient(t, ca, WithDialer(dialFixedAddress(gatewayAddress))) + client := newTestClient(t, ca, withDialer(dialFixedAddress(gatewayAddress))) conn, err := client.DialContext(context.Background(), "192.0.2.10:443") if err != nil { @@ -356,7 +356,7 @@ func TestClientDialContextRejected(t *testing.T) { body := "denied by policy" _, _ = fmt.Fprintf(conn, "HTTP/1.1 403 Forbidden\r\nContent-Length: %d\r\n\r\n%s", len(body), body) }) - client := newTestClient(t, ca, WithDialer(dialFixedAddress(gatewayAddress))) + client := newTestClient(t, ca, withDialer(dialFixedAddress(gatewayAddress))) _, err := client.DialContext(context.Background(), "192.0.2.10:443") if err == nil || !strings.Contains(err.Error(), "denied by policy") { @@ -377,7 +377,7 @@ func TestClientDialContextRejected(t *testing.T) { } })) var tcpDials atomic.Int32 - client := newTestClient(t, ca, WithDialer(func(ctx context.Context, network, _ string) (net.Conn, error) { + client := newTestClient(t, ca, withDialer(func(ctx context.Context, network, _ string) (net.Conn, error) { tcpDials.Add(1) return (&net.Dialer{}).DialContext(ctx, network, gatewayAddress) })) @@ -387,7 +387,7 @@ func TestClientDialContextRejected(t *testing.T) { if !errors.As(err, &rejected) { t.Fatalf("DialContext error = %v, want *ConnectRejectedError", err) } - if rejected.StatusCode != http.StatusForbidden || rejected.Message != "denied by policy" { + if rejected.StatusCode != http.StatusForbidden || rejected.message != "denied by policy" { t.Errorf("ConnectRejectedError = %+v, want 403 denied by policy", rejected) } @@ -431,7 +431,7 @@ func TestClientDialContextGatewayRefusesClientCertificate(t *testing.T) { // client's certificate, which is the shape of an actor presenting a // podidentity credential to a door that only accepts actor identity. gatewayAddress := serveTestRefusingGateway(t, ca, newTestCA(t), tt.maxTLSVersion, tt.nextProtos) - client := newTestClient(t, ca, WithDialer(dialFixedAddress(gatewayAddress))) + client := newTestClient(t, ca, withDialer(dialFixedAddress(gatewayAddress))) _, err := client.DialContext(context.Background(), "192.0.2.10:443") if err == nil { @@ -461,7 +461,7 @@ func TestClientDialContextGatewayHangsUpBeforeResponding(t *testing.T) { gatewayAddress := serveTestConnectGateway(t, ca, func(conn net.Conn, _ *http.Request) { _ = conn.Close() }) - client := newTestClient(t, ca, WithDialer(dialFixedAddress(gatewayAddress))) + client := newTestClient(t, ca, withDialer(dialFixedAddress(gatewayAddress))) _, err := client.DialContext(context.Background(), "192.0.2.10:443") if err == nil { @@ -500,7 +500,7 @@ func TestClientDialContextGatewayHangsUpBeforeResponding(t *testing.T) { _ = tlsConn.Close() }() - client := newTestClient(t, ca, WithDialer(dialFixedAddress(listener.Addr().String()))) + client := newTestClient(t, ca, withDialer(dialFixedAddress(listener.Addr().String()))) _, err = client.DialContext(context.Background(), "192.0.2.10:443") if err == nil { t.Fatal("DialContext succeeded against an HTTP/2 gateway that hung up") @@ -600,7 +600,7 @@ func TestClientClose(t *testing.T) { })) connClosed := make(chan struct{}) - client := newTestClient(t, ca, WithDialer(func(ctx context.Context, network, _ string) (net.Conn, error) { + client := newTestClient(t, ca, withDialer(func(ctx context.Context, network, _ string) (net.Conn, error) { c, err := (&net.Dialer{}).DialContext(ctx, network, gatewayAddr) if err != nil { return nil, err @@ -628,13 +628,13 @@ func TestClientClose(t *testing.T) { // dialFixedAddress ignores the requested address and connects to address, so // tests can point a client at a listener on an ephemeral port. -func dialFixedAddress(address string) DialFunc { +func dialFixedAddress(address string) dialFunc { return func(ctx context.Context, network, _ string) (net.Conn, error) { return (&net.Dialer{}).DialContext(ctx, network, address) } } -func newTestClient(t *testing.T, ca *testCA, opts ...ClientOption) *Client { +func newTestClient(t *testing.T, ca *testCA, opts ...clientOption) *Client { t.Helper() dir := t.TempDir() trustPath := filepath.Join(dir, "trust.pem") diff --git a/internal/atunnel/credential.go b/internal/atunnel/egress/credential.go similarity index 83% rename from internal/atunnel/credential.go rename to internal/atunnel/egress/credential.go index 6a2c861422..5cf3a7b5be 100644 --- a/internal/atunnel/credential.go +++ b/internal/atunnel/egress/credential.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package atunnel +package egress import ( "context" @@ -26,8 +26,8 @@ import ( "sync" "time" - "github.com/agent-substrate/substrate/internal/ateletdial" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/ateletdial" + "go.ate.dev/substrate/internal/proto/ateletpb" ) // BrokerCertificateSource owns atunnel's actor private key and obtains the @@ -80,15 +80,15 @@ func NewBrokerCertificateSource(cfg BrokerConfig) (*BrokerCertificateSource, err } if cfg.AteletSPIFFEID == "" { - return nil, fmt.Errorf("atunnel: expected atelet SPIFFE ID is required") + return nil, fmt.Errorf("egress: expected atelet SPIFFE ID is required") } tlsConfig, err := ateletdial.TLSConfig(cfg.CredentialBundlePath, cfg.TrustBundlePath, cfg.AteletSPIFFEID) if err != nil { - return nil, fmt.Errorf("atunnel: %w", err) + return nil, fmt.Errorf("egress: %w", err) } privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { - return nil, fmt.Errorf("atunnel: generate actor private key: %w", err) + return nil, fmt.Errorf("egress: generate actor private key: %w", err) } return &BrokerCertificateSource{ @@ -106,7 +106,7 @@ func NewBrokerCertificateSource(cfg BrokerConfig) (*BrokerCertificateSource, err func (s *BrokerCertificateSource) MintWorkerCertificate(ctx context.Context) (time.Time, error) { csr, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{}, s.privateKey) if err != nil { - return time.Time{}, fmt.Errorf("atunnel: create actor CSR: %w", err) + return time.Time{}, fmt.Errorf("egress: create actor CSR: %w", err) } // TODO(identity): We should not be re-establishing a gRPC connection for @@ -126,7 +126,7 @@ func (s *BrokerCertificateSource) MintWorkerCertificate(ctx context.Context) (ti CertificateSigningRequest: csr, }) if err != nil { - return time.Time{}, fmt.Errorf("atunnel: mint actor certificate: %w", err) + return time.Time{}, fmt.Errorf("egress: mint actor certificate: %w", err) } // TODO(identity): I don't think we need to check all this stuff. We can @@ -134,21 +134,21 @@ func (s *BrokerCertificateSource) MintWorkerCertificate(ctx context.Context) (ti chain := resp.GetActorCertificates() if len(chain) == 0 { - return time.Time{}, fmt.Errorf("atunnel: credential broker returned no actor certificate") + return time.Time{}, fmt.Errorf("egress: credential broker returned no actor certificate") } leaf, err := x509.ParseCertificate(chain[0]) if err != nil { - return time.Time{}, fmt.Errorf("atunnel: parse actor certificate: %w", err) + return time.Time{}, fmt.Errorf("egress: parse actor certificate: %w", err) } if !s.privateKey.PublicKey.Equal(leaf.PublicKey) { - return time.Time{}, fmt.Errorf("atunnel: actor certificate does not match private key") + return time.Time{}, fmt.Errorf("egress: actor certificate does not match private key") } now := time.Now() if now.Before(leaf.NotBefore) || !leaf.NotAfter.After(now) { - return time.Time{}, fmt.Errorf("atunnel: credential broker returned an invalid actor certificate lifetime") + return time.Time{}, fmt.Errorf("egress: credential broker returned an invalid actor certificate lifetime") } if !slices.Contains(leaf.ExtKeyUsage, x509.ExtKeyUsageClientAuth) { - return time.Time{}, fmt.Errorf("atunnel: actor certificate cannot authenticate a TLS client") + return time.Time{}, fmt.Errorf("egress: actor certificate cannot authenticate a TLS client") } cert := &tls.Certificate{Certificate: chain, PrivateKey: s.privateKey, Leaf: leaf} s.mu.Lock() @@ -163,7 +163,7 @@ func (s *BrokerCertificateSource) GetClientCertificate(*tls.CertificateRequestIn s.mu.RLock() defer s.mu.RUnlock() if s.certificate == nil || !s.certificate.Leaf.NotAfter.After(time.Now()) { - return nil, fmt.Errorf("atunnel: no valid actor certificate") + return nil, fmt.Errorf("egress: no valid actor certificate") } return s.certificate, nil } diff --git a/internal/atunnel/credential_test.go b/internal/atunnel/egress/credential_test.go similarity index 97% rename from internal/atunnel/credential_test.go rename to internal/atunnel/egress/credential_test.go index 381a6ee3b4..22f96bc462 100644 --- a/internal/atunnel/credential_test.go +++ b/internal/atunnel/egress/credential_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package atunnel +package egress import ( "context" @@ -31,9 +31,9 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/installdefaults" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/substratex509" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/substratex509" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" diff --git a/internal/atunnel/egress.go b/internal/atunnel/egress/egress.go similarity index 75% rename from internal/atunnel/egress.go rename to internal/atunnel/egress/egress.go index 422ce1d928..c3d7017555 100644 --- a/internal/atunnel/egress.go +++ b/internal/atunnel/egress/egress.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package atunnel +package egress import ( "context" @@ -22,6 +22,7 @@ import ( "log/slog" "math/rand/v2" "net" + "strconv" "sync" "time" @@ -29,8 +30,8 @@ import ( "google.golang.org/grpc/status" ) -// egressDialer opens an authenticated tunnel to an original destination. -type egressDialer interface { +// tunnelDialer opens an authenticated tunnel to an original destination. +type tunnelDialer interface { DialContext(context.Context, string) (net.Conn, error) Close() error } @@ -39,25 +40,25 @@ type actorCertificateSource interface { MintWorkerCertificate(context.Context) (time.Time, error) } -// OriginalDestination returns the address that a transparently intercepted +// originalDestinationFunc returns the address that a transparently intercepted // connection originally targeted. -type OriginalDestination func(net.Conn) (string, error) +type originalDestinationFunc func(net.Conn) (string, error) -// Egress proxies actor TCP connections through an egress CONNECT dialer. It is +// Proxy proxies actor TCP connections through an egress CONNECT dialer. It is // long-lived across actor activations, but only carries traffic while an actor // is assigned to its worker. -type Egress struct { - originalDestination OriginalDestination +type Proxy struct { + originalDestination originalDestinationFunc // tunnelOpenTimeout bounds the wait for the gateway's CONNECT answer. tunnelOpenTimeout time.Duration mu sync.Mutex // Keyed by actor UID, supplied by the namespace-specific listener. - active map[string]*egressActivation + active map[string]*activation } -type egressActivation struct { - dialer egressDialer +type activation struct { + dialer tunnelDialer certificateSource actorCertificateSource expiresAt time.Time @@ -68,26 +69,26 @@ type egressActivation struct { wg sync.WaitGroup } -// NewEgress creates an activation-aware egress proxy. -func NewEgress(originalDestination OriginalDestination) (*Egress, error) { +// New creates an activation-aware egress proxy. +func New(originalDestination originalDestinationFunc) (*Proxy, error) { if originalDestination == nil { - return nil, fmt.Errorf("atunnel: original destination resolver is required") + return nil, fmt.Errorf("egress: original destination resolver is required") } // The gateway answers within its 5s authorization timeout; a longer wait // means it has queued the CONNECT behind its connection cap, and closing // the connection is what frees that queue slot. const tunnelOpenTimeout = 10 * time.Second - return &Egress{ + return &Proxy{ originalDestination: originalDestination, tunnelOpenTimeout: tunnelOpenTimeout, - active: map[string]*egressActivation{}, + active: map[string]*activation{}, }, nil } // Bind captures the actor's activation before its listeners start serving. -func (e *Egress) Bind(actorUID string) (func(context.Context, net.Listener) error, error) { +func (e *Proxy) Bind(actorUID string) (func(context.Context, net.Listener) error, error) { if actorUID == "" { - return nil, fmt.Errorf("atunnel: actor UID is required") + return nil, fmt.Errorf("egress: actor UID is required") } e.mu.Lock() active := e.activationLocked(actorUID) @@ -105,17 +106,17 @@ func (e *Egress) Bind(actorUID string) (func(context.Context, net.Listener) erro }, nil } -func (e *Egress) activationLocked(actorUID string) *egressActivation { +func (e *Proxy) activationLocked(actorUID string) *activation { active := e.active[actorUID] if active == nil { ctx, cancel := context.WithCancel(context.Background()) - active = &egressActivation{ctx: ctx, cancel: cancel} + active = &activation{ctx: ctx, cancel: cancel} e.active[actorUID] = active } return active } -func (e *Egress) serve(ctx context.Context, listener net.Listener, active *egressActivation) error { +func (e *Proxy) serve(ctx context.Context, listener net.Listener, active *activation) error { done := make(chan struct{}) go func() { select { @@ -134,31 +135,31 @@ func (e *Egress) serve(ctx context.Context, listener net.Listener, active *egres if ctx.Err() != nil || errors.Is(err, net.ErrClosed) { return nil } - return fmt.Errorf("atunnel: accepting actor egress connection: %w", err) + return fmt.Errorf("egress: accepting actor egress connection: %w", err) } e.handle(conn, active) } } // Activate enables the actor's egress and certificate renewal until deactivation. -func (e *Egress) Activate(actorUID string, dialer egressDialer, certificateSource actorCertificateSource, expiresAt time.Time) error { +func (e *Proxy) Activate(actorUID string, dialer tunnelDialer, certificateSource actorCertificateSource, expiresAt time.Time) error { if actorUID == "" { - return fmt.Errorf("atunnel: actor UID is required") + return fmt.Errorf("egress: actor UID is required") } if dialer == nil { - return fmt.Errorf("atunnel: egress dialer is required") + return fmt.Errorf("egress: dialer is required") } if certificateSource == nil { - return fmt.Errorf("atunnel: actor certificate source is required") + return fmt.Errorf("egress: actor certificate source is required") } if !expiresAt.After(time.Now()) { - return fmt.Errorf("atunnel: valid actor certificate is required") + return fmt.Errorf("egress: valid actor certificate is required") } e.mu.Lock() defer e.mu.Unlock() active := e.activationLocked(actorUID) if active.dialer != nil { - return fmt.Errorf("atunnel: actor %s already has active egress", actorUID) + return fmt.Errorf("egress: actor %s already has active egress", actorUID) } active.dialer = dialer active.certificateSource = certificateSource @@ -168,7 +169,7 @@ func (e *Egress) Activate(actorUID string, dialer egressDialer, certificateSourc return nil } -func (e *Egress) renew(active *egressActivation, expiresAt time.Time) { +func (e *Proxy) renew(active *activation, expiresAt time.Time) { defer active.wg.Done() // Schedule from the credential's remaining lifetime: renew at 90%, then // keep retrying after expiry so egress can recover without reactivation. @@ -249,7 +250,7 @@ func waitForRenewal(ctx context.Context, delay time.Duration) bool { } // Deactivate disables the actor's egress and closes and drains its streams. -func (e *Egress) Deactivate(ctx context.Context, actorUID string) error { +func (e *Proxy) Deactivate(ctx context.Context, actorUID string) error { e.mu.Lock() active := e.active[actorUID] delete(e.active, actorUID) @@ -271,7 +272,7 @@ func (e *Egress) Deactivate(ctx context.Context, actorUID string) error { select { case <-done: case <-ctx.Done(): - err = fmt.Errorf("atunnel: waiting for active egress streams to stop: %w", ctx.Err()) + err = fmt.Errorf("egress: waiting for active egress streams to stop: %w", ctx.Err()) } if active.dialer != nil { err = errors.Join(err, active.dialer.Close()) @@ -279,7 +280,7 @@ func (e *Egress) Deactivate(ctx context.Context, actorUID string) error { return err } -func (e *Egress) handle(downstream net.Conn, active *egressActivation) { +func (e *Proxy) handle(downstream net.Conn, active *activation) { e.mu.Lock() if active == nil || active.ctx.Err() != nil { e.mu.Unlock() @@ -346,17 +347,27 @@ func closeWrite(conn net.Conn) { } } -// EgressPort is the port from a listen address, which each sandbox's redirect -// aims at. The address itself is never bound: egress is served from inside the -// sandbox namespaces. -func EgressPort(listenAddress string) (uint16, error) { +// PortFromListenAddr is the port from a listen address, which each sandbox's +// redirect aims at. The address itself is never bound: egress is served from +// inside the sandbox namespaces. +func PortFromListenAddr(listenAddress string) (uint16, error) { _, port, err := net.SplitHostPort(listenAddress) if err != nil { - return 0, fmt.Errorf("atunnel: egress listen address %q: %w", listenAddress, err) + return 0, fmt.Errorf("egress: listen address %q: %w", listenAddress, err) } - p, ok := ParsePort(port) + p, ok := parsePort(port) if !ok { - return 0, fmt.Errorf("atunnel: egress listen address %q has no usable port", listenAddress) + return 0, fmt.Errorf("egress: listen address %q has no usable port", listenAddress) } return uint16(p), nil } + +// parsePort parses s as a TCP port number, returning ok=false for anything +// outside the valid 1-65535 range (including non-numeric input). +func parsePort(s string) (port int, ok bool) { + p, err := strconv.Atoi(s) + if err != nil || p < 1 || p > 65535 { + return 0, false + } + return p, true +} diff --git a/internal/atunnel/egress_test.go b/internal/atunnel/egress/egress_test.go similarity index 91% rename from internal/atunnel/egress_test.go rename to internal/atunnel/egress/egress_test.go index 75de28f6d0..e3b067e5fc 100644 --- a/internal/atunnel/egress_test.go +++ b/internal/atunnel/egress/egress_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package atunnel +package egress import ( "context" @@ -55,7 +55,7 @@ func (listener *delayedEgressListener) Accept() (net.Conn, error) { func TestEgressBindingDoesNotOutliveItsActivation(t *testing.T) { for _, lateServe := range []bool{false, true} { t.Run(fmt.Sprintf("lateServe=%t", lateServe), func(t *testing.T) { - egress, err := NewEgress(func(net.Conn) (string, error) { return "192.0.2.10:443", nil }) + egress, err := New(func(net.Conn) (string, error) { return "192.0.2.10:443", nil }) if err != nil { t.Fatal(err) } @@ -63,7 +63,7 @@ func TestEgressBindingDoesNotOutliveItsActivation(t *testing.T) { dialed := make(chan string, 2) activate := func(generation string) { t.Helper() - if err := egress.Activate(testActorUID, egressDialerFunc(func(context.Context, string) (net.Conn, error) { + if err := egress.Activate(testActorUID, tunnelDialerFunc(func(context.Context, string) (net.Conn, error) { dialed <- generation return nil, errors.New("test dial") }), fakeActorCertificateSource{}, time.Now().Add(time.Hour)); err != nil { @@ -148,7 +148,7 @@ func TestEgressBindingDoesNotOutliveItsActivation(t *testing.T) { } func TestEgressUnactivatedBindingCloses(t *testing.T) { - egress, err := NewEgress(func(net.Conn) (string, error) { return "", nil }) + egress, err := New(func(net.Conn) (string, error) { return "", nil }) if err != nil { t.Fatal(err) } @@ -173,11 +173,11 @@ func TestEgressUnactivatedBindingCloses(t *testing.T) { } func TestEgressActivationFailsClosed(t *testing.T) { - egress, err := NewEgress(func(net.Conn) (string, error) { return "", nil }) + egress, err := New(func(net.Conn) (string, error) { return "", nil }) if err != nil { t.Fatal(err) } - dialer := egressDialerFunc(func(context.Context, string) (net.Conn, error) { + dialer := tunnelDialerFunc(func(context.Context, string) (net.Conn, error) { t.Fatal("dialed after failed activation") return nil, nil }) @@ -200,11 +200,11 @@ func TestEgressExpiryRejectsNewButPreservesEstablished(t *testing.T) { defer upstreamGateway.Close() var dials atomic.Int32 var mints atomic.Int32 - dialer := egressDialerFunc(func(context.Context, string) (net.Conn, error) { + dialer := tunnelDialerFunc(func(context.Context, string) (net.Conn, error) { dials.Add(1) return upstreamProxy, nil }) - egress, err := NewEgress(func(net.Conn) (string, error) { return "192.0.2.10:443", nil }) + egress, err := New(func(net.Conn) (string, error) { return "192.0.2.10:443", nil }) if err != nil { t.Fatal(err) } @@ -257,10 +257,10 @@ func TestEgressRenewsBeforeExpiry(t *testing.T) { } upstream, gateway := net.Pipe() defer gateway.Close() - dialer := egressDialerFunc(func(context.Context, string) (net.Conn, error) { + dialer := tunnelDialerFunc(func(context.Context, string) (net.Conn, error) { return upstream, nil }) - egress, err := NewEgress(func(net.Conn) (string, error) { return "192.0.2.10:443", nil }) + egress, err := New(func(net.Conn) (string, error) { return "192.0.2.10:443", nil }) if err != nil { t.Fatal(err) } @@ -303,11 +303,11 @@ func TestEgressStopsAfterTerminalRenewalFailure(t *testing.T) { for _, code := range []codes.Code{codes.Aborted, codes.FailedPrecondition, codes.PermissionDenied} { t.Run(code.String(), func(t *testing.T) { called := make(chan struct{}, 1) - egress, err := NewEgress(func(net.Conn) (string, error) { return "", nil }) + egress, err := New(func(net.Conn) (string, error) { return "", nil }) if err != nil { t.Fatal(err) } - if err := egress.Activate(testActorUID, egressDialerFunc(func(context.Context, string) (net.Conn, error) { + if err := egress.Activate(testActorUID, tunnelDialerFunc(func(context.Context, string) (net.Conn, error) { t.Fatal("dialed after renewal was denied") return nil, nil }), fakeActorCertificateSource{ @@ -343,11 +343,11 @@ func TestEgressStopsAfterTerminalRenewalFailure(t *testing.T) { func TestEgressDeactivationDropsConcurrentRenewal(t *testing.T) { started := make(chan struct{}, 1) release := make(chan struct{}) - egress, err := NewEgress(func(net.Conn) (string, error) { return "", nil }) + egress, err := New(func(net.Conn) (string, error) { return "", nil }) if err != nil { t.Fatal(err) } - dialer := egressDialerFunc(func(context.Context, string) (net.Conn, error) { return nil, nil }) + dialer := tunnelDialerFunc(func(context.Context, string) (net.Conn, error) { return nil, nil }) if err := egress.Activate(testActorUID, dialer, fakeActorCertificateSource{ expiresAt: time.Now().Add(time.Hour), called: started, @@ -409,9 +409,9 @@ func TestEgressEndToEnd(t *testing.T) { t.Errorf("writing gateway payload: %v", err) } }) - client := newTestClient(t, ca, WithDialer(dialFixedAddress(gatewayAddress))) + client := newTestClient(t, ca, withDialer(dialFixedAddress(gatewayAddress))) - egress, err := NewEgress(func(net.Conn) (string, error) { + egress, err := New(func(net.Conn) (string, error) { return "192.0.2.10:443", nil }) if err != nil { @@ -462,7 +462,7 @@ func TestEgressEndToEnd(t *testing.T) { } func TestEgressRejectsInactiveConnection(t *testing.T) { - egress, err := NewEgress(func(net.Conn) (string, error) { + egress, err := New(func(net.Conn) (string, error) { t.Fatal("inactive egress resolved destination") return "", nil }) @@ -492,9 +492,9 @@ func TestEgressBoundsHowLongATunnelWaitsToOpen(t *testing.T) { <-r.Context().Done() withdrawn <- struct{}{} })) - client := newTestClient(t, ca, WithDialer(dialFixedAddress(gatewayAddress))) + client := newTestClient(t, ca, withDialer(dialFixedAddress(gatewayAddress))) - egress, err := NewEgress(func(net.Conn) (string, error) { return "192.0.2.10:443", nil }) + egress, err := New(func(net.Conn) (string, error) { return "192.0.2.10:443", nil }) if err != nil { t.Fatal(err) } @@ -523,13 +523,13 @@ func TestEgressBoundsHowLongATunnelWaitsToOpen(t *testing.T) { } } -type egressDialerFunc func(context.Context, string) (net.Conn, error) +type tunnelDialerFunc func(context.Context, string) (net.Conn, error) -func (f egressDialerFunc) DialContext(ctx context.Context, destination string) (net.Conn, error) { +func (f tunnelDialerFunc) DialContext(ctx context.Context, destination string) (net.Conn, error) { return f(ctx, destination) } -func (f egressDialerFunc) Close() error { +func (f tunnelDialerFunc) Close() error { return nil } @@ -560,12 +560,12 @@ func (s fakeActorCertificateSource) MintWorkerCertificate(context.Context) (time // Each actor's egress is armed and disarmed on its own: one actor's teardown // must not cut off another's tunnels. func TestEgressIsArmedPerActor(t *testing.T) { - egress, err := NewEgress(func(net.Conn) (string, error) { return "example.com:443", nil }) + egress, err := New(func(net.Conn) (string, error) { return "example.com:443", nil }) if err != nil { t.Fatal(err) } source := fakeActorCertificateSource{expiresAt: time.Now().Add(time.Hour)} - dialer := egressDialerFunc(func(context.Context, string) (net.Conn, error) { + dialer := tunnelDialerFunc(func(context.Context, string) (net.Conn, error) { return nil, errors.New("not dialed in this test") }) for _, uid := range []string{"actor-uid-1", "actor-uid-2"} { @@ -601,7 +601,7 @@ func TestEgressDeactivateClosesIdleH2Connection(t *testing.T) { })) connClosed := make(chan struct{}) - client := newTestClient(t, ca, WithDialer(func(ctx context.Context, network, _ string) (net.Conn, error) { + client := newTestClient(t, ca, withDialer(func(ctx context.Context, network, _ string) (net.Conn, error) { c, err := (&net.Dialer{}).DialContext(ctx, network, gatewayAddr) if err != nil { return nil, err @@ -609,7 +609,7 @@ func TestEgressDeactivateClosesIdleH2Connection(t *testing.T) { return &closeNotifyingConn{Conn: c, closed: connClosed}, nil })) - egress, err := NewEgress(func(net.Conn) (string, error) { return "192.0.2.10:443", nil }) + egress, err := New(func(net.Conn) (string, error) { return "192.0.2.10:443", nil }) if err != nil { t.Fatal(err) } diff --git a/internal/atunnel/egress/testhelpers_test.go b/internal/atunnel/egress/testhelpers_test.go new file mode 100644 index 0000000000..91708eecf3 --- /dev/null +++ b/internal/atunnel/egress/testhelpers_test.go @@ -0,0 +1,138 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package egress + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net/url" + "os" + "testing" + "time" +) + +type testCA struct { + cert *x509.Certificate + key *ecdsa.PrivateKey + certPEM []byte +} + +func newTestCA(t *testing.T) *testCA { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + now := time.Now() + template := &x509.Certificate{ + SerialNumber: big.NewInt(now.UnixNano()), + Subject: pkix.Name{CommonName: "test CA"}, + NotBefore: now.Add(-time.Minute), + NotAfter: now.Add(time.Hour), + IsCA: true, + KeyUsage: x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatal(err) + } + return &testCA{ + cert: cert, + key: key, + certPEM: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), + } +} + +func (ca *testCA) issue(t *testing.T, spiffeID string, usages []x509.ExtKeyUsage) tls.Certificate { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + now := time.Now() + template := &x509.Certificate{ + SerialNumber: big.NewInt(now.UnixNano()), + NotBefore: now.Add(-time.Minute), + NotAfter: now.Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: usages, + } + if spiffeID != "" { + uri, err := url.Parse(spiffeID) + if err != nil { + t.Fatal(err) + } + template.URIs = []*url.URL{uri} + } + der, err := x509.CreateCertificate(rand.Reader, template, ca.cert, &key.PublicKey, ca.key) + if err != nil { + t.Fatal(err) + } + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatal(err) + } + cert, err := tls.X509KeyPair( + append(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), ca.certPEM...), + pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), + ) + if err != nil { + t.Fatal(err) + } + return cert +} + +func writeCredentialBundle(t *testing.T, path string, cert tls.Certificate) { + t.Helper() + key, ok := cert.PrivateKey.(*ecdsa.PrivateKey) + if !ok { + t.Fatalf("private key has type %T", cert.PrivateKey) + } + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatal(err) + } + var bundle []byte + for _, der := range cert.Certificate { + bundle = append(bundle, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})...) + } + bundle = append(bundle, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER})...) + if err := os.WriteFile(path, bundle, 0o600); err != nil { + t.Fatal(err) + } +} + +func receiveWithin[T any](t *testing.T, channel <-chan T, description string) T { + t.Helper() + select { + case value := <-channel: + return value + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for %s", description) + var zero T + return zero + } +} diff --git a/internal/atunnel/ingress.go b/internal/atunnel/ingress/ingress.go similarity index 88% rename from internal/atunnel/ingress.go rename to internal/atunnel/ingress/ingress.go index e921725438..d291ded8e4 100644 --- a/internal/atunnel/ingress.go +++ b/internal/atunnel/ingress/ingress.go @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package atunnel carries actor ingress and egress through a worker pod. -package atunnel +// Package ingress carries actor ingress through a worker pod. +package ingress import ( "context" @@ -32,9 +32,9 @@ import ( "sync" "time" - "github.com/agent-substrate/substrate/internal/atenet" - "github.com/agent-substrate/substrate/internal/credbundle" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/atenet" + "go.ate.dev/substrate/internal/credbundle" + "go.ate.dev/substrate/internal/resources" ) const ( @@ -42,9 +42,9 @@ const ( // mTLS CONNECT tunnels from the ingress router. DefaultConnectPort = 8443 - // StaleAssignmentHeader distinguishes an atunnel routing rejection from a + // staleAssignmentHeader distinguishes an atunnel routing rejection from a // 421 returned by the actor application itself. - StaleAssignmentHeader = "X-Ate-Assignment-Stale" + staleAssignmentHeader = "X-Ate-Assignment-Stale" // TargetPortHeader carries the port to reach on the actor: the CONNECT // :authority's port for arbitrary-port ingress, or the default 80 // otherwise (see atenet-router's HandleRequestHeaders). cfg.Upstream is @@ -63,6 +63,9 @@ func ParsePort(s string) (port int, ok bool) { return p, true } +// DialFunc dials a network address. It matches net.Dialer.DialContext. +type DialFunc func(ctx context.Context, network, address string) (net.Conn, error) + // Config configures an ingress Server. type Config struct { CredentialBundlePath string @@ -101,16 +104,16 @@ type activation struct { // NewServer creates a Server and validates its TLS material. func NewServer(cfg Config) (*Server, error) { if cfg.CredentialBundlePath == "" { - return nil, fmt.Errorf("atunnel: credential bundle path is required") + return nil, fmt.Errorf("ingress: credential bundle path is required") } if cfg.TrustBundlePath == "" { - return nil, fmt.Errorf("atunnel: trust bundle path is required") + return nil, fmt.Errorf("ingress: trust bundle path is required") } if cfg.AllowedClientID == "" { - return nil, fmt.Errorf("atunnel: allowed client identity is required") + return nil, fmt.Errorf("ingress: allowed client identity is required") } if cfg.Upstream == nil || cfg.Upstream.Scheme == "" || cfg.Upstream.Host == "" { - return nil, fmt.Errorf("atunnel: upstream URL is required") + return nil, fmt.Errorf("ingress: upstream URL is required") } // Load once at startup so a malformed or missing projection fails the pod @@ -121,7 +124,7 @@ func NewServer(cfg Config) (*Server, error) { } loadClientCAs := credbundle.PoolLoader(cfg.TrustBundlePath) if _, err := loadClientCAs(); err != nil { - return nil, fmt.Errorf("atunnel: loading trust bundle: %w", err) + return nil, fmt.Errorf("ingress: loading trust bundle: %w", err) } s := &Server{ @@ -134,14 +137,14 @@ func NewServer(cfg Config) (*Server, error) { } verifyConnection := func(cs tls.ConnectionState) error { if len(cs.PeerCertificates) == 0 { - return fmt.Errorf("atunnel: client certificate is required") + return fmt.Errorf("ingress: client certificate is required") } for _, uri := range cs.PeerCertificates[0].URIs { if uri.String() == cfg.AllowedClientID { return nil } } - return fmt.Errorf("atunnel: client is not %q", cfg.AllowedClientID) + return fmt.Errorf("ingress: client is not %q", cfg.AllowedClientID) } s.tlsConfig = &tls.Config{ MinVersion: tls.VersionTLS12, @@ -191,7 +194,7 @@ func newActorProxy(upstream *url.URL, dial DialFunc) *httputil.ReverseProxy { // Prevent untrusted actor responses from spoofing routing rejections // to trigger cache evictions and stream restarts in the ingress router. ModifyResponse: func(resp *http.Response) error { - resp.Header.Del(StaleAssignmentHeader) + resp.Header.Del(staleAssignmentHeader) return nil }, Transport: newProtocolMirrorTransport(dial), @@ -268,11 +271,11 @@ func (t protocolMirrorTransport) CloseIdleConnections() { func loadCredentialBundle(path string) (*tls.Certificate, error) { pemBytes, err := os.ReadFile(path) if err != nil { - return nil, fmt.Errorf("atunnel: reading credential bundle: %w", err) + return nil, fmt.Errorf("ingress: reading credential bundle: %w", err) } cert, err := tls.X509KeyPair(pemBytes, pemBytes) if err != nil { - return nil, fmt.Errorf("atunnel: parsing credential bundle: %w", err) + return nil, fmt.Errorf("ingress: parsing credential bundle: %w", err) } return &cert, nil } @@ -293,7 +296,7 @@ func (s *Server) Serve(ctx context.Context, lis net.Listener) error { // decisions move to the router's route config, leaving atunnel with no // request parsing at all. func (s *Server) ServeConnect(ctx context.Context, lis net.Listener) error { - return s.serve(ctx, lis, http.HandlerFunc(s.ServeConnectHTTP), s.tlsConfig) + return s.serve(ctx, lis, http.HandlerFunc(s.serveConnectHTTP), s.tlsConfig) } func (s *Server) serve(ctx context.Context, lis net.Listener, handler http.Handler, tlsConfig *tls.Config) error { @@ -318,9 +321,9 @@ func (s *Server) serve(ctx context.Context, lis net.Listener, handler http.Handl return err } -// ServeConnectHTTP accepts a router-authenticated CONNECT request and relays +// serveConnectHTTP accepts a router-authenticated CONNECT request and relays // its tunnel to the named port on the currently active actor. -func (s *Server) ServeConnectHTTP(w http.ResponseWriter, r *http.Request) { +func (s *Server) serveConnectHTTP(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodConnect { http.Error(w, "CONNECT required", http.StatusMethodNotAllowed) return @@ -328,7 +331,7 @@ func (s *Server) ServeConnectHTTP(w http.ResponseWriter, r *http.Request) { active, ctx, release, ok := s.authorize(r) if !ok { slog.DebugContext(r.Context(), "atunnel CONNECT rejected", slog.Any("actor_header", r.Header.Values(atenet.TargetActorHeader))) - s.reject(w) + sendRejectResponse(w) return } defer release() @@ -382,7 +385,7 @@ func (s *Server) serveH1Connect(w http.ResponseWriter, upstream net.Conn, ctx co return } - relayIngressWithHalfClose(ctx, log, upstream, rw, downstream, downstream) + relayWithHalfClose(ctx, log, upstream, rw, downstream, downstream) } func (s *Server) serveH2Connect(w http.ResponseWriter, r *http.Request, upstream net.Conn, ctx context.Context, log *slog.Logger) { @@ -394,14 +397,14 @@ func (s *Server) serveH2Connect(w http.ResponseWriter, r *http.Request, upstream return } - relayIngressWithHalfClose(ctx, log, upstream, r.Body, flushingWriter{ResponseWriter: w}, r.Body) + relayWithHalfClose(ctx, log, upstream, r.Body, flushingWriter{ResponseWriter: w}, r.Body) } -// relayIngressWithHalfClose copies a CONNECT stream between the router +// relayWithHalfClose copies a CONNECT stream between the router // (downstream) and the actor (upstream). It half-closes the actor connection // when the router stops sending, and closes the downstream when the actor // does, so a router that reuses tunnels doesn't send into a dead one. -func relayIngressWithHalfClose(ctx context.Context, log *slog.Logger, upstream net.Conn, downstreamReader io.Reader, downstreamWriter io.Writer, downstreamCloser io.Closer) { +func relayWithHalfClose(ctx context.Context, log *slog.Logger, upstream net.Conn, downstreamReader io.Reader, downstreamWriter io.Writer, downstreamCloser io.Closer) { start := time.Now() go func() { n, err := io.Copy(upstream, downstreamReader) @@ -443,10 +446,10 @@ func (w flushingWriter) Write(p []byte) (int, error) { // same name is replaced. func (s *Server) Activate(atespace, actorName, actorUID string, dial DialFunc) error { if !resources.IsValidResourceName(atespace) || !resources.IsValidResourceName(actorName) { - return fmt.Errorf("atunnel: invalid actor reference %q/%q", atespace, actorName) + return fmt.Errorf("ingress: invalid actor reference %q/%q", atespace, actorName) } if dial == nil { - return fmt.Errorf("atunnel: actor dialer is required") + return fmt.Errorf("ingress: actor dialer is required") } ref := resources.ActorRef{Atespace: atespace, Name: actorName} @@ -454,7 +457,7 @@ func (s *Server) Activate(atespace, actorName, actorUID string, dial DialFunc) e defer s.mu.Unlock() if old, ok := s.active[ref]; ok { if old.uid == actorUID { - return fmt.Errorf("atunnel: actor %s is already active", ref) + return fmt.Errorf("ingress: actor %s is already active", ref) } old.cancel() closeIdleUpstreamConnections(old) @@ -521,7 +524,7 @@ func (s *Server) Deactivate(ctx context.Context, atespace, actorName, actorUID s return nil case <-ctx.Done(): closeIdleUpstreamConnections(active) - return fmt.Errorf("atunnel: waiting for active requests to stop: %w", ctx.Err()) + return fmt.Errorf("ingress: waiting for active requests to stop: %w", ctx.Err()) } } @@ -535,7 +538,7 @@ func closeIdleUpstreamConnections(active *activation) { func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { active, requestCtx, release, ok := s.authorize(r) if !ok { - s.reject(w) + sendRejectResponse(w) return } defer release() @@ -573,9 +576,15 @@ func (s *Server) authorize(r *http.Request) (*activation, context.Context, func( // errActorDeactivated is the cancel cause of a request whose actor was // deactivated, which tells a suspend apart from the router hanging up. -var errActorDeactivated = errors.New("atunnel: actor deactivated") +var errActorDeactivated = errors.New("ingress: actor deactivated") -func (s *Server) reject(w http.ResponseWriter) { - w.Header().Set(StaleAssignmentHeader, "true") +func sendRejectResponse(w http.ResponseWriter) { + w.Header().Set(staleAssignmentHeader, "true") http.Error(w, "misdirected request", http.StatusMisdirectedRequest) } + +func closeWrite(conn net.Conn) { + if conn, ok := conn.(interface{ CloseWrite() error }); ok { + _ = conn.CloseWrite() + } +} diff --git a/internal/atunnel/ingress_test.go b/internal/atunnel/ingress/ingress_test.go similarity index 97% rename from internal/atunnel/ingress_test.go rename to internal/atunnel/ingress/ingress_test.go index ca25e299e3..0e715cf3a0 100644 --- a/internal/atunnel/ingress_test.go +++ b/internal/atunnel/ingress/ingress_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package atunnel +package ingress import ( "bytes" @@ -40,9 +40,9 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/atenet" + "go.ate.dev/substrate/internal/atenet" ) func TestActivationDialerClosesLateConnection(t *testing.T) { @@ -131,7 +131,7 @@ func TestDeactivateIsolatesLateDials(t *testing.T) { } } -func TestRelayIngressWithHalfClose(t *testing.T) { +func TestRelayWithHalfClose(t *testing.T) { listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) @@ -158,7 +158,7 @@ func TestRelayIngressWithHalfClose(t *testing.T) { var downstreamOutput bytes.Buffer done := make(chan struct{}) go func() { - relayIngressWithHalfClose(context.Background(), slog.New(slog.DiscardHandler), upstream, downstreamReader, &downstreamOutput, downstreamReader) + relayWithHalfClose(context.Background(), slog.New(slog.DiscardHandler), upstream, downstreamReader, &downstreamOutput, downstreamReader) close(done) }() @@ -207,7 +207,7 @@ func TestRelayIngressClosesDownstreamWhenActorCloses(t *testing.T) { log := slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})) done := make(chan struct{}) go func() { - relayIngressWithHalfClose(context.Background(), log, upstream, downstreamReader, &downstreamOutput, downstreamReader) + relayWithHalfClose(context.Background(), log, upstream, downstreamReader, &downstreamOutput, downstreamReader) close(done) }() @@ -243,7 +243,7 @@ func TestRelayIngressCancellationClosesBothSides(t *testing.T) { log := slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})) done := make(chan struct{}) go func() { - relayIngressWithHalfClose(ctx, log, upstream, downstreamReader, io.Discard, downstreamReader) + relayWithHalfClose(ctx, log, upstream, downstreamReader, io.Discard, downstreamReader) close(done) }() @@ -261,7 +261,7 @@ func TestRelayIngressCancellationClosesBothSides(t *testing.T) { if _, err := io.WriteString(downstreamInput, "request"); err == nil { t.Fatal("downstream remained open after relay cancellation") } - if want := `msg="atunnel CONNECT canceled" cause="atunnel: actor deactivated"`; !strings.Contains(logs.String(), want) { + if want := `msg="atunnel CONNECT canceled" cause="ingress: actor deactivated"`; !strings.Contains(logs.String(), want) { t.Errorf("relay logs = %q, want a line containing %q", logs.String(), want) } } @@ -337,8 +337,8 @@ func TestServeHTTP(t *testing.T) { if rec.Code != tt.wantStatus { t.Fatalf("status = %d, want %d", rec.Code, tt.wantStatus) } - if tt.wantStatus == http.StatusMisdirectedRequest && rec.Header().Get(StaleAssignmentHeader) != "true" { - t.Errorf("missing %s response header", StaleAssignmentHeader) + if tt.wantStatus == http.StatusMisdirectedRequest && rec.Header().Get(staleAssignmentHeader) != "true" { + t.Errorf("missing %s response header", staleAssignmentHeader) } }) } @@ -411,8 +411,8 @@ func TestServerRejectsDuplicateOrCommaTargetActorHeaders(t *testing.T) { if rec.Code != http.StatusMisdirectedRequest { t.Fatalf("status = %d, want %d", rec.Code, http.StatusMisdirectedRequest) } - if rec.Header().Get(StaleAssignmentHeader) != "true" { - t.Errorf("missing %s response header", StaleAssignmentHeader) + if rec.Header().Get(staleAssignmentHeader) != "true" { + t.Errorf("missing %s response header", staleAssignmentHeader) } if upstreamCalled { t.Errorf("upstream was dialed, expected request to be rejected") @@ -430,7 +430,7 @@ func TestServeHTTPStripsStaleAssignmentHeaderFromActor(t *testing.T) { s := newTestServer(t, upstreamURL) actorTransport := roundTripFunc(func(r *http.Request) (*http.Response, error) { h := make(http.Header) - h.Set(StaleAssignmentHeader, "true") + h.Set(staleAssignmentHeader, "true") return &http.Response{ StatusCode: http.StatusOK, Header: h, @@ -450,8 +450,8 @@ func TestServeHTTPStripsStaleAssignmentHeaderFromActor(t *testing.T) { if rec.Code != http.StatusOK { t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK) } - if got := rec.Header().Get(StaleAssignmentHeader); got != "" { - t.Errorf("expected %s to be stripped from actor response, got %q", StaleAssignmentHeader, got) + if got := rec.Header().Get(staleAssignmentHeader); got != "" { + t.Errorf("expected %s to be stripped from actor response, got %q", staleAssignmentHeader, got) } } @@ -546,7 +546,7 @@ func TestServeConnectHTTPValidatesMethodAndAuthority(t *testing.T) { req.Header.Set(atenet.TargetActorHeader, "team-a/actor-2") } rec := httptest.NewRecorder() - s.ServeConnectHTTP(rec, req) + s.serveConnectHTTP(rec, req) if rec.Code != tt.want { t.Fatalf("status = %d, want %d", rec.Code, tt.want) } @@ -1260,7 +1260,7 @@ func TestServeConnectHTTPDialsTheSandbox(t *testing.T) { req.Host = "actor-1.team-a.actors.resources.substrate.ate.dev:9090" req.Header.Set(atenet.TargetActorHeader, "team-a/actor-1") rec := httptest.NewRecorder() - s.ServeConnectHTTP(rec, req) + s.serveConnectHTTP(rec, req) if rec.Code != http.StatusOK { t.Fatalf("status = %d, want %d: the tunnel never reached the sandbox", rec.Code, http.StatusOK) diff --git a/internal/benchmarking/boomer/agentsession/agentsession.go b/internal/benchmarking/boomer/agentsession/agentsession.go index e91873e735..e727a2a335 100644 --- a/internal/benchmarking/boomer/agentsession/agentsession.go +++ b/internal/benchmarking/boomer/agentsession/agentsession.go @@ -40,17 +40,17 @@ import ( "sync" "time" - "github.com/agent-substrate/substrate/internal/ateinterceptors" - "github.com/agent-substrate/substrate/internal/atenet" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/boomerutil" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/dynconfig" - bmetrics "github.com/agent-substrate/substrate/internal/benchmarking/boomer/metrics" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/userclass" - "github.com/agent-substrate/substrate/internal/benchmarking/glutton" - gluttonpb "github.com/agent-substrate/substrate/internal/proto/glutton" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/uuid" "github.com/prometheus/client_golang/prometheus" + "go.ate.dev/substrate/internal/ateinterceptors" + "go.ate.dev/substrate/internal/atenet" + "go.ate.dev/substrate/internal/benchmarking/boomer/boomerutil" + "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" + bmetrics "go.ate.dev/substrate/internal/benchmarking/boomer/metrics" + "go.ate.dev/substrate/internal/benchmarking/boomer/userclass" + "go.ate.dev/substrate/internal/benchmarking/glutton" + gluttonpb "go.ate.dev/substrate/internal/proto/glutton" + "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/propagation" "google.golang.org/grpc" diff --git a/internal/benchmarking/boomer/agentsession/agentsession_test.go b/internal/benchmarking/boomer/agentsession/agentsession_test.go index d1fc69144c..b6f9618d28 100644 --- a/internal/benchmarking/boomer/agentsession/agentsession_test.go +++ b/internal/benchmarking/boomer/agentsession/agentsession_test.go @@ -29,10 +29,10 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/dynconfig" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/userclass" - "github.com/agent-substrate/substrate/internal/benchmarking/glutton/fake" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" + "go.ate.dev/substrate/internal/benchmarking/boomer/userclass" + "go.ate.dev/substrate/internal/benchmarking/glutton/fake" + "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel" "google.golang.org/grpc" "google.golang.org/grpc/codes" diff --git a/internal/benchmarking/boomer/boomerutil/client.go b/internal/benchmarking/boomer/boomerutil/client.go index 01d6e04c33..a27e607b6f 100644 --- a/internal/benchmarking/boomer/boomerutil/client.go +++ b/internal/benchmarking/boomer/boomerutil/client.go @@ -17,9 +17,9 @@ package boomerutil import ( "fmt" - "github.com/agent-substrate/substrate/internal/ateapiauth" - _ "github.com/agent-substrate/substrate/internal/k8sresolver" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/ateapiauth" + _ "go.ate.dev/substrate/internal/k8sresolver" + "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "k8s.io/client-go/kubernetes" diff --git a/internal/benchmarking/boomer/dynconfig/dynconfig_test.go b/internal/benchmarking/boomer/dynconfig/dynconfig_test.go index 8479594179..eff1e5c332 100644 --- a/internal/benchmarking/boomer/dynconfig/dynconfig_test.go +++ b/internal/benchmarking/boomer/dynconfig/dynconfig_test.go @@ -41,7 +41,7 @@ func TestParseValid(t *testing.T) { "durdir_template": "glutton-durdir-data", "cpu_cores": 2, "cpu_duty_cycle": 0.1, - "sweperf_template": "swebench-astropy-7336", + "sweperf_template": "sweperf-astropy-7336", "sweperf_total_steps": 21, "sweperf_num_cycles": 4, "sweperf_poll_interval_ms": 100, @@ -94,8 +94,8 @@ func TestParseValid(t *testing.T) { if cfg.CPUDutyCycle != 0.1 { t.Errorf("CPUDutyCycle: got %f, want 0.1", cfg.CPUDutyCycle) } - if cfg.SweperfTemplate != "swebench-astropy-7336" { - t.Errorf("SweperfTemplate: got %q, want swebench-astropy-7336", cfg.SweperfTemplate) + if cfg.SweperfTemplate != "sweperf-astropy-7336" { + t.Errorf("SweperfTemplate: got %q, want sweperf-astropy-7336", cfg.SweperfTemplate) } if cfg.SweperfTotalSteps != 21 { t.Errorf("SweperfTotalSteps: got %d, want 21", cfg.SweperfTotalSteps) diff --git a/internal/benchmarking/boomer/glutton/cpuload_test.go b/internal/benchmarking/boomer/glutton/cpuload_test.go index aacf4eaddd..162cc70164 100644 --- a/internal/benchmarking/boomer/glutton/cpuload_test.go +++ b/internal/benchmarking/boomer/glutton/cpuload_test.go @@ -19,8 +19,8 @@ import ( "net/http" "testing" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/dynconfig" - "github.com/agent-substrate/substrate/internal/benchmarking/glutton/fake" + "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" + "go.ate.dev/substrate/internal/benchmarking/glutton/fake" ) func TestEnsureCPULoadRequestsConfiguredLoad(t *testing.T) { diff --git a/internal/benchmarking/boomer/glutton/durdir.go b/internal/benchmarking/boomer/glutton/durdir.go index ff9e072f8a..65f6bfa2b2 100644 --- a/internal/benchmarking/boomer/glutton/durdir.go +++ b/internal/benchmarking/boomer/glutton/durdir.go @@ -28,15 +28,15 @@ import ( "sync" "time" - "github.com/agent-substrate/substrate/internal/ateinterceptors" - "github.com/agent-substrate/substrate/internal/atenet" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/boomerutil" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/dynconfig" - bmetrics "github.com/agent-substrate/substrate/internal/benchmarking/boomer/metrics" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/userclass" - gluttonpb "github.com/agent-substrate/substrate/internal/proto/glutton" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/uuid" + "go.ate.dev/substrate/internal/ateinterceptors" + "go.ate.dev/substrate/internal/atenet" + "go.ate.dev/substrate/internal/benchmarking/boomer/boomerutil" + "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" + bmetrics "go.ate.dev/substrate/internal/benchmarking/boomer/metrics" + "go.ate.dev/substrate/internal/benchmarking/boomer/userclass" + gluttonpb "go.ate.dev/substrate/internal/proto/glutton" + "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/propagation" diff --git a/internal/benchmarking/boomer/glutton/durdir_test.go b/internal/benchmarking/boomer/glutton/durdir_test.go index dd1b5d9410..bcc11330cc 100644 --- a/internal/benchmarking/boomer/glutton/durdir_test.go +++ b/internal/benchmarking/boomer/glutton/durdir_test.go @@ -24,11 +24,11 @@ import ( "slices" "testing" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/boomerutil" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/dynconfig" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/userclass" - "github.com/agent-substrate/substrate/internal/benchmarking/glutton/fake" - gluttonpb "github.com/agent-substrate/substrate/internal/proto/glutton" + "go.ate.dev/substrate/internal/benchmarking/boomer/boomerutil" + "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" + "go.ate.dev/substrate/internal/benchmarking/boomer/userclass" + "go.ate.dev/substrate/internal/benchmarking/glutton/fake" + gluttonpb "go.ate.dev/substrate/internal/proto/glutton" ) func TestDurDirLoopSequence(t *testing.T) { diff --git a/internal/benchmarking/boomer/glutton/fixture_test.go b/internal/benchmarking/boomer/glutton/fixture_test.go index 170afe8cca..797aeae4d2 100644 --- a/internal/benchmarking/boomer/glutton/fixture_test.go +++ b/internal/benchmarking/boomer/glutton/fixture_test.go @@ -19,10 +19,10 @@ import ( "sync" "testing" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/dynconfig" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/userclass" - "github.com/agent-substrate/substrate/internal/benchmarking/glutton/fake" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" + "go.ate.dev/substrate/internal/benchmarking/boomer/userclass" + "go.ate.dev/substrate/internal/benchmarking/glutton/fake" + "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel" "google.golang.org/grpc" ) diff --git a/internal/benchmarking/boomer/glutton/lifecycle.go b/internal/benchmarking/boomer/glutton/lifecycle.go index 53c3f76105..e1e13e08b6 100644 --- a/internal/benchmarking/boomer/glutton/lifecycle.go +++ b/internal/benchmarking/boomer/glutton/lifecycle.go @@ -29,15 +29,15 @@ import ( "sync" "time" - "github.com/agent-substrate/substrate/internal/ateinterceptors" - "github.com/agent-substrate/substrate/internal/atenet" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/boomerutil" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/dynconfig" - bmetrics "github.com/agent-substrate/substrate/internal/benchmarking/boomer/metrics" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/userclass" - gluttonpb "github.com/agent-substrate/substrate/internal/proto/glutton" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/uuid" + "go.ate.dev/substrate/internal/ateinterceptors" + "go.ate.dev/substrate/internal/atenet" + "go.ate.dev/substrate/internal/benchmarking/boomer/boomerutil" + "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" + bmetrics "go.ate.dev/substrate/internal/benchmarking/boomer/metrics" + "go.ate.dev/substrate/internal/benchmarking/boomer/userclass" + gluttonpb "go.ate.dev/substrate/internal/proto/glutton" + "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/propagation" diff --git a/internal/benchmarking/boomer/glutton/lifecycle_test.go b/internal/benchmarking/boomer/glutton/lifecycle_test.go index b1e637f38c..8673f5809f 100644 --- a/internal/benchmarking/boomer/glutton/lifecycle_test.go +++ b/internal/benchmarking/boomer/glutton/lifecycle_test.go @@ -19,10 +19,10 @@ import ( "slices" "testing" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/boomerutil" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/dynconfig" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/userclass" - "github.com/agent-substrate/substrate/internal/benchmarking/glutton/fake" + "go.ate.dev/substrate/internal/benchmarking/boomer/boomerutil" + "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" + "go.ate.dev/substrate/internal/benchmarking/boomer/userclass" + "go.ate.dev/substrate/internal/benchmarking/glutton/fake" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) diff --git a/internal/benchmarking/boomer/glutton/memfill_test.go b/internal/benchmarking/boomer/glutton/memfill_test.go index 00485ae8ce..ac3499c028 100644 --- a/internal/benchmarking/boomer/glutton/memfill_test.go +++ b/internal/benchmarking/boomer/glutton/memfill_test.go @@ -18,10 +18,10 @@ import ( "context" "testing" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/dynconfig" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/userclass" - "github.com/agent-substrate/substrate/internal/benchmarking/glutton/fake" - gluttonpb "github.com/agent-substrate/substrate/internal/proto/glutton" + "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" + "go.ate.dev/substrate/internal/benchmarking/boomer/userclass" + "go.ate.dev/substrate/internal/benchmarking/glutton/fake" + gluttonpb "go.ate.dev/substrate/internal/proto/glutton" ) func newTestGluttonActor(t *testing.T, srv *fake.Server, dyn dynconfig.Config) *gluttonActor { diff --git a/internal/benchmarking/boomer/glutton/resume_test.go b/internal/benchmarking/boomer/glutton/resume_test.go index 3cb808fda3..418a7c28bf 100644 --- a/internal/benchmarking/boomer/glutton/resume_test.go +++ b/internal/benchmarking/boomer/glutton/resume_test.go @@ -20,9 +20,9 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/boomerutil" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/userclass" - "github.com/agent-substrate/substrate/internal/benchmarking/glutton/fake" + "go.ate.dev/substrate/internal/benchmarking/boomer/boomerutil" + "go.ate.dev/substrate/internal/benchmarking/boomer/userclass" + "go.ate.dev/substrate/internal/benchmarking/glutton/fake" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) diff --git a/internal/benchmarking/boomer/glutton/spawn.go b/internal/benchmarking/boomer/glutton/spawn.go index d9df58df7e..d14e0a74f8 100644 --- a/internal/benchmarking/boomer/glutton/spawn.go +++ b/internal/benchmarking/boomer/glutton/spawn.go @@ -27,14 +27,14 @@ import ( "sync/atomic" "time" - "github.com/agent-substrate/substrate/internal/ateinterceptors" - "github.com/agent-substrate/substrate/internal/atenet" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/boomerutil" - bmetrics "github.com/agent-substrate/substrate/internal/benchmarking/boomer/metrics" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/userclass" - gluttonpb "github.com/agent-substrate/substrate/internal/proto/glutton" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/uuid" + "go.ate.dev/substrate/internal/ateinterceptors" + "go.ate.dev/substrate/internal/atenet" + "go.ate.dev/substrate/internal/benchmarking/boomer/boomerutil" + bmetrics "go.ate.dev/substrate/internal/benchmarking/boomer/metrics" + "go.ate.dev/substrate/internal/benchmarking/boomer/userclass" + gluttonpb "go.ate.dev/substrate/internal/proto/glutton" + "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/propagation" diff --git a/internal/benchmarking/boomer/glutton/spawn_test.go b/internal/benchmarking/boomer/glutton/spawn_test.go index 0199d8109b..9c33f73bdd 100644 --- a/internal/benchmarking/boomer/glutton/spawn_test.go +++ b/internal/benchmarking/boomer/glutton/spawn_test.go @@ -24,11 +24,11 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/boomerutil" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/dynconfig" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/userclass" - gluttonpb "github.com/agent-substrate/substrate/internal/proto/glutton" "github.com/prometheus/client_golang/prometheus" + "go.ate.dev/substrate/internal/benchmarking/boomer/boomerutil" + "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" + "go.ate.dev/substrate/internal/benchmarking/boomer/userclass" + gluttonpb "go.ate.dev/substrate/internal/proto/glutton" "go.opentelemetry.io/otel" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" diff --git a/internal/benchmarking/boomer/glutton/wait_test.go b/internal/benchmarking/boomer/glutton/wait_test.go index de9313de40..56aef7b17d 100644 --- a/internal/benchmarking/boomer/glutton/wait_test.go +++ b/internal/benchmarking/boomer/glutton/wait_test.go @@ -18,8 +18,8 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/dynconfig" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/userclass" + "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" + "go.ate.dev/substrate/internal/benchmarking/boomer/userclass" ) func TestUniformWaitStaysInRange(t *testing.T) { diff --git a/internal/benchmarking/boomer/resumepaused/resumepaused.go b/internal/benchmarking/boomer/resumepaused/resumepaused.go index 4967d63544..585edef2ef 100644 --- a/internal/benchmarking/boomer/resumepaused/resumepaused.go +++ b/internal/benchmarking/boomer/resumepaused/resumepaused.go @@ -35,11 +35,11 @@ import ( "sync/atomic" "time" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/boomerutil" - bmetrics "github.com/agent-substrate/substrate/internal/benchmarking/boomer/metrics" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/userclass" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/uuid" + "go.ate.dev/substrate/internal/benchmarking/boomer/boomerutil" + bmetrics "go.ate.dev/substrate/internal/benchmarking/boomer/metrics" + "go.ate.dev/substrate/internal/benchmarking/boomer/userclass" + "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel" "golang.org/x/time/rate" "google.golang.org/grpc/codes" diff --git a/internal/benchmarking/boomer/resumepaused/resumepaused_test.go b/internal/benchmarking/boomer/resumepaused/resumepaused_test.go index 7aa1572acf..64dc06759c 100644 --- a/internal/benchmarking/boomer/resumepaused/resumepaused_test.go +++ b/internal/benchmarking/boomer/resumepaused/resumepaused_test.go @@ -27,10 +27,10 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/dynconfig" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/userclass" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/prometheus/client_golang/prometheus/promhttp" + "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" + "go.ate.dev/substrate/internal/benchmarking/boomer/userclass" + "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel" "google.golang.org/grpc" "google.golang.org/grpc/codes" diff --git a/internal/benchmarking/boomer/sweperf/sweperf.go b/internal/benchmarking/boomer/sweperf/sweperf.go index 37dd467bb1..598b1523d3 100644 --- a/internal/benchmarking/boomer/sweperf/sweperf.go +++ b/internal/benchmarking/boomer/sweperf/sweperf.go @@ -15,7 +15,8 @@ // Package sweperf implements the boomer-Go implementation of the // SweperfUser locust test. Each user creates an actor from a SWE-bench workload // template and drives it through a trajectory of steps that are partitioned into -// cycles. This is workload-agnostic and can be used to benchmark any SWE-bench +// cycles. Once the trajectory is done the actor is deleted and a fresh one is +// created to run it again. This is workload-agnostic and can be used to benchmark any SWE-bench // workload by providing the appropriate template and dynamic configuration. package sweperf @@ -32,15 +33,16 @@ import ( "net/http" "strings" "sync" + "sync/atomic" "time" - "github.com/agent-substrate/substrate/internal/ateinterceptors" - "github.com/agent-substrate/substrate/internal/atenet" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/boomerutil" - bmetrics "github.com/agent-substrate/substrate/internal/benchmarking/boomer/metrics" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/userclass" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/uuid" + "go.ate.dev/substrate/internal/ateinterceptors" + "go.ate.dev/substrate/internal/atenet" + "go.ate.dev/substrate/internal/benchmarking/boomer/boomerutil" + bmetrics "go.ate.dev/substrate/internal/benchmarking/boomer/metrics" + "go.ate.dev/substrate/internal/benchmarking/boomer/userclass" + "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/propagation" @@ -60,7 +62,7 @@ const ( // environment supplies a value. const ( sweperfUserClass = "SweperfUser" - defaultSweperfTemplate = "swebench-astropy-7336" + defaultSweperfTemplate = "sweperf-astropy-7336" defaultSweperfTotalSteps = 21 defaultSweperfNumCycles = 4 // defaultSweperfPollInterval trades CycleCEL-to-ack accuracy against @@ -126,24 +128,75 @@ func generateDynamicChunks(totalSteps int, numCycles int) []chunk { // initSweperf creates a runtime tied to cfg and returns a boomer-compatible // task function plus a shutdown hook the caller should run before exit (it -// suspend+deletes every actor this worker created). +// deletes every actor this worker created). func initSweperf(cfg *userclass.Config) (taskFn func(), shutdown func(context.Context)) { if cfg.Tracer == nil { cfg.Tracer = otel.Tracer("substrate-boomer/sweperf") } - rt := &sweperfRuntime{ - cfg: cfg, - } + rt := newSweperfRuntime(cfg) return rt.iterate, rt.shutdown } +// newSweperfRuntime returns a runtime whose run context lives until shutdown. +func newSweperfRuntime(cfg *userclass.Config) *sweperfRuntime { + ctx, cancel := context.WithCancel(context.Background()) + return &sweperfRuntime{cfg: cfg, runCtx: ctx, cancelRun: cancel} +} + // sweperfRuntime is the per-worker state shared by every boomer goroutine. // Each goroutine keeps its own session in users, keyed by goroutine ID, // because boomer offers no per-VU context. type sweperfRuntime struct { cfg *userclass.Config users sync.Map // goroutineID -> *sweperfUser + // runCtx carries every startup and cycle; shutdown cancels it so a slow + // liveness poll or cycle does not hold up teardown. + runCtx context.Context + cancelRun context.CancelFunc + // mu orders enter against shutdown, so no inflight.Add can race the + // inflight.Wait in shutdown. + mu sync.Mutex + // stopping is set once by shutdown, under mu, and never cleared. + stopping bool + // inflight counts iterate calls in progress, each of which may create or + // delete an actor. shutdown waits for them before sweeping users. + inflight sync.WaitGroup + // atespaceReady is set once CreateAtespace succeeds, so later actors skip + // it. Cleared when CreateActor reports the atespace missing, so the next + // start recreates it. + atespaceReady atomic.Bool +} + +// enter registers an iterate call with inflight, or reports false once +// shutdown has begun. A true result must be paired with inflight.Done. +func (r *sweperfRuntime) enter() bool { + r.mu.Lock() + defer r.mu.Unlock() + if r.stopping { + return false + } + r.inflight.Add(1) + return true +} + +// teardownTimeout bounds deleting an actor once the run context may already +// be canceled. A var so tests can shorten it. +var teardownTimeout = 30 * time.Second + +// teardownCtx returns a context for deleting an actor that outlives the run +// context, so cancellation by shutdown does not strand the actor. +func (r *sweperfRuntime) teardownCtx() (context.Context, context.CancelFunc) { + return context.WithTimeout(context.WithoutCancel(r.runCtx), teardownTimeout) +} + +// canceled reports whether ctx was canceled, which only happens when shutdown +// cuts a call short, so the call says nothing about the system under test. +// It checks the context, not the error: gRPC returns a codes.Canceled status +// that errors.Is does not match. Timeouts are DeadlineExceeded, so teardown +// calls that run out of time are still recorded. +func canceled(ctx context.Context) bool { + return errors.Is(ctx.Err(), context.Canceled) } // resolveConfig returns the template, total step count and cycle count for a @@ -194,44 +247,62 @@ func (r *sweperfRuntime) dynamicWait() time.Duration { // iterate is the boomer task function, one call per goroutine per iteration. // It binds a session to the calling goroutine on first use and runs one cycle -// per call thereafter, looping back to the first cycle when the trace is -// exhausted so the actor keeps serving load. A session that fails to start is -// retried on the next iteration. +// per call thereafter. When the trace is exhausted the session's actor is +// deleted and the goroutine is unbound, so the next call starts a fresh actor +// that runs the trajectory again. A session that fails to start is retried on +// the next iteration. Once shutdown has begun no new actor is created. func (r *sweperfRuntime) iterate() { + if !r.enter() { + return + } + r.runCycle() + time.Sleep(r.dynamicWait()) +} + +// runCycle is the part of iterate that shutdown waits for: it may start, +// cycle or delete an actor, but excludes the think time. +func (r *sweperfRuntime) runCycle() { + defer r.inflight.Done() gid := boomerutil.GoroutineID() val, loaded := r.users.Load(gid) + user, _ := val.(*sweperfUser) if !loaded { - u, err := r.startUser(context.Background()) - if err != nil { - slog.Warn("sweperf on_start failed; goroutine will retry next iter", - slog.String("err", err.Error())) - time.Sleep(r.dynamicWait()) + if user = r.bindUser(gid); user == nil { return } - val, _ = r.users.LoadOrStore(gid, u) } - user := val.(*sweperfUser) + if r.runCtx.Err() != nil { + return // shutdown began; its sweep tears the actor down + } - ctx := context.Background() - user.step(ctx) + user.step(r.runCtx) if user.isDone() { - user.recordTaskMetrics() - user.resetCycles() - slog.Info("Actor finished all cycles and reset session back to cycle 1", - slog.String("actor", user.actorName), - slog.Int("cycles", len(user.chunks)), - ) + if r.runCtx.Err() == nil { // a task finished as shutdown began records nothing + user.recordTaskMetrics() + } + r.retireUser(gid, user) } +} - time.Sleep(r.dynamicWait()) +// bindUser starts a session and publishes it for goroutine gid. It returns +// nil if the session failed to start. shutdown waits for the calling +// runCycle, so it always sees a published session. +func (r *sweperfRuntime) bindUser(gid int64) *sweperfUser { + u, err := r.startUser(r.runCtx) + if err != nil { + return nil // startUser logged it + } + r.users.Store(gid, u) + return u } // startUser creates an actor and blocks until its sandbox serves, so the // caller gets a session that is ready to take cycles. Resolves the config per // session, which lets a value changed mid-run apply to later sessions. On -// failure it tears the actor down. -func (r *sweperfRuntime) startUser(ctx context.Context) (*sweperfUser, error) { +// failure it tears the actor down and logs the error with the actor and +// template, so a failed start can be tied to its task. +func (r *sweperfRuntime) startUser(ctx context.Context) (_ *sweperfUser, err error) { tmpl, totalSteps, numCycles := r.resolveConfig() chunks := generateDynamicChunks(totalSteps, numCycles) @@ -248,22 +319,46 @@ func (r *sweperfRuntime) startUser(ctx context.Context) (*sweperfUser, error) { slog.String("actor", u.actorName), slog.String("template", u.templateName), ) + defer func() { + if err != nil { + slog.Warn("sweperf on_start failed; goroutine will retry next iter", + slog.String("actor", u.actorName), + slog.String("template", u.templateName), + slog.String("err", err.Error()), + ) + } + }() bmetrics.UpdateUsers(u.userClass, 1) - if err := u.ensureAtespace(ctx); err != nil { - bmetrics.UpdateUsers(u.userClass, -1) - return nil, fmt.Errorf("ensureAtespace: %w", err) + if !r.atespaceReady.Load() { + if err := u.ensureAtespace(ctx); err != nil { + bmetrics.UpdateUsers(u.userClass, -1) + return nil, fmt.Errorf("ensureAtespace: %w", err) + } + r.atespaceReady.Store(true) } - if err := u.create(ctx); err != nil { + // CreateActor runs detached from ctx: canceled mid-call, the actor might + // exist without this session knowing to delete it. + createCtx, cancelCreate := r.teardownCtx() + err = u.create(createCtx) + cancelCreate() + if err != nil { + // ate-api answers FailedPrecondition when the atespace is missing. + if status.Code(err) == codes.FailedPrecondition { + r.atespaceReady.Store(false) + } bmetrics.UpdateUsers(u.userClass, -1) return nil, fmt.Errorf("createActor: %w", err) } // poll status for liveness if err := u.pollLiveness(ctx); err != nil { - u.suspendAndDelete(ctx) - bmetrics.UpdateUsers(u.userClass, -1) + // ctx may be canceled by shutdown; tear down regardless. + tctx, cancel := r.teardownCtx() + defer cancel() + // teardown also decrements the user gauge. + u.teardown(tctx) return nil, fmt.Errorf("pollLiveness: %w", err) } @@ -274,23 +369,98 @@ func (r *sweperfRuntime) startUser(ctx context.Context) (*sweperfUser, error) { return u, nil } -// shutdown suspends and deletes every actor this worker created. Boomer has -// no per-VU stop hook, so a mid-run decrease in user count leaks actors until -// this runs — acceptable for a run that ramps up, holds, then tears down. +// retireUser deletes the actor of a session that finished its trajectory and +// unbinds it from goroutine gid, so the next iterate starts a fresh actor. The +// final cycle leaves the actor running, and DeleteActor with AnyState removes +// it from any state, so no suspend is sent first. The delete runs detached +// from the run context so shutdown cannot cut it short. A failed delete is +// retried once, with a fresh timeout; if that also fails the actor is left +// behind and logged. +func (r *sweperfRuntime) retireUser(gid int64, u *sweperfUser) { + r.users.CompareAndDelete(gid, u) + if !u.cleanedUp.CompareAndSwap(false, true) { + return // shutdown already owns this actor's teardown + } + bmetrics.UpdateUsers(u.userClass, -1) + deleteActor := func() error { + ctx, cancel := r.teardownCtx() + defer cancel() + return u.delete(ctx) + } + if err := deleteActor(); err != nil { + if err := deleteActor(); err != nil { + slog.Error("DeleteActor failed twice for finished actor; it is leaked", + slog.String("actor", u.actorName), + slog.String("err", err.Error()), + ) + return + } + } + slog.Info("Actor finished all cycles and was deleted; a new actor will run the trajectory next", + slog.String("actor", u.actorName), + slog.Int("cycles", len(u.chunks)), + ) +} + +// inflightWaitTimeout caps how long shutdown waits for in-flight iterations. +// They are canceled and normally end at once; the cap leaves the rest of the +// shutdown budget for the sweep. A var so tests can shorten it. +var inflightWaitTimeout = 10 * time.Second + +// shutdownConcurrency bounds how many actors shutdown deletes at once. +// In-cluster the pod's 30s termination grace period, not boomer-worker's +// 60s shutdown context, is the real budget, and a serial sweep over thousands +// of sessions would overrun it. +const shutdownConcurrency = 64 + +// shutdown stops new iterations, cancels in-flight startups and cycles, waits +// up to inflightWaitTimeout for them to finish, then deletes every actor still +// bound to a goroutine, shutdownConcurrency at a time, until done or ctx +// expires. Boomer has no per-VU stop hook, so a mid-run decrease in user +// count leaks actors until this runs — acceptable for a run that ramps up, +// holds, then tears down. func (r *sweperfRuntime) shutdown(ctx context.Context) { + slog.Info("sweperf shutdown started") + r.mu.Lock() + r.stopping = true + r.mu.Unlock() + r.cancelRun() + + waitCtx, cancelWait := context.WithTimeout(ctx, inflightWaitTimeout) + defer cancelWait() + done := make(chan struct{}) + go func() { + r.inflight.Wait() + close(done) + }() + select { + case <-done: + case <-waitCtx.Done(): + slog.Warn("shutdown timed out waiting for in-flight iterations; their actors may leak") + } + + var wg sync.WaitGroup + sem := make(chan struct{}, shutdownConcurrency) r.users.Range(func(_, val any) bool { - u := val.(*sweperfUser) - if !u.cleanedUp { - u.suspendAndDelete(ctx) - u.cleanedUp = true + select { + case sem <- struct{}{}: + case <-ctx.Done(): + return false } + wg.Add(1) + go func(u *sweperfUser) { + defer wg.Done() + defer func() { <-sem }() + u.teardown(ctx) + }(val.(*sweperfUser)) return true }) + wg.Wait() } // sweperfUser is one benchmark session: a single actor, the cycle plan it -// replays, and its progress through that plan. Owned by one boomer goroutine, -// so its fields need no locking. +// runs once, and its progress through that plan. Owned by one boomer goroutine, +// so its fields need no locking, except cleanedUp, which shutdown also claims. type sweperfUser struct { cfg *userclass.Config actorName string @@ -298,7 +468,9 @@ type sweperfUser struct { userClass string chunks []chunk cycleIndex int - cleanedUp bool + // cleanedUp is claimed by whichever of the owning goroutine or shutdown + // tears the actor down first, so the actor is deleted once. + cleanedUp atomic.Bool // awake is set while the actor is still running from pollLiveness; the // next resume is then a no-op and its success samples are not recorded. awake bool @@ -390,9 +562,9 @@ func (u *sweperfUser) suspend(ctx context.Context) bool { } // delete removes the actor. Errors are recorded as a failed DeleteActor row -// and otherwise ignored, since the only caller is already tearing down. -func (u *sweperfUser) delete(ctx context.Context) { - _ = u.tracedCall(ctx, "DeleteActor", func(callCtx context.Context, tr *metadata.MD) error { +// and returned so the caller can retry. +func (u *sweperfUser) delete(ctx context.Context) error { + return u.tracedCall(ctx, "DeleteActor", func(callCtx context.Context, tr *metadata.MD) error { _, err := u.cfg.APIStub.DeleteActor(callCtx, &ateapipb.DeleteActorRequest{ Actor: u.ref(), // Teardown discards the actor, so skip the SUSPENDED precondition. @@ -402,20 +574,16 @@ func (u *sweperfUser) delete(ctx context.Context) { }) } -// suspendAndDelete releases the actor and the worker it holds, and decrements -// the user gauge. It suspends first because a worker only frees an actor it -// can account for, and an actor still awake at delete risks being left -// CRASHED. Safe to call more than once: it is a no-op after the first. -func (u *sweperfUser) suspendAndDelete(ctx context.Context) { - if u.cleanedUp { +// teardown releases the actor and the worker it holds, and decrements the +// user gauge. Like retireUser, it deletes with AnyState and sends no suspend +// first. Safe to call more than once and from several goroutines: only the +// first call does anything. +func (u *sweperfUser) teardown(ctx context.Context) { + if !u.cleanedUp.CompareAndSwap(false, true) { return } - _, _ = u.cfg.APIStub.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ - Actor: u.ref(), - }) - u.delete(ctx) + _ = u.delete(ctx) bmetrics.UpdateUsers(u.userClass, -1) - u.cleanedUp = true } // tracedCall runs one control-plane RPC under a span named name and records @@ -437,6 +605,9 @@ func (u *sweperfUser) tracedCallRecord(ctx context.Context, name string, recordS var tr metadata.MD err := do(ctx, &tr) clientLatency := time.Since(start) + if err != nil && canceled(ctx) { + return err + } latency, source := boomerutil.ElapsedFromMD(tr, ateinterceptors.ServerElapsedTrailer, clientLatency) if source == boomerutil.SourceServer { @@ -471,15 +642,20 @@ type statusResponse struct { Status string `json:"status"` } +// livenessTimeout bounds how long pollLiveness waits for a new actor. +const livenessTimeout = time.Minute + // pollLiveness waits for the actor's in-sandbox server to answer /status with -// "up", for up to a minute. The first request through the router is also what -// wakes a newly created actor, so this doubles as the implicit resume. +// "up", for up to livenessTimeout. The first request through the router is +// also what wakes a newly created actor, so this doubles as the implicit +// resume. It polls at the job-completion interval, since every task starts +// a fresh actor. func (u *sweperfUser) pollLiveness(ctx context.Context) error { statusURL := u.cfg.RouterURL + "/status" - maxRetries := 30 - retryInterval := 2 * time.Second + retryInterval := pollInterval(u.cfg) + deadline := time.Now().Add(livenessTimeout) - for attempt := 0; attempt < maxRetries; attempt++ { + for attempt := 0; time.Now().Before(deadline); attempt++ { req, err := http.NewRequestWithContext(ctx, http.MethodGet, statusURL, nil) if err != nil { return err @@ -517,14 +693,9 @@ func (u *sweperfUser) isDone() bool { return u.cycleIndex >= len(u.chunks) } -// resetCycles rewinds to the first cycle so the same actor replays the trace -// again, which is how one session keeps producing load for the whole run. -func (u *sweperfUser) resetCycles() { - u.cycleIndex = 0 -} - // step runs one cycle: resume the actor, replay that cycle's slice of the -// trace, then suspend it again. +// trace, then suspend it again. The final cycle skips the suspend, since the +// actor is deleted straight after. func (u *sweperfUser) step(ctx context.Context) { if u.cycleIndex >= len(u.chunks) { return @@ -545,7 +716,11 @@ func (u *sweperfUser) step(ctx context.Context) { resumeStart := time.Now() // Summed per cycle so the between-cycle think time stays out of the total. defer func() { u.loopWall += time.Since(resumeStart) }() + // A cycle cut short by shutdown returns without recording or advancing. if !u.resume(ctx) { + if canceled(ctx) { + return + } u.loopFailed = true bmetrics.RecordFailure(derivedMetricMethod, resumeToFirstExecMetric, u.userClass, time.Since(resumeStart), "ResumeActor failed") @@ -556,6 +731,9 @@ func (u *sweperfUser) step(ctx context.Context) { // 2. Execute step range chunk inside sandbox ackAt, execDur, err := u.execute(ctx, cycleNum, chunk.start, chunk.end) + if err != nil && canceled(ctx) { + return + } if err != nil { u.loopFailed = true slog.Error("execute steps failed", @@ -572,8 +750,11 @@ func (u *sweperfUser) step(ctx context.Context) { } u.recordCycleCEL(execDur, err) - // 3. Suspend actor - if !u.suspend(ctx) { + // 3. Suspend actor, unless this was the last cycle + if cycleNum < len(u.chunks) && !u.suspend(ctx) { + if canceled(ctx) { + return + } u.loopFailed = true } @@ -759,14 +940,19 @@ func (u *sweperfUser) execute(ctx context.Context, cycleNum int, startIdx int, e // httpJSONCall posts body to route on the actor's in-sandbox server and // records the result under metricName. validate inspects the response body and -// decides the outcome. +// decides the outcome. A call canceled by shutdown records no failure. func (u *sweperfUser) httpJSONCall(ctx context.Context, metricName, route string, body []byte, validate func([]byte) error) ([]byte, error) { ctx, span := u.cfg.Tracer.Start(ctx, metricName) defer span.End() + recordFailure := func(latency time.Duration, msg string) { + if !canceled(ctx) { + bmetrics.RecordFailure("http", metricName, u.userClass, latency, msg) + } + } httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, u.cfg.RouterURL+route, bytes.NewReader(body)) if err != nil { - bmetrics.RecordFailure("http", metricName, u.userClass, 0, err.Error()) + recordFailure(0, err.Error()) return nil, err } u.setActorRouting(httpReq) @@ -777,14 +963,14 @@ func (u *sweperfUser) httpJSONCall(ctx context.Context, metricName, route string resp, err := u.cfg.HTTPClient.Do(httpReq) clientLatency := time.Since(start) if err != nil { - bmetrics.RecordFailure("http", metricName, u.userClass, clientLatency, err.Error()) + recordFailure(clientLatency, err.Error()) return nil, err } defer resp.Body.Close() respBody, readErr := io.ReadAll(resp.Body) if readErr != nil { - bmetrics.RecordFailure("http", metricName, u.userClass, clientLatency, readErr.Error()) + recordFailure(clientLatency, readErr.Error()) return nil, readErr } @@ -796,7 +982,7 @@ func (u *sweperfUser) httpJSONCall(ctx context.Context, metricName, route string if resp.StatusCode >= 400 { httpErr := fmt.Errorf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(respBody))) boomerutil.LogSampledTrace(span, metricName, clientLatency, boomerutil.SourceClient, httpErr) - bmetrics.RecordFailure("http", metricName, u.userClass, clientLatency, httpErr.Error()) + recordFailure(clientLatency, httpErr.Error()) return nil, httpErr } @@ -804,7 +990,7 @@ func (u *sweperfUser) httpJSONCall(ctx context.Context, metricName, route string if err := validate(respBody); err != nil { elapsed := time.Since(start) boomerutil.LogSampledTrace(span, metricName, elapsed, boomerutil.SourceClient, err) - bmetrics.RecordFailure("http", metricName, u.userClass, elapsed, err.Error()) + recordFailure(elapsed, err.Error()) return nil, err } } diff --git a/internal/benchmarking/boomer/sweperf/sweperf_test.go b/internal/benchmarking/boomer/sweperf/sweperf_test.go index 0993c52589..98698991d8 100644 --- a/internal/benchmarking/boomer/sweperf/sweperf_test.go +++ b/internal/benchmarking/boomer/sweperf/sweperf_test.go @@ -15,10 +15,12 @@ package sweperf import ( + "bytes" "context" "encoding/json" "errors" "fmt" + "log/slog" "net/http" "net/http/httptest" "reflect" @@ -27,13 +29,13 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/ateinterceptors" - "github.com/agent-substrate/substrate/internal/atenet" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/boomerutil" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/dynconfig" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/userclass" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/prometheus/client_golang/prometheus" + "go.ate.dev/substrate/internal/ateinterceptors" + "go.ate.dev/substrate/internal/atenet" + "go.ate.dev/substrate/internal/benchmarking/boomer/boomerutil" + "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" + "go.ate.dev/substrate/internal/benchmarking/boomer/userclass" + "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -107,6 +109,8 @@ type fakeControlClient struct { resumeElapsedUs string // AnyState carried by the most recent DeleteActor request. deleteAnyState bool + // beforeDelete, if set, runs at the start of each DeleteActor, outside mu. + beforeDelete func() } func (f *fakeControlClient) CreateAtespace(ctx context.Context, in *ateapipb.CreateAtespaceRequest, opts ...grpc.CallOption) (*ateapipb.Atespace, error) { @@ -163,10 +167,16 @@ func (f *fakeControlClient) SuspendActor(ctx context.Context, in *ateapipb.Suspe } func (f *fakeControlClient) DeleteActor(ctx context.Context, in *ateapipb.DeleteActorRequest, opts ...grpc.CallOption) (*ateapipb.Actor, error) { + if f.beforeDelete != nil { + f.beforeDelete() + } f.mu.Lock() defer f.mu.Unlock() f.calls = append(f.calls, "DeleteActor") f.deleteAnyState = in.GetAnyState() + if err := ctx.Err(); err != nil { + return nil, err + } if f.deleteErr != nil { return nil, f.deleteErr } @@ -425,6 +435,43 @@ func TestEnsureAtespaceHandling(t *testing.T) { }) } +func TestAtespaceCreatedOncePerWorker(t *testing.T) { + cfg, _, fakeCtrl := newTestConfig(t, asyncJobHandler()) + r := newSweperfRuntime(cfg) + countCreateAtespace := func() int { + n := 0 + for _, c := range fakeCtrl.recordedCalls() { + if c == "CreateAtespace" { + n++ + } + } + return n + } + + for range 2 { + if _, err := r.startUser(context.Background()); err != nil { + t.Fatalf("startUser: %v", err) + } + } + if n := countCreateAtespace(); n != 1 { + t.Errorf("CreateAtespace sent %d times for two actors, want 1", n) + } + + // A missing atespace comes back from CreateActor as FailedPrecondition; + // the next start recreates it. + fakeCtrl.createActorErr = status.Error(codes.FailedPrecondition, "Atespace benchmark-test not found") + if _, err := r.startUser(context.Background()); err == nil { + t.Fatalf("startUser succeeded despite CreateActor failing") + } + fakeCtrl.createActorErr = nil + if _, err := r.startUser(context.Background()); err != nil { + t.Fatalf("startUser: %v", err) + } + if n := countCreateAtespace(); n != 2 { + t.Errorf("CreateAtespace sent %d times, want 2 after the atespace went missing", n) + } +} + func TestControlClientErrors(t *testing.T) { t.Run("ResumeActor error returns false", func(t *testing.T) { cfg, _, fakeCtrl := newTestConfig(t, http.HandlerFunc(nil)) @@ -517,6 +564,66 @@ func TestSweperfPollLiveness(t *testing.T) { t.Errorf("pollLiveness expected error on timeout, got nil") } }) + + t.Run("retries at the sweperf poll interval", func(t *testing.T) { + var mu sync.Mutex + calls := 0 + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + calls++ + n := calls + mu.Unlock() + if n < 3 { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + json.NewEncoder(w).Encode(statusResponse{Status: "up"}) + }) + cfg, _, _ := newTestConfig(t, handler) + cfg.Dyn = dynconfig.NewHolder(dynconfig.Config{SweperfPollIntervalMs: 10}) + u := &sweperfUser{cfg: cfg, actorName: "act"} + + start := time.Now() + if err := u.pollLiveness(context.Background()); err != nil { + t.Fatalf("pollLiveness: %v", err) + } + // Two retries at a fixed 2s interval would take 4s. + if d := time.Since(start); d > time.Second { + t.Errorf("pollLiveness took %v for two retries at a 10ms interval", d) + } + }) +} + +func TestStartupFailureLogsActorAndTemplate(t *testing.T) { + cfg, _, fakeCtrl := newTestConfig(t, asyncJobHandler()) + fakeCtrl.createActorErr = errors.New("quota exceeded") + r := newSweperfRuntime(cfg) + + var buf bytes.Buffer + prev := slog.Default() + slog.SetDefault(slog.New(slog.NewJSONHandler(&buf, nil))) + t.Cleanup(func() { slog.SetDefault(prev) }) + + if _, err := r.startUser(context.Background()); err == nil { + t.Fatalf("startUser succeeded despite CreateActor failing") + } + + var failure map[string]any + for _, line := range strings.Split(strings.TrimSpace(buf.String()), "\n") { + var rec map[string]any + if json.Unmarshal([]byte(line), &rec) == nil && strings.Contains(fmt.Sprint(rec["msg"]), "on_start failed") { + failure = rec + } + } + if failure == nil { + t.Fatalf("no startup failure logged; got:\n%s", buf.String()) + } + if actor, _ := failure["actor"].(string); !strings.HasPrefix(actor, "sb-") { + t.Errorf("failure log actor = %q, want the new actor's name", actor) + } + if got := failure["template"]; got != defaultSweperfTemplate { + t.Errorf("failure log template = %v, want %q", got, defaultSweperfTemplate) + } } func TestSweperfPollJobCompletion(t *testing.T) { @@ -556,7 +663,7 @@ func TestSweperfPollJobCompletion(t *testing.T) { }) } -func TestSweperfUserLifecycleAndReset(t *testing.T) { +func TestSweperfUserIsDone(t *testing.T) { u := &sweperfUser{ chunks: []chunk{{0, 5}, {5, 10}}, cycleIndex: 0, @@ -570,11 +677,6 @@ func TestSweperfUserLifecycleAndReset(t *testing.T) { if !u.isDone() { t.Errorf("isDone = false, want true") } - - u.resetCycles() - if u.cycleIndex != 0 || u.isDone() { - t.Errorf("resetCycles failed: cycleIndex = %d, isDone = %v", u.cycleIndex, u.isDone()) - } } func TestDynamicWait(t *testing.T) { @@ -660,7 +762,7 @@ func asyncJobHandler() http.HandlerFunc { func TestNoopFirstResumeNotRecorded(t *testing.T) { cfg, _, fakeCtrl := newTestConfig(t, asyncJobHandler()) fakeCtrl.resumeElapsedUs = "5000" - rt := &sweperfRuntime{cfg: cfg} + rt := newSweperfRuntime(cfg) u, err := rt.startUser(context.Background()) if err != nil { t.Fatalf("startUser: %v", err) @@ -690,16 +792,6 @@ func TestNoopFirstResumeNotRecorded(t *testing.T) { t.Errorf("cycle 2: %s recorded %d samples, want 1", name, got.count) } } - - // The next pass starts from a real suspend, so its first resume counts. - u.resetCycles() - before = readStats(t) - u.step(context.Background()) - for _, name := range resumeRows { - if got := recordedSince(t, before, name, "success"); got.count != 1 { - t.Errorf("next pass cycle 1: %s recorded %d samples, want 1", name, got.count) - } - } } func TestNoopFirstResumeFailureStillRecorded(t *testing.T) { @@ -722,7 +814,7 @@ func TestNoopFirstResumeFailureStillRecorded(t *testing.T) { } } -func TestSweperfBootstrapFailureSuspendsBeforeDelete(t *testing.T) { +func TestStartupFailureDeletesWithoutSuspend(t *testing.T) { handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "server not ready", http.StatusInternalServerError) }) @@ -731,19 +823,18 @@ func TestSweperfBootstrapFailureSuspendsBeforeDelete(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() - rt := &sweperfRuntime{cfg: cfg} + rt := newSweperfRuntime(cfg) _, err := rt.startUser(ctx) if err == nil { t.Fatalf("startUser expected error on liveness failure, got nil") } - calls := fakeCtrl.recordedCalls() - if len(calls) < 2 || calls[len(calls)-2] != "SuspendActor" || calls[len(calls)-1] != "DeleteActor" { - t.Errorf("recordedCalls must end with [SuspendActor, DeleteActor], got %v", calls) + if got, want := fakeCtrl.recordedCalls(), []string{"CreateAtespace", "CreateActor", "DeleteActor"}; !reflect.DeepEqual(got, want) { + t.Errorf("recordedCalls = %v, want %v", got, want) } } -func TestSweperfShutdownSuspendsBeforeDelete(t *testing.T) { +func TestShutdownDeletesWithoutSuspend(t *testing.T) { cfg, _, fakeCtrl := newTestConfig(t, http.HandlerFunc(nil)) u := &sweperfUser{ @@ -752,13 +843,12 @@ func TestSweperfShutdownSuspendsBeforeDelete(t *testing.T) { userClass: sweperfUserClass, } - rt := &sweperfRuntime{cfg: cfg} + rt := newSweperfRuntime(cfg) rt.users.Store("goroutine-1", u) rt.shutdown(context.Background()) - calls := fakeCtrl.recordedCalls() - if len(calls) < 2 || calls[len(calls)-2] != "SuspendActor" || calls[len(calls)-1] != "DeleteActor" { - t.Errorf("recordedCalls must end with [SuspendActor, DeleteActor], got %v", calls) + if got, want := fakeCtrl.recordedCalls(), []string{"DeleteActor"}; !reflect.DeepEqual(got, want) { + t.Errorf("recordedCalls = %v, want %v", got, want) } // Without AnyState the server refuses to delete an actor caught mid-cycle. if !fakeCtrl.deleteAnyState { @@ -869,7 +959,7 @@ func TestTaskCELOnlyOnFullTrajectory(t *testing.T) { } }) cfg, _, _ := newTestConfig(t, handler) - r := &sweperfRuntime{cfg: cfg} + r := newSweperfRuntime(cfg) u := &sweperfUser{ cfg: cfg, actorName: "act", @@ -903,9 +993,129 @@ func TestTaskCELOnlyOnFullTrajectory(t *testing.T) { if gotWall := recordedSince(t, before, "TaskWallClock", "success"); gotWall.count != 1 { t.Errorf("TaskWallClock recorded %d samples, want 1", gotWall.count) } - if u.loopCEL != 0 || u.loopWall != 0 || u.loopFailed || u.cycleIndex != 0 { - t.Errorf("after the task loopCEL=%v loopWall=%v loopFailed=%v cycleIndex=%d, want all rearmed", - u.loopCEL, u.loopWall, u.loopFailed, u.cycleIndex) + if u.loopCEL != 0 || u.loopWall != 0 || u.loopFailed { + t.Errorf("after the task loopCEL=%v loopWall=%v loopFailed=%v, want all rearmed", + u.loopCEL, u.loopWall, u.loopFailed) + } +} + +func TestFinishedTrajectoryDeletesActorAndStartsFresh(t *testing.T) { + cfg, _, fakeCtrl := newTestConfig(t, asyncJobHandler()) + r := newSweperfRuntime(cfg) + gid := boomerutil.GoroutineID() + u := &sweperfUser{ + cfg: cfg, + actorName: "act-1", + userClass: sweperfUserClass, + chunks: []chunk{{0, 5}}, + } + r.users.Store(gid, u) + + // The single cycle finishes the trajectory, so the actor must go. + r.iterate() + if !u.cleanedUp.Load() { + t.Errorf("cleanedUp = false after the trajectory finished; the actor was not deleted") + } + // The final cycle is not suspended; the actor is deleted straight away. + if got, want := fakeCtrl.recordedCalls(), []string{"ResumeActor", "DeleteActor"}; !reflect.DeepEqual(got, want) { + t.Errorf("recordedCalls = %v, want %v", got, want) + } + if _, ok := r.users.Load(gid); ok { + t.Fatalf("finished session still bound to its goroutine; the next iterate would reuse it") + } + + // The next iterate creates a brand-new actor for the next trajectory. + before := len(fakeCtrl.recordedCalls()) + r.iterate() + val, ok := r.users.Load(gid) + if !ok { + t.Fatalf("no session bound after the next iterate; want a fresh actor") + } + next := val.(*sweperfUser) + if next == u || next.actorName == u.actorName { + t.Errorf("next session reuses actor %q, want a new one", u.actorName) + } + newCalls := fakeCtrl.recordedCalls()[before:] + if len(newCalls) < 2 || newCalls[0] != "CreateAtespace" || newCalls[1] != "CreateActor" { + t.Errorf("next iterate calls = %v, want to start with CreateAtespace, CreateActor", newCalls) + } + r.shutdown(context.Background()) +} + +func TestRetireRetriesFailedDeleteOnce(t *testing.T) { + cfg, _, fakeCtrl := newTestConfig(t, asyncJobHandler()) + fakeCtrl.deleteErr = errors.New("unavailable") + r := newSweperfRuntime(cfg) + u := &sweperfUser{cfg: cfg, actorName: "act-1", userClass: sweperfUserClass, chunks: []chunk{{0, 5}}} + r.users.Store(boomerutil.GoroutineID(), u) + + r.iterate() + + // One retry, then the actor is given up on rather than retried forever. + want := []string{"ResumeActor", "DeleteActor", "DeleteActor"} + if got := fakeCtrl.recordedCalls(); !reflect.DeepEqual(got, want) { + t.Errorf("recordedCalls = %v, want %v", got, want) + } +} + +func TestRetireRetryGetsFreshTeardownCtx(t *testing.T) { + prev := teardownTimeout + teardownTimeout = 50 * time.Millisecond + t.Cleanup(func() { teardownTimeout = prev }) + + cfg, _, fakeCtrl := newTestConfig(t, asyncJobHandler()) + // The first DeleteActor outlasts its teardown context; the retry is quick. + var once sync.Once + fakeCtrl.beforeDelete = func() { once.Do(func() { time.Sleep(100 * time.Millisecond) }) } + r := newSweperfRuntime(cfg) + u := &sweperfUser{cfg: cfg, actorName: "act-1", userClass: sweperfUserClass, chunks: []chunk{{0, 5}}} + r.users.Store(boomerutil.GoroutineID(), u) + + before := readStats(t) + r.iterate() + + if got := recordedSince(t, before, "DeleteActor", "failure"); got.count != 1 { + t.Errorf("DeleteActor recorded %d failures, want 1 for the timed-out first attempt", got.count) + } + if got := recordedSince(t, before, "DeleteActor", "success"); got.count != 1 { + t.Errorf("DeleteActor recorded %d successes, want the retry to succeed on a fresh context", got.count) + } +} + +func TestOnlyFinalCycleSkipsSuspend(t *testing.T) { + cfg, _, fakeCtrl := newTestConfig(t, asyncJobHandler()) + r := newSweperfRuntime(cfg) + u := &sweperfUser{cfg: cfg, actorName: "act-1", userClass: sweperfUserClass, chunks: []chunk{{0, 2}, {2, 5}}} + r.users.Store(boomerutil.GoroutineID(), u) + + r.iterate() + r.iterate() + + want := []string{"ResumeActor", "SuspendActor", "ResumeActor", "DeleteActor"} + if got := fakeCtrl.recordedCalls(); !reflect.DeepEqual(got, want) { + t.Errorf("recordedCalls = %v, want %v", got, want) + } +} + +func TestShutdownAndRetireDeleteOnce(t *testing.T) { + cfg, _, fakeCtrl := newTestConfig(t, asyncJobHandler()) + r := newSweperfRuntime(cfg) + gid := boomerutil.GoroutineID() + u := &sweperfUser{cfg: cfg, actorName: "act-1", userClass: sweperfUserClass, chunks: []chunk{{0, 5}}} + r.users.Store(gid, u) + + // Shutdown claims the actor first; a late retire must not delete it again. + r.shutdown(context.Background()) + r.retireUser(gid, u) + + n := 0 + for _, c := range fakeCtrl.recordedCalls() { + if c == "DeleteActor" { + n++ + } + } + if n != 1 { + t.Errorf("DeleteActor sent %d times, want 1", n) } } @@ -956,7 +1166,8 @@ func TestStepSuspendFailureFailsTask(t *testing.T) { cfg: cfg, actorName: "act", userClass: sweperfUserClass, - chunks: []chunk{{0, 5}}, + // Two cycles: the final one is never suspended. + chunks: []chunk{{0, 2}, {2, 5}}, } u.step(context.Background()) @@ -991,3 +1202,279 @@ func TestStepSyncExitSkipsResumeToFirstExec(t *testing.T) { t.Errorf("loopFailed = false after the command exited non-zero") } } + +// usersGauge reads the locust_users gauge for the sweperf user class. The +// registry is global to the test binary, so tests using it must not run in +// parallel. +func usersGauge(t *testing.T) float64 { + t.Helper() + families, err := prometheus.DefaultGatherer.Gather() + if err != nil { + t.Fatalf("gather metrics: %v", err) + } + for _, mf := range families { + if mf.GetName() != "locust_users" { + continue + } + for _, m := range mf.GetMetric() { + for _, lp := range m.GetLabel() { + if lp.GetName() == "user_class" && lp.GetValue() == sweperfUserClass { + return m.GetGauge().GetValue() + } + } + } + } + return 0 +} + +func TestPollLivenessFailureDecrementsUsersOnce(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "server not ready", http.StatusInternalServerError) + }) + cfg, _, _ := newTestConfig(t, handler) + r := newSweperfRuntime(cfg) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + before := usersGauge(t) + if _, err := r.startUser(ctx); err == nil { + t.Fatalf("startUser expected error on liveness failure, got nil") + } + if got := usersGauge(t); got != before { + t.Errorf("locust_users = %v after a failed start, want %v unchanged", got, before) + } +} + +func TestNoNewActorAfterShutdown(t *testing.T) { + cfg, _, fakeCtrl := newTestConfig(t, asyncJobHandler()) + r := newSweperfRuntime(cfg) + + r.shutdown(context.Background()) + r.iterate() + + if calls := fakeCtrl.recordedCalls(); len(calls) != 0 { + t.Errorf("iterate after shutdown sent %v, want no calls", calls) + } + if _, ok := r.users.Load(boomerutil.GoroutineID()); ok { + t.Errorf("iterate after shutdown bound a session") + } +} + +func TestShutdownCancelsSlowStartup(t *testing.T) { + entered := make(chan struct{}) + var once sync.Once + handler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + once.Do(func() { close(entered) }) + <-req.Context().Done() // the sandbox never comes up + }) + cfg, _, fakeCtrl := newTestConfig(t, handler) + r := newSweperfRuntime(cfg) + + before := usersGauge(t) + go r.iterate() + <-entered + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + start := time.Now() + r.shutdown(ctx) + if d := time.Since(start); d > 2*time.Second { + t.Errorf("shutdown took %v; the liveness poll was not canceled", d) + } + + if got, want := fakeCtrl.recordedCalls(), []string{"CreateAtespace", "CreateActor", "DeleteActor"}; !reflect.DeepEqual(got, want) { + t.Errorf("recordedCalls = %v, want %v: the starting actor deleted despite cancellation", got, want) + } + if got := usersGauge(t); got != before { + t.Errorf("locust_users = %v after shutdown, want %v unchanged", got, before) + } +} + +func TestTracedCallSkipsFailureWhenCanceled(t *testing.T) { + cfg, _, fakeCtrl := newTestConfig(t, http.HandlerFunc(nil)) + // gRPC reports cancellation as a status, which errors.Is does not match. + fakeCtrl.resumeErr = status.Error(codes.Canceled, "context canceled") + fakeCtrl.resumeElapsedUs = "5000" + u := &sweperfUser{cfg: cfg, actorName: "act", userClass: sweperfUserClass} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + before := readStats(t) + if ok := u.resume(ctx); ok { + t.Fatalf("resume = true on RPC failure, want false") + } + for _, name := range []string{"ResumeActor", "ResumeActor_rtt"} { + if got := recordedSince(t, before, name, "failure"); got.count != 0 { + t.Errorf("%s recorded %d failures for a canceled call, want 0", name, got.count) + } + } +} + +func TestShutdownMidCycleRecordsNoFailures(t *testing.T) { + entered := make(chan struct{}) + var once sync.Once + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/execute": + json.NewEncoder(w).Encode(executeResponse{JobID: "job-1", Status: "ACCEPTED"}) + case r.URL.Path == "/status" && r.URL.Query().Get("job_id") != "": + once.Do(func() { close(entered) }) + <-r.Context().Done() // the job never finishes + default: + http.NotFound(w, r) + } + }) + cfg, _, fakeCtrl := newTestConfig(t, handler) + cfg.Dyn = dynconfig.NewHolder(dynconfig.Config{SweperfPollIntervalMs: 10}) + r := newSweperfRuntime(cfg) + // The last cycle, so an interrupted task would also emit task rows. + u := &sweperfUser{cfg: cfg, actorName: "act-1", userClass: sweperfUserClass, chunks: []chunk{{0, 5}}} + + before := readStats(t) + go func() { + r.users.Store(boomerutil.GoroutineID(), u) + r.iterate() + }() + <-entered // the cycle is polling its job + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + r.shutdown(ctx) + + for _, name := range []string{ + "ResumeActor", "ResumeActor_rtt", "ResumeToFirstExec", "Workload_Cycle_1", + "CycleCEL", "TaskCEL", "TaskWallClock", "SuspendActor", "DeleteActor", + } { + if got := recordedSince(t, before, name, "failure"); got.count != 0 { + t.Errorf("%s recorded %d failures after shutdown canceled the cycle, want 0", name, got.count) + } + } + if got, want := fakeCtrl.recordedCalls(), []string{"ResumeActor", "DeleteActor"}; !reflect.DeepEqual(got, want) { + t.Errorf("recordedCalls = %v, want %v: the interrupted actor deleted by the sweep", got, want) + } +} + +func TestShutdownWaitsForRetire(t *testing.T) { + cfg, _, fakeCtrl := newTestConfig(t, asyncJobHandler()) + entered := make(chan struct{}) + release := make(chan struct{}) + var once sync.Once + fakeCtrl.beforeDelete = func() { + once.Do(func() { close(entered) }) + <-release + } + unblock := sync.OnceFunc(func() { close(release) }) + t.Cleanup(unblock) + r := newSweperfRuntime(cfg) + u := &sweperfUser{cfg: cfg, actorName: "act-1", userClass: sweperfUserClass, chunks: []chunk{{0, 5}}} + + // Sessions are keyed by goroutine, so bind u on the goroutine that iterates. + go func() { + r.users.Store(boomerutil.GoroutineID(), u) + r.iterate() + }() + <-entered // retireUser is inside DeleteActor + + done := make(chan struct{}) + go func() { + r.shutdown(context.Background()) + close(done) + }() + select { + case <-done: + t.Fatalf("shutdown returned mid-DeleteActor; the process would exit and leak the actor") + case <-time.After(50 * time.Millisecond): + } + + unblock() + <-done + n := 0 + for _, c := range fakeCtrl.recordedCalls() { + if c == "DeleteActor" { + n++ + } + } + if n != 1 { + t.Errorf("DeleteActor sent %d times, want 1", n) + } +} + +func TestTeardownClaimsOnce(t *testing.T) { + cfg, _, fakeCtrl := newTestConfig(t, http.HandlerFunc(nil)) + u := &sweperfUser{cfg: cfg, actorName: "act", userClass: sweperfUserClass} + + // Shutdown's sweep and a goroutine it stopped waiting for may both try to + // tear the actor down; only one may. + var wg sync.WaitGroup + for range 8 { + wg.Add(1) + go func() { + defer wg.Done() + u.teardown(context.Background()) + }() + } + wg.Wait() + + n := 0 + for _, c := range fakeCtrl.recordedCalls() { + if c == "DeleteActor" { + n++ + } + } + if n != 1 { + t.Errorf("DeleteActor sent %d times, want 1", n) + } +} + +func TestShutdownSweepsConcurrently(t *testing.T) { + cfg, _, fakeCtrl := newTestConfig(t, http.HandlerFunc(nil)) + fakeCtrl.beforeDelete = func() { time.Sleep(50 * time.Millisecond) } + r := newSweperfRuntime(cfg) + const n = 20 + for i := range n { + r.users.Store(int64(i), &sweperfUser{cfg: cfg, actorName: fmt.Sprintf("act-%d", i), userClass: sweperfUserClass}) + } + + start := time.Now() + r.shutdown(context.Background()) + + // A serial sweep would take n * 50ms = 1s. + if d := time.Since(start); d > 500*time.Millisecond { + t.Errorf("shutdown took %v for %d actors; the sweep is not concurrent", d, n) + } + deletes := 0 + for _, c := range fakeCtrl.recordedCalls() { + if c == "DeleteActor" { + deletes++ + } + } + if deletes != n { + t.Errorf("DeleteActor sent %d times, want %d", deletes, n) + } +} + +func TestShutdownSweepsAfterInflightWaitCap(t *testing.T) { + prev := inflightWaitTimeout + inflightWaitTimeout = 50 * time.Millisecond + t.Cleanup(func() { inflightWaitTimeout = prev }) + + cfg, _, fakeCtrl := newTestConfig(t, http.HandlerFunc(nil)) + r := newSweperfRuntime(cfg) + r.users.Store(int64(1), &sweperfUser{cfg: cfg, actorName: "act-1", userClass: sweperfUserClass}) + // An iteration that never finishes. + r.inflight.Add(1) + t.Cleanup(r.inflight.Done) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + start := time.Now() + r.shutdown(ctx) + + if d := time.Since(start); d > time.Second { + t.Errorf("shutdown took %v; the in-flight wait used the whole budget", d) + } + if got, want := fakeCtrl.recordedCalls(), []string{"DeleteActor"}; !reflect.DeepEqual(got, want) { + t.Errorf("recordedCalls = %v, want the bound actor swept despite the stuck iteration", got) + } +} diff --git a/internal/benchmarking/boomer/userclass/config.go b/internal/benchmarking/boomer/userclass/config.go index 6e73314a6b..28789ba3ec 100644 --- a/internal/benchmarking/boomer/userclass/config.go +++ b/internal/benchmarking/boomer/userclass/config.go @@ -18,8 +18,8 @@ import ( "net/http" "time" - "github.com/agent-substrate/substrate/internal/benchmarking/boomer/dynconfig" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/benchmarking/boomer/dynconfig" + "go.ate.dev/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel/trace" ) diff --git a/internal/benchmarking/glutton/cpu_load.go b/internal/benchmarking/glutton/cpu_load.go index 32833519f1..69174472f7 100644 --- a/internal/benchmarking/glutton/cpu_load.go +++ b/internal/benchmarking/glutton/cpu_load.go @@ -26,7 +26,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - gluttonpb "github.com/agent-substrate/substrate/internal/proto/glutton" + gluttonpb "go.ate.dev/substrate/internal/proto/glutton" ) // UseCPU replaces the currently-running CPU load pool. A num_cores of 0 diff --git a/internal/benchmarking/glutton/cpu_load_test.go b/internal/benchmarking/glutton/cpu_load_test.go index b9afa25cea..ca6a5ca1a3 100644 --- a/internal/benchmarking/glutton/cpu_load_test.go +++ b/internal/benchmarking/glutton/cpu_load_test.go @@ -24,7 +24,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - gluttonpb "github.com/agent-substrate/substrate/internal/proto/glutton" + gluttonpb "go.ate.dev/substrate/internal/proto/glutton" ) // TestUseCPULifecycle covers start, replace, and stop through a series of diff --git a/internal/benchmarking/glutton/fake/server.go b/internal/benchmarking/glutton/fake/server.go index 15bc983f93..49d9c19536 100644 --- a/internal/benchmarking/glutton/fake/server.go +++ b/internal/benchmarking/glutton/fake/server.go @@ -24,9 +24,9 @@ import ( "sync" "testing" - "github.com/agent-substrate/substrate/internal/ateinterceptors" - "github.com/agent-substrate/substrate/internal/benchmarking/glutton" - gluttonpb "github.com/agent-substrate/substrate/internal/proto/glutton" + "go.ate.dev/substrate/internal/ateinterceptors" + "go.ate.dev/substrate/internal/benchmarking/glutton" + gluttonpb "go.ate.dev/substrate/internal/proto/glutton" "google.golang.org/protobuf/proto" ) diff --git a/internal/benchmarking/glutton/gossip.go b/internal/benchmarking/glutton/gossip.go index 6af172e72a..588e02b3ea 100644 --- a/internal/benchmarking/glutton/gossip.go +++ b/internal/benchmarking/glutton/gossip.go @@ -29,7 +29,7 @@ import ( "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/status" - gluttonpb "github.com/agent-substrate/substrate/internal/proto/glutton" + gluttonpb "go.ate.dev/substrate/internal/proto/glutton" ) type peerGossip struct { diff --git a/internal/benchmarking/glutton/ram_test.go b/internal/benchmarking/glutton/ram_test.go index 468f565b3f..cf6a35caeb 100644 --- a/internal/benchmarking/glutton/ram_test.go +++ b/internal/benchmarking/glutton/ram_test.go @@ -19,7 +19,7 @@ import ( "context" "testing" - gluttonpb "github.com/agent-substrate/substrate/internal/proto/glutton" + gluttonpb "go.ate.dev/substrate/internal/proto/glutton" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) diff --git a/internal/benchmarking/glutton/server.go b/internal/benchmarking/glutton/server.go index 17908ac50d..03f625d285 100644 --- a/internal/benchmarking/glutton/server.go +++ b/internal/benchmarking/glutton/server.go @@ -34,8 +34,8 @@ import ( "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" - "github.com/agent-substrate/substrate/internal/ateinterceptors" - gluttonpb "github.com/agent-substrate/substrate/internal/proto/glutton" + "go.ate.dev/substrate/internal/ateinterceptors" + gluttonpb "go.ate.dev/substrate/internal/proto/glutton" ) // Handler builds the request handler for the given wire mode. ModeGRPC serves diff --git a/internal/benchmarking/glutton/server_test.go b/internal/benchmarking/glutton/server_test.go index 78d6cbea75..98b979a625 100644 --- a/internal/benchmarking/glutton/server_test.go +++ b/internal/benchmarking/glutton/server_test.go @@ -30,8 +30,8 @@ import ( "google.golang.org/grpc/credentials/insecure" "google.golang.org/protobuf/proto" - "github.com/agent-substrate/substrate/internal/ateinterceptors" - gluttonpb "github.com/agent-substrate/substrate/internal/proto/glutton" + "go.ate.dev/substrate/internal/ateinterceptors" + gluttonpb "go.ate.dev/substrate/internal/proto/glutton" ) // TestSplitGRPCServesReadyzAndGRPCOnOneListener starts the grpc-mode handler diff --git a/internal/benchmarking/glutton/service.go b/internal/benchmarking/glutton/service.go index ecf7f24d3b..bd765324ec 100644 --- a/internal/benchmarking/glutton/service.go +++ b/internal/benchmarking/glutton/service.go @@ -34,7 +34,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - gluttonpb "github.com/agent-substrate/substrate/internal/proto/glutton" + gluttonpb "go.ate.dev/substrate/internal/proto/glutton" ) // diskKeyRE rejects anything that could escape the data dir or hit a diff --git a/internal/benchmarking/glutton/service_test.go b/internal/benchmarking/glutton/service_test.go index f2e1fbc6de..d38aa5f991 100644 --- a/internal/benchmarking/glutton/service_test.go +++ b/internal/benchmarking/glutton/service_test.go @@ -25,7 +25,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - gluttonpb "github.com/agent-substrate/substrate/internal/proto/glutton" + gluttonpb "go.ate.dev/substrate/internal/proto/glutton" ) func TestWriteDiskReadDiskRoundTrip(t *testing.T) { diff --git a/internal/benchmarking/glutton/session_ops_test.go b/internal/benchmarking/glutton/session_ops_test.go index 6e89e59b4b..a4778c6049 100644 --- a/internal/benchmarking/glutton/session_ops_test.go +++ b/internal/benchmarking/glutton/session_ops_test.go @@ -26,7 +26,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - gluttonpb "github.com/agent-substrate/substrate/internal/proto/glutton" + gluttonpb "go.ate.dev/substrate/internal/proto/glutton" ) // newTestService builds a Service rooted in a per-test temp dir. diff --git a/internal/benchmarking/glutton/sizes_test.go b/internal/benchmarking/glutton/sizes_test.go index a7d0c1497d..5d8d8a1286 100644 --- a/internal/benchmarking/glutton/sizes_test.go +++ b/internal/benchmarking/glutton/sizes_test.go @@ -18,7 +18,7 @@ import ( "context" "testing" - gluttonpb "github.com/agent-substrate/substrate/internal/proto/glutton" + gluttonpb "go.ate.dev/substrate/internal/proto/glutton" ) func TestParseBytes(t *testing.T) { diff --git a/internal/contextlogging/contextlogging.go b/internal/contextlogging/contextlogging.go index af784b6229..2713537467 100644 --- a/internal/contextlogging/contextlogging.go +++ b/internal/contextlogging/contextlogging.go @@ -22,8 +22,8 @@ import ( "go.opentelemetry.io/otel/trace" "google.golang.org/protobuf/proto" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/protoredact" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/protoredact" ) type ContextHandler struct { diff --git a/internal/contextlogging/contextlogging_test.go b/internal/contextlogging/contextlogging_test.go index 29eb97b8d8..72c031e7bf 100644 --- a/internal/contextlogging/contextlogging_test.go +++ b/internal/contextlogging/contextlogging_test.go @@ -26,9 +26,9 @@ import ( "go.opentelemetry.io/otel/trace" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/protoredact" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/protoredact" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/reflect/protoreflect" ) diff --git a/internal/e2e/clients.go b/internal/e2e/clients.go index 4495812700..49927a5189 100644 --- a/internal/e2e/clients.go +++ b/internal/e2e/clients.go @@ -19,8 +19,8 @@ import ( "fmt" "time" - "github.com/agent-substrate/substrate/internal/ateclient" - "github.com/agent-substrate/substrate/pkg/client/clientset/versioned" + "go.ate.dev/substrate/internal/ateclient" + "go.ate.dev/substrate/pkg/client/clientset/versioned" apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/client-go/kubernetes" ) diff --git a/internal/e2e/collector_metrics.go b/internal/e2e/collector_metrics.go index 218ead3a52..4ba3e0c7b4 100644 --- a/internal/e2e/collector_metrics.go +++ b/internal/e2e/collector_metrics.go @@ -23,8 +23,8 @@ import ( "strings" "time" - "github.com/agent-substrate/substrate/internal/ateclient" - "github.com/agent-substrate/substrate/internal/portforward" + "go.ate.dev/substrate/internal/ateclient" + "go.ate.dev/substrate/internal/portforward" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" diff --git a/internal/e2e/credprovider.go b/internal/e2e/credprovider.go index e219182786..bd18c9d819 100644 --- a/internal/e2e/credprovider.go +++ b/internal/e2e/credprovider.go @@ -18,7 +18,7 @@ import ( "path/filepath" "testing" - "github.com/agent-substrate/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/installdefaults" ) // TODO(yufan-su): Move these helpers into an internal/e2e/credprovider package. diff --git a/internal/e2e/egresspolicy.go b/internal/e2e/egresspolicy.go index dae98f054b..dc8c98048b 100644 --- a/internal/e2e/egresspolicy.go +++ b/internal/e2e/egresspolicy.go @@ -21,8 +21,8 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // TODO(yufan-su): Move these constructors into an internal/e2e/egresspolicy package. diff --git a/internal/e2e/env.go b/internal/e2e/env.go index 40184026b6..0491442528 100644 --- a/internal/e2e/env.go +++ b/internal/e2e/env.go @@ -18,7 +18,7 @@ import ( "fmt" "os" - "github.com/agent-substrate/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/installdefaults" ) // CheckEnv checks the list of env vars exist and returns their value. diff --git a/internal/e2e/fixture.go b/internal/e2e/fixture.go index b2767cbc67..4c35de5bc3 100644 --- a/internal/e2e/fixture.go +++ b/internal/e2e/fixture.go @@ -20,7 +20,7 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/encoding/protojson" diff --git a/internal/e2e/fixtures/capabilities/capabilities-templates.yaml.tmpl b/internal/e2e/fixtures/capabilities/capabilities-templates.yaml.tmpl index e783dce184..c0fe161dc2 100644 --- a/internal/e2e/fixtures/capabilities/capabilities-templates.yaml.tmpl +++ b/internal/e2e/fixtures/capabilities/capabilities-templates.yaml.tmpl @@ -29,7 +29,7 @@ workerSelector: workload: caps containers: - name: probe - image: ko://github.com/agent-substrate/substrate/internal/e2e/fixtures/probe + image: ko://go.ate.dev/substrate/internal/e2e/fixtures/probe command: ["/ko-app/probe"] wakeupProbe: httpGet: @@ -59,7 +59,7 @@ workerSelector: workload: caps containers: - name: probe - image: ko://github.com/agent-substrate/substrate/internal/e2e/fixtures/probe + image: ko://go.ate.dev/substrate/internal/e2e/fixtures/probe command: ["/ko-app/probe"] # The probe binds :80, which is exactly what NET_BIND_SERVICE permits, so # this template also proves the granted capability is usable and not merely diff --git a/internal/e2e/fixtures/probe/main_test.go b/internal/e2e/fixtures/probe/main_test.go index cf10505e29..94fff20588 100644 --- a/internal/e2e/fixtures/probe/main_test.go +++ b/internal/e2e/fixtures/probe/main_test.go @@ -27,7 +27,7 @@ import ( "github.com/google/go-cmp/cmp" - "github.com/agent-substrate/substrate/internal/testcert" + "go.ate.dev/substrate/internal/testcert" ) // The capabilities e2e assertions are only as trustworthy as this decoder and diff --git a/internal/e2e/fixtures/probe/probe-multiactor-template.yaml.tmpl b/internal/e2e/fixtures/probe/probe-multiactor-template.yaml.tmpl index 2cf48d5028..fdfa031a94 100644 --- a/internal/e2e/fixtures/probe/probe-multiactor-template.yaml.tmpl +++ b/internal/e2e/fixtures/probe/probe-multiactor-template.yaml.tmpl @@ -22,7 +22,7 @@ workerSelector: workload: probe-multiactor containers: - name: probe - image: ko://github.com/agent-substrate/substrate/internal/e2e/fixtures/probe + image: ko://go.ate.dev/substrate/internal/e2e/fixtures/probe command: ["/ko-app/probe"] volumeMounts: # Identity returned by /whoami. diff --git a/internal/e2e/fixtures/probe/probe-parking-template.yaml.tmpl b/internal/e2e/fixtures/probe/probe-parking-template.yaml.tmpl index c7ca64d87f..120513cfe0 100644 --- a/internal/e2e/fixtures/probe/probe-parking-template.yaml.tmpl +++ b/internal/e2e/fixtures/probe/probe-parking-template.yaml.tmpl @@ -25,7 +25,7 @@ workerSelector: workload: probe-parking containers: - name: probe - image: ko://github.com/agent-substrate/substrate/internal/e2e/fixtures/probe + image: ko://go.ate.dev/substrate/internal/e2e/fixtures/probe command: ["/ko-app/probe"] volumeMounts: - name: system-info diff --git a/internal/e2e/fixtures/probe/probe-sized-template.yaml.tmpl b/internal/e2e/fixtures/probe/probe-sized-template.yaml.tmpl index 1638fc4c9a..b1ed5d078f 100644 --- a/internal/e2e/fixtures/probe/probe-sized-template.yaml.tmpl +++ b/internal/e2e/fixtures/probe/probe-sized-template.yaml.tmpl @@ -25,7 +25,7 @@ workerSelector: workload: probe-sized containers: - name: probe - image: ko://github.com/agent-substrate/substrate/internal/e2e/fixtures/probe + image: ko://go.ate.dev/substrate/internal/e2e/fixtures/probe command: ["/ko-app/probe"] # Gate the golden snapshot on the probe actually serving, so the single # (un-retried) GET /resources the suite makes after resuming cannot race a diff --git a/internal/e2e/fixtures/probe/probe-template.yaml.tmpl b/internal/e2e/fixtures/probe/probe-template.yaml.tmpl index da362a14cb..a5e22aebab 100644 --- a/internal/e2e/fixtures/probe/probe-template.yaml.tmpl +++ b/internal/e2e/fixtures/probe/probe-template.yaml.tmpl @@ -26,7 +26,7 @@ workerSelector: workload: probe${FIXTURE_SUFFIX} containers: - name: probe - image: ko://github.com/agent-substrate/substrate/internal/e2e/fixtures/probe + image: ko://go.ate.dev/substrate/internal/e2e/fixtures/probe command: ["/ko-app/probe"] volumeMounts: # the probe reads actor metadata and trust bundles under /run/ate diff --git a/internal/e2e/fixtures/security/atespace-template.yaml.tmpl b/internal/e2e/fixtures/security/atespace-template.yaml.tmpl index f635ddcc5f..ae64bf69a2 100644 --- a/internal/e2e/fixtures/security/atespace-template.yaml.tmpl +++ b/internal/e2e/fixtures/security/atespace-template.yaml.tmpl @@ -37,7 +37,7 @@ workerSelector: workload: ate-e2e-security containers: - name: probe - image: ko://github.com/agent-substrate/substrate/internal/e2e/fixtures/probe + image: ko://go.ate.dev/substrate/internal/e2e/fixtures/probe command: ["/ko-app/probe"] volumeMounts: # The probe expects its identity files under /run/ate. The mount also makes diff --git a/internal/e2e/fixtures/testserver/common_test.go b/internal/e2e/fixtures/testserver/common_test.go index a64d46153c..9a5098903f 100644 --- a/internal/e2e/fixtures/testserver/common_test.go +++ b/internal/e2e/fixtures/testserver/common_test.go @@ -24,7 +24,7 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/testcert" + "go.ate.dev/substrate/internal/testcert" ) type originCertificate struct { diff --git a/internal/e2e/fixtures/testserver/egressprobe.go b/internal/e2e/fixtures/testserver/egressprobe.go index a91541b505..535f0075bb 100644 --- a/internal/e2e/fixtures/testserver/egressprobe.go +++ b/internal/e2e/fixtures/testserver/egressprobe.go @@ -28,8 +28,8 @@ import ( "github.com/spf13/cobra" - "github.com/agent-substrate/substrate/internal/atunnel" - "github.com/agent-substrate/substrate/internal/credbundle" + "go.ate.dev/substrate/internal/atunnel/egress" + "go.ate.dev/substrate/internal/credbundle" ) // tunnelDestination is the CONNECT authority. atunnel takes this from @@ -122,7 +122,7 @@ func handshake(w http.ResponseWriter, r *http.Request, cfg probeConfig) { if err != nil { result.Stage = stage result.Error = err.Error() - var rejected *atunnel.ConnectRejectedError + var rejected *egress.ConnectRejectedError if errors.As(err, &rejected) { result.ConnectStatus = rejected.StatusCode } @@ -145,7 +145,7 @@ func encodeChain(chain []*x509.Certificate) string { // chain it was served. On failure it also returns the stage that failed, since // which hop said no is the assertion most callers are actually making. func fetchChain(ctx context.Context, sni, credentialBundle string, cfg probeConfig) ([]*x509.Certificate, string, error) { - client, err := atunnel.NewClient(atunnel.ClientConfig{ + client, err := egress.NewClient(egress.ClientConfig{ GatewayAddress: cfg.gatewayAddress, ServerName: serverName(cfg.gatewayAddress), GetClientCertificate: credbundle.ClientLoader(credentialBundle), @@ -183,9 +183,9 @@ func fetchChain(ctx context.Context, sni, credentialBundle string, cfg probeConf // typed errors rather than its messages, so the mapping survives a reworded // error. func dialStage(err error) string { - var rejected *atunnel.ConnectRejectedError + var rejected *egress.ConnectRejectedError switch { - case errors.Is(err, atunnel.ErrGatewayHandshake): + case errors.Is(err, egress.ErrGatewayHandshake): return stageGatewayTLS case errors.As(err, &rejected): return stageConnect diff --git a/internal/e2e/fixtures/testserver/egressprobe.yaml.tmpl b/internal/e2e/fixtures/testserver/egressprobe.yaml.tmpl index 7dc0b2f676..9c4a973aae 100644 --- a/internal/e2e/fixtures/testserver/egressprobe.yaml.tmpl +++ b/internal/e2e/fixtures/testserver/egressprobe.yaml.tmpl @@ -30,7 +30,7 @@ spec: restartPolicy: Never containers: - name: egressprobe - image: ko://github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver + image: ko://go.ate.dev/substrate/internal/e2e/fixtures/testserver args: - "egressprobe" - "--listen=:8080" diff --git a/internal/e2e/fixtures/testserver/grpc.go b/internal/e2e/fixtures/testserver/grpc.go index ce80f37d75..14cb6ca572 100644 --- a/internal/e2e/fixtures/testserver/grpc.go +++ b/internal/e2e/fixtures/testserver/grpc.go @@ -30,7 +30,7 @@ import ( healthpb "google.golang.org/grpc/health/grpc_health_v1" "google.golang.org/grpc/status" - "github.com/agent-substrate/substrate/internal/proto/grpcechopb" + "go.ate.dev/substrate/internal/proto/grpcechopb" ) // maxStreamCount bounds EchoStream. A test asks for a handful of messages; a diff --git a/internal/e2e/fixtures/testserver/grpc_test.go b/internal/e2e/fixtures/testserver/grpc_test.go index 71d6430f23..35d5b1c8c4 100644 --- a/internal/e2e/fixtures/testserver/grpc_test.go +++ b/internal/e2e/fixtures/testserver/grpc_test.go @@ -31,7 +31,7 @@ import ( healthpb "google.golang.org/grpc/health/grpc_health_v1" "google.golang.org/grpc/status" - "github.com/agent-substrate/substrate/internal/proto/grpcechopb" + "go.ate.dev/substrate/internal/proto/grpcechopb" ) // The e2e suite reads this fixture's answers as evidence about the egress diff --git a/internal/e2e/fixtures/testserver/grpcecho-template.yaml.tmpl b/internal/e2e/fixtures/testserver/grpcecho-template.yaml.tmpl index 8f81a7081d..e894e82c4e 100644 --- a/internal/e2e/fixtures/testserver/grpcecho-template.yaml.tmpl +++ b/internal/e2e/fixtures/testserver/grpcecho-template.yaml.tmpl @@ -28,7 +28,7 @@ workerSelector: workload: grpcecho${FIXTURE_SUFFIX} containers: - name: grpcecho - image: ko://github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver + image: ko://go.ate.dev/substrate/internal/e2e/fixtures/testserver command: ["/ko-app/testserver"] # gRPC on the Actor's primary port, so the ingress test reaches it at the # Actor's DNS name with no CONNECT and no port juggling -- the plain path diff --git a/internal/e2e/fixtures/testserver/websocket-template.yaml.tmpl b/internal/e2e/fixtures/testserver/websocket-template.yaml.tmpl index 18617008f3..16e8d95f33 100644 --- a/internal/e2e/fixtures/testserver/websocket-template.yaml.tmpl +++ b/internal/e2e/fixtures/testserver/websocket-template.yaml.tmpl @@ -24,7 +24,7 @@ workerSelector: workload: websocket${FIXTURE_SUFFIX} containers: - name: websocket - image: ko://github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver + image: ko://go.ate.dev/substrate/internal/e2e/fixtures/testserver args: ["websocket"] wakeupProbe: httpGet: diff --git a/internal/e2e/pod.go b/internal/e2e/pod.go index baf9bff29f..ed608e1511 100644 --- a/internal/e2e/pod.go +++ b/internal/e2e/pod.go @@ -24,7 +24,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/agent-substrate/substrate/internal/portforward" + "go.ate.dev/substrate/internal/portforward" ) // WaitForPodReady blocks until the pod passes its readiness probe, and fails the diff --git a/internal/e2e/preflight.go b/internal/e2e/preflight.go index a7127b42f9..a3559eb2a7 100644 --- a/internal/e2e/preflight.go +++ b/internal/e2e/preflight.go @@ -21,7 +21,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // PreflightChecks checks that the test environment is ready for the test suite. diff --git a/internal/e2e/preview.go b/internal/e2e/preview.go index de8068af1a..fb997c64c2 100644 --- a/internal/e2e/preview.go +++ b/internal/e2e/preview.go @@ -22,7 +22,7 @@ import ( "google.golang.org/grpc" - "github.com/agent-substrate/substrate/internal/preview" + "go.ate.dev/substrate/internal/preview" "github.com/spf13/pflag" ) diff --git a/internal/e2e/probe.go b/internal/e2e/probe.go index f96707db4b..fbd99e339a 100644 --- a/internal/e2e/probe.go +++ b/internal/e2e/probe.go @@ -19,7 +19,7 @@ import ( "slices" "testing" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // ProbeName is the name of the probe fixture's WorkerPool and ActorTemplate, diff --git a/internal/e2e/probe_test.go b/internal/e2e/probe_test.go index 012cec6deb..66468bcf2d 100644 --- a/internal/e2e/probe_test.go +++ b/internal/e2e/probe_test.go @@ -20,7 +20,7 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // TestProbeTemplate_TrustBundle checks that the probe template projects diff --git a/internal/e2e/resume.go b/internal/e2e/resume.go index 2dd0f61526..c753e4b830 100644 --- a/internal/e2e/resume.go +++ b/internal/e2e/resume.go @@ -19,7 +19,7 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) diff --git a/internal/e2e/router_client.go b/internal/e2e/router_client.go index 9b8c68f10a..b46ebd16e1 100644 --- a/internal/e2e/router_client.go +++ b/internal/e2e/router_client.go @@ -28,10 +28,10 @@ import ( "sync" "time" - "github.com/agent-substrate/substrate/internal/ateclient" - "github.com/agent-substrate/substrate/internal/atenet" - "github.com/agent-substrate/substrate/internal/portforward" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/ateclient" + "go.ate.dev/substrate/internal/atenet" + "go.ate.dev/substrate/internal/portforward" + "go.ate.dev/substrate/internal/resources" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" ) @@ -133,7 +133,7 @@ func (c *RouterClient) request(ctx context.Context, method string, actorRef reso // travels in the CONNECT authority (e.g. "my-actor.team-a...:9090"), the same // way a real client reaches a port other than an actor's primary one. On a // non-2xx response the returned error carries the status and body, mirroring -// atunnel.Client.DialContext's handling of the same failure mode on the +// egress.Client.DialContext's handling of the same failure mode on the // egress side. The caller owns the returned connection and must Close it; // the underlying port-forward is torn down by RouterClient.Close. func (c *RouterClient) Connect(ctx context.Context, actorRef resources.ActorRef, port int) (net.Conn, error) { @@ -206,7 +206,7 @@ func (c *RouterClient) ensureConnectPortForward(ctx context.Context) error { } // bufferedConn recovers bytes http.ReadResponse buffered past the header -// boundary, mirroring internal/atunnel/client.go's identical need on the +// boundary, mirroring internal/atunnel/egress/client.go's identical need on the // egress CONNECT client. type bufferedConn struct { net.Conn diff --git a/internal/e2e/router_client_test.go b/internal/e2e/router_client_test.go index d8b60007fc..f01aa04fc4 100644 --- a/internal/e2e/router_client_test.go +++ b/internal/e2e/router_client_test.go @@ -21,8 +21,8 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/internal/atenet" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/atenet" + "go.ate.dev/substrate/internal/resources" ) func TestRouterClientPostJSON(t *testing.T) { diff --git a/internal/e2e/sandbox.go b/internal/e2e/sandbox.go index 702b88629b..e8de901130 100644 --- a/internal/e2e/sandbox.go +++ b/internal/e2e/sandbox.go @@ -178,7 +178,7 @@ func RenderFixtureManifest(t *testing.T, relPath, bucket, name string) string { func fixtureSubstitutions(bucket, name string) (inline, blocks map[string]string) { inline = map[string]string{ "${BUCKET_NAME}": bucket, - "${WORKER_IMAGE}": "ko://github.com/agent-substrate/substrate/cmd/ateworker-gvisor", + "${WORKER_IMAGE}": "ko://go.ate.dev/substrate/cmd/ateworker-gvisor", // The manifest-side half of FixtureName: it suffixes the fixture's // namespace, and with it the snapshot prefix underneath. "${FIXTURE_SUFFIX}": "-" + name, @@ -193,7 +193,7 @@ func fixtureSubstitutions(bucket, name string) (inline, blocks map[string]string return inline, blocks } - inline["${WORKER_IMAGE}"] = "ko://github.com/agent-substrate/substrate/cmd/ateworker-microvm" + inline["${WORKER_IMAGE}"] = "ko://go.ate.dev/substrate/cmd/ateworker-microvm" inline["${FIXTURE_SUFFIX}"] = "-" + SandboxClassMicroVM + "-" + name // The micro-VM ActorTemplates name the cluster-wide SandboxConfig // hack/install-microvm-deps.sh installs (configName: microvm), so a diff --git a/internal/e2e/sandbox_test.go b/internal/e2e/sandbox_test.go index d04f1058cc..6e545dd52a 100644 --- a/internal/e2e/sandbox_test.go +++ b/internal/e2e/sandbox_test.go @@ -19,8 +19,8 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/pkg/proto/ateapipb" "sigs.k8s.io/yaml" ) diff --git a/internal/e2e/serverpod_test.go b/internal/e2e/serverpod_test.go index df6ad7db9b..d61fcb9a34 100644 --- a/internal/e2e/serverpod_test.go +++ b/internal/e2e/serverpod_test.go @@ -80,14 +80,14 @@ func renderServerPodDocs(t *testing.T, spec ServerPod) (*corev1.Pod, *corev1.Ser func TestRenderServerPod_GRPCProbe(t *testing.T) { pod, service := renderServerPodDocs(t, ServerPod{ Name: "grpcecho", - ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + ImportPath: "go.ate.dev/substrate/internal/e2e/fixtures/testserver", Args: []string{"grpc"}, Port: 50051, GRPCProbe: true, }) container := pod.Spec.Containers[0] - if got, want := container.Image, "ko://github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver"; got != want { + if got, want := container.Image, "ko://go.ate.dev/substrate/internal/e2e/fixtures/testserver"; got != want { t.Errorf("container image = %q, want %q", got, want) } // Args carry the subcommand ahead of the --listen the template appends. The @@ -137,7 +137,7 @@ func TestRenderServerPod_GRPCProbe(t *testing.T) { func TestRenderServerPod_HTTPProbe(t *testing.T) { pod, _ := renderServerPodDocs(t, ServerPod{ Name: "httporigin", - ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + ImportPath: "go.ate.dev/substrate/internal/e2e/fixtures/testserver", Args: []string{"http"}, Port: 8080, }) @@ -170,7 +170,7 @@ func TestRenderServerPod(t *testing.T) { name: "maps a privileged published port to an unprivileged listener", spec: ServerPod{ Name: "egresshttp", - ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + ImportPath: "go.ate.dev/substrate/internal/e2e/fixtures/testserver", Args: []string{"http"}, Port: 80, TargetPort: 8080, @@ -181,7 +181,7 @@ func TestRenderServerPod(t *testing.T) { name: "defaults the listener to Port when TargetPort is unset", spec: ServerPod{ Name: "httporigin", - ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + ImportPath: "go.ate.dev/substrate/internal/e2e/fixtures/testserver", Args: []string{"http"}, Port: 8080, }, @@ -227,7 +227,7 @@ func TestRenderServerPod(t *testing.T) { func TestRenderServerPod_Volumes(t *testing.T) { pod, _ := renderServerPodDocs(t, ServerPod{ Name: "egressprobe", - ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + ImportPath: "go.ate.dev/substrate/internal/e2e/fixtures/testserver", Args: []string{"egressprobe"}, Port: 8080, VolumeMounts: []corev1.VolumeMount{ @@ -319,7 +319,7 @@ func TestRenderServerPod_TLSPort(t *testing.T) { func TestServiceDNSTLS(t *testing.T) { base := ServerPod{ Name: "wss-origin", - ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + ImportPath: "go.ate.dev/substrate/internal/e2e/fixtures/testserver", Args: []string{"websocket", "--echo"}, HealthPath: "/readyz", Volumes: []corev1.Volume{{Name: "extra", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}}, diff --git a/internal/e2e/statusz.go b/internal/e2e/statusz.go index cae64166fd..c86b57efef 100644 --- a/internal/e2e/statusz.go +++ b/internal/e2e/statusz.go @@ -21,8 +21,8 @@ import ( "net/http" "time" - "github.com/agent-substrate/substrate/internal/ateclient" - "github.com/agent-substrate/substrate/internal/portforward" + "go.ate.dev/substrate/internal/ateclient" + "go.ate.dev/substrate/internal/portforward" "k8s.io/client-go/kubernetes" ) diff --git a/internal/e2e/suites/authz/authz_test.go b/internal/e2e/suites/authz/authz_test.go new file mode 100644 index 0000000000..f6c9709833 --- /dev/null +++ b/internal/e2e/suites/authz/authz_test.go @@ -0,0 +1,46 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package authz runs against an install that enforces ateapi's runtime +// authorization with the ate-client ServiceAccount, the identity the e2e +// clients use, among its bootstrap owners, as the default install does. +// +// The unit tests in cmd/ateapi/internal/authz cover which role may make +// which call. This suite covers what only an installed system shows: a user +// granted access through an AccessPolicy using Substrate end to end, and +// the security boundaries that depend on the install, real credentials, +// stored state, and policy changes reaching every ateapi replica. +package authz + +import ( + "context" + "testing" + + "go.ate.dev/substrate/internal/e2e" +) + +// TestAuthz deploys one fixture, the expensive part, and runs every check +// against it. The subtests run in order, not in parallel: each sets the +// fixture atespace's AccessPolicy for its own principals. +func TestAuthz(t *testing.T) { + env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO") + if err != nil { + t.Fatalf("CheckEnv failed: %v", err) + } + ctx := context.Background() + f := deployFixture(t, ctx, env["BUCKET_NAME"]) + + t.Run("BasicFlow", func(t *testing.T) { testBasicFlow(t, ctx, f) }) + t.Run("SecurityRegression", func(t *testing.T) { testSecurityRegression(t, ctx, f) }) +} diff --git a/internal/e2e/suites/authz/flow_test.go b/internal/e2e/suites/authz/flow_test.go new file mode 100644 index 0000000000..891879abb1 --- /dev/null +++ b/internal/e2e/suites/authz/flow_test.go @@ -0,0 +1,78 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package authz + +import ( + "context" + "testing" + + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/pkg/proto/ateapipb" +) + +// testBasicFlow runs an actor's whole lifecycle as users granted access +// through an AccessPolicy, not as the bootstrap owner every other suite runs +// as. Alice, an editor, creates, resumes, suspends and deletes an actor; Bob, +// a viewer, can watch it but not change it. Each step needs the one before. +func testBasicFlow(t *testing.T, ctx context.Context, f *fixture) { + alice := newPrincipal(t, ctx, f.clients, f.atespace, "flow-alice") + bob := newPrincipal(t, ctx, f.clients, f.atespace, "flow-bob") + setAtespacePolicy(t, ctx, f.clients, f.atespace, binding("editor", alice), binding("viewer", bob)) + cleanupAtespacePolicy(t, f.clients, f.atespace) + + // Wait for the grants to reach every ateapi replica. + expectAll(t, ctx, []call{ + {"GetActor", alice, getActor(f.actor), allowed}, + {"GetActor", bob, getActor(f.actor), allowed}, + }) + if t.Failed() { + return + } + + actor := &ateapipb.ObjectRef{Atespace: f.atespace, Name: "flow-actor"} + if _, err := alice.api.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: actor.GetAtespace(), Name: actor.GetName()}, + ActorTemplate: f.template, + }}); err != nil { + t.Fatalf("CreateActor as alice: %v", err) + } + t.Cleanup(func() { + // Best effort, in case a step below failed partway. + _, _ = f.clients.SubstrateAPI.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: actor}) + _, _ = f.clients.SubstrateAPI.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{Actor: actor}) + }) + + if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, &e2e.Clients{SubstrateAPI: alice.api}, &ateapipb.ResumeActorRequest{Actor: actor}); err != nil { + t.Fatalf("ResumeActor as alice: %v", err) + } + + // Bob sees the running actor but can't suspend it. + expectAll(t, ctx, []call{ + {"GetActor", bob, getActor(actor), allowed}, + {"ListActors", bob, listActors(f.atespace), allowed}, + {"SuspendActor", bob, suspendActor(actor), denied}, + }) + if t.Failed() { + return + } + + if _, err := alice.api.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actor}); err != nil { + t.Fatalf("SuspendActor as alice: %v", err) + } + waitForActorState(t, ctx, f.clients, actor, ateapipb.ActorState_ACTOR_STATE_SUSPENDED) + if _, err := alice.api.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: actor}); err != nil { + t.Fatalf("DeleteActor as alice: %v", err) + } +} diff --git a/internal/e2e/suites/authz/helpers_test.go b/internal/e2e/suites/authz/helpers_test.go new file mode 100644 index 0000000000..cad437f5a0 --- /dev/null +++ b/internal/e2e/suites/authz/helpers_test.go @@ -0,0 +1,362 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package authz + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + authv1 "k8s.io/api/authentication/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "go.ate.dev/substrate/internal/ateclient" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" +) + +const ( + actorName = "authz-actor" + // propagationTimeout bounds how long a policy change may take to reach + // every ateapi replica's checks. + propagationTimeout = 30 * time.Second +) + +// fixture is what every test shares: an atespace holding the probe template +// and one suspended actor from it, all created by the bootstrap owner. +type fixture struct { + clients *e2e.Clients + bootstrap *principal + atespace string + template *ateapipb.ObjectRef + actor *ateapipb.ObjectRef +} + +// outcome is the set of status codes a call may return to pass. +type outcome struct { + name string + ok func(codes.Code) bool +} + +func exactly(c codes.Code) outcome { + return outcome{name: c.String(), ok: func(got codes.Code) bool { return got == c }} +} + +var ( + allowed = exactly(codes.OK) + denied = exactly(codes.PermissionDenied) + // passedCheck is any outcome but a permission or authentication failure. + // The permission check runs before the handler validates or looks + // anything up, so a request the handler then rejects (an invalid + // request, a missing object) shows the caller got past the check + // without changing anything. + // + // The check also rejects a malformed identifier with InvalidArgument, so + // every request that expects passedCheck is paired with a caller that + // expects denied on the same request: were its identifiers malformed, + // that caller would get InvalidArgument and fail. + passedCheck = outcome{name: "anything but PermissionDenied or Unauthenticated", ok: func(c codes.Code) bool { + return c != codes.PermissionDenied && c != codes.Unauthenticated + }} + // rejected is a caller refused for who it is, whether by the permission + // check or by a handler that authenticates the caller itself. + rejected = outcome{name: "PermissionDenied or Unauthenticated", ok: func(c codes.Code) bool { + return c == codes.PermissionDenied || c == codes.Unauthenticated + }} +) + +// rpc is one request, made with whichever client a call supplies. +type rpc func(ctx context.Context, api *ateclient.Client) error + +// call is an rpc made as who, and the outcome it must have. +type call struct { + name string + who *principal + do rpc + want outcome +} + +// expectAll polls calls until each has its outcome, since a policy change is +// not guaranteed to reach every ateapi replica at once. The calls share one +// deadline, so a step costs at most propagationTimeout however many it has. +func expectAll(t *testing.T, ctx context.Context, calls []call) { + t.Helper() + deadline := time.Now().Add(propagationTimeout) + pending := calls + for { + lastErr := map[int]error{} + var still []call + for _, c := range pending { + err := c.do(ctx, c.who.api) + if !c.want.ok(status.Code(err)) { + lastErr[len(still)] = err + still = append(still, c) + } + } + if len(still) == 0 { + return + } + if time.Now().After(deadline) { + for i, c := range still { + t.Errorf("%s %s: got %v, want %s", c.who.name, c.name, lastErr[i], c.want.name) + } + return + } + pending = still + time.Sleep(time.Second) + } +} + +// principal is an identity the tests make calls as. +type principal struct { + name string + member string // AccessPolicy member: "user:" + the token subject + api *ateclient.Client +} + +// newPrincipal creates a ServiceAccount in namespace and returns a principal +// that authenticates with its token. +func newPrincipal(t *testing.T, ctx context.Context, clients *e2e.Clients, namespace, name string) *principal { + t.Helper() + sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}} + if _, err := clients.K8s.CoreV1().ServiceAccounts(namespace).Create(ctx, sa, metav1.CreateOptions{}); err != nil { + t.Fatalf("creating ServiceAccount %s/%s: %v", namespace, name, err) + } + return principalFor(t, ctx, clients, namespace, name) +} + +// principalFor returns a principal that authenticates with a token minted +// for the existing ServiceAccount namespace/name. +func principalFor(t *testing.T, ctx context.Context, clients *e2e.Clients, namespace, name string) *principal { + t.Helper() + expiration := int64(3600) + token, err := clients.K8s.CoreV1().ServiceAccounts(namespace).CreateToken(ctx, name, &authv1.TokenRequest{ + Spec: authv1.TokenRequestSpec{ + Audiences: []string{installdefaults.APIServiceName + "." + installdefaults.SystemNamespace + ".svc"}, + ExpirationSeconds: &expiration, + }, + }, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("minting a token for %s/%s: %v", namespace, name, err) + } + return &principal{ + name: name, + member: fmt.Sprintf("user:system:serviceaccount:%s:%s", namespace, name), + api: clientWithToken(t, ctx, token.Status.Token), + } +} + +// clientWithToken returns an ateapi client that sends token as its bearer +// token. +func clientWithToken(t *testing.T, ctx context.Context, token string) *ateclient.Client { + t.Helper() + tokenFile := filepath.Join(t.TempDir(), "token") + if err := os.WriteFile(tokenFile, []byte(token), 0o600); err != nil { + t.Fatalf("writing token: %v", err) + } + connectCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + api, err := ateclient.NewClient(connectCtx, e2e.KubeConfig, e2e.KubeContext, "", tokenFile, false) + if err != nil { + t.Fatalf("ateclient.NewClient: %v", err) + } + t.Cleanup(api.Close) + return api +} + +// serviceAccountExists reports whether namespace/name exists, failing the +// test on any other error. +func serviceAccountExists(t *testing.T, ctx context.Context, clients *e2e.Clients, namespace, name string) bool { + t.Helper() + _, err := clients.K8s.CoreV1().ServiceAccounts(namespace).Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return false + } + if err != nil { + t.Fatalf("getting ServiceAccount %s/%s: %v", namespace, name, err) + } + return true +} + +func binding(role string, ps ...*principal) *ateapipb.Binding { + b := &ateapipb.Binding{Role: role} + for _, p := range ps { + b.Members = append(b.Members, p.member) + } + return b +} + +// setAtespacePolicy replaces atespace's AccessPolicy bindings, creating the +// policy if it doesn't exist yet. The caller deletes it, with +// cleanupAtespacePolicy at the scope the policy must last for. +func setAtespacePolicy(t *testing.T, ctx context.Context, clients *e2e.Clients, atespace string, bindings ...*ateapipb.Binding) { + t.Helper() + ref := &ateapipb.ObjectRef{Name: atespace} + policy := &ateapipb.AccessPolicy{Metadata: &ateapipb.ResourceMetadata{Name: resources.SingletonName}, Bindings: bindings} + current, err := clients.SubstrateAPI.GetAtespaceAccessPolicy(ctx, &ateapipb.GetAtespaceAccessPolicyRequest{Atespace: ref, Name: resources.SingletonName}) + switch { + case status.Code(err) == codes.NotFound: + _, err = clients.SubstrateAPI.CreateAtespaceAccessPolicy(ctx, &ateapipb.CreateAtespaceAccessPolicyRequest{Atespace: ref, AccessPolicy: policy}) + case err == nil: + policy.Metadata = current.GetMetadata() + _, err = clients.SubstrateAPI.UpdateAtespaceAccessPolicy(ctx, &ateapipb.UpdateAtespaceAccessPolicyRequest{Atespace: ref, AccessPolicy: policy}) + } + if err != nil { + t.Fatalf("setting the AccessPolicy of atespace %q: %v", atespace, err) + } +} + +// cleanupAtespacePolicy deletes atespace's AccessPolicy when t ends. +func cleanupAtespacePolicy(t *testing.T, clients *e2e.Clients, atespace string) { + t.Cleanup(func() { + if err := deleteAtespacePolicy(context.Background(), clients, atespace); err != nil { + t.Logf("cleanup: deleting the AccessPolicy of atespace %q: %v", atespace, err) + } + }) +} + +// deleteAtespacePolicy deletes atespace's AccessPolicy; one already gone is +// not an error. +func deleteAtespacePolicy(ctx context.Context, clients *e2e.Clients, atespace string) error { + _, err := clients.SubstrateAPI.DeleteAtespaceAccessPolicy(ctx, &ateapipb.DeleteAtespaceAccessPolicyRequest{ + Atespace: &ateapipb.ObjectRef{Name: atespace}, + Name: resources.SingletonName, + }) + if status.Code(err) == codes.NotFound { + return nil + } + return err +} + +// createAtespace creates an empty atespace as the bootstrap owner and deletes +// it when t ends. Deleting it also deletes its AccessPolicy. +func createAtespace(t *testing.T, ctx context.Context, clients *e2e.Clients, name string) { + t.Helper() + if _, err := clients.SubstrateAPI.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{ + Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: name}}, + }); err != nil { + t.Fatalf("CreateAtespace %q: %v", name, err) + } + t.Cleanup(func() { + if err := deleteAtespace(context.Background(), clients, name); err != nil { + t.Logf("cleanup: DeleteAtespace %q: %v", name, err) + } + }) +} + +// deleteAtespace deletes an atespace; one already gone is not an error. +func deleteAtespace(ctx context.Context, clients *e2e.Clients, name string) error { + _, err := clients.SubstrateAPI.DeleteAtespace(ctx, &ateapipb.DeleteAtespaceRequest{Atespace: &ateapipb.ObjectRef{Name: name}}) + if status.Code(err) == codes.NotFound { + return nil + } + return err +} + +// deployFixture deploys the probe template into a new atespace and creates +// one actor from it, all as the bootstrap owner. The actor is never resumed: +// the tests only need an actor that exists. +func deployFixture(t *testing.T, ctx context.Context, bucket string) *fixture { + t.Helper() + clients := e2e.GetClients() + atespace, _ := e2e.DeployProbe(t, bucket, "authz") + actor := &ateapipb.ObjectRef{Atespace: atespace, Name: actorName} + template := &ateapipb.ObjectRef{Atespace: atespace, Name: e2e.ProbeName} + if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: atespace, Name: actorName}, + ActorTemplate: template, + }}); err != nil { + t.Fatalf("CreateActor %s/%s: %v", atespace, actorName, err) + } + t.Cleanup(func() { + if _, err := clients.SubstrateAPI.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{Actor: actor}); err != nil { + t.Logf("cleanup: DeleteActor %s/%s: %v", atespace, actorName, err) + } + }) + return &fixture{ + clients: clients, + bootstrap: &principal{name: "bootstrap-owner", api: clients.SubstrateAPI}, + atespace: atespace, + template: template, + actor: actor, + } +} + +// waitForActorState polls actor, as the bootstrap owner, until it is in want. +func waitForActorState(t *testing.T, ctx context.Context, clients *e2e.Clients, actor *ateapipb.ObjectRef, want ateapipb.ActorState) { + t.Helper() + deadline := time.Now().Add(60 * time.Second) + for time.Now().Before(deadline) { + resp, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{Actor: actor}) + if err == nil && resp.GetStatus().GetState() == want { + return + } + time.Sleep(time.Second) + } + t.Fatalf("timed out waiting for actor %s/%s to reach %v", actor.GetAtespace(), actor.GetName(), want) +} + +// The requests the tests make, shared so that each test pairs callers on +// the same request. + +func getAtespace(name string) rpc { + return func(ctx context.Context, api *ateclient.Client) error { + _, err := api.GetAtespace(ctx, &ateapipb.GetAtespaceRequest{Atespace: &ateapipb.ObjectRef{Name: name}}) + return err + } +} + +func getActor(ref *ateapipb.ObjectRef) rpc { + return func(ctx context.Context, api *ateclient.Client) error { + _, err := api.GetActor(ctx, &ateapipb.GetActorRequest{Actor: ref}) + return err + } +} + +func listActors(atespace string) rpc { + return func(ctx context.Context, api *ateclient.Client) error { + _, err := api.ListActors(ctx, &ateapipb.ListActorsRequest{Atespace: atespace}) + return err + } +} + +func suspendActor(ref *ateapipb.ObjectRef) rpc { + return func(ctx context.Context, api *ateclient.Client) error { + _, err := api.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: ref}) + return err + } +} + +// createActorUnnamed asks for an actor with no name: the check reads only +// its atespace and template, and validation then rejects it, so nothing is +// created. +func createActorUnnamed(atespace string, template *ateapipb.ObjectRef) rpc { + return func(ctx context.Context, api *ateclient.Client) error { + _, err := api.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: atespace}, + ActorTemplate: template, + }}) + return err + } +} diff --git a/internal/e2e/suites/authz/security_regression_test.go b/internal/e2e/suites/authz/security_regression_test.go new file mode 100644 index 0000000000..7d80979e4a --- /dev/null +++ b/internal/e2e/suites/authz/security_regression_test.go @@ -0,0 +1,318 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package authz + +import ( + "context" + "testing" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "go.ate.dev/substrate/internal/ateclient" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" +) + +// testSecurityRegression checks one case per threat, each in a subtest +// named for it. Every negative check has a positive control showing the +// request could have succeeded, because a denial that stops exercising its +// boundary keeps passing. +func testSecurityRegression(t *testing.T, ctx context.Context, f *fixture) { + t.Run("authorization is enforced by default", func(t *testing.T) { testEnforcedByDefault(t, ctx, f) }) + t.Run("invalid credentials are rejected", func(t *testing.T) { testInvalidCredentials(t, ctx, f) }) + t.Run("no access across atespaces", func(t *testing.T) { testTenantCrossing(t, ctx, f) }) + t.Run("editor cannot grant itself owner", func(t *testing.T) { testPrivilegeEscalation(t, ctx, f) }) + t.Run("revocation takes effect", func(t *testing.T) { testRevocation(t, ctx, f) }) + t.Run("recreated atespace drops old grants", func(t *testing.T) { testAtespaceRecreated(t, ctx, f) }) + t.Run("recreated AccessPolicy rejects stale updates", func(t *testing.T) { testAccessPolicyRecreated(t, ctx, f) }) + t.Run("WorkerService accepts only atelet", func(t *testing.T) { testWorkerServiceRequiresAtelet(t, ctx, f) }) +} + +// testEnforcedByDefault checks that a default install enforces +// authorization: a ServiceAccount no policy names gets nothing. An install +// that left it off would pass every unit test. +func testEnforcedByDefault(t *testing.T, ctx context.Context, f *fixture) { + nobody := newPrincipal(t, ctx, f.clients, f.atespace, "boundary-nobody") + expectAll(t, ctx, []call{ + {"GetActor", f.bootstrap, getActor(f.actor), allowed}, + {"GetActor", nobody, getActor(f.actor), denied}, + {"CreateActor", nobody, createActorUnnamed(f.atespace, f.template), denied}, + }) +} + +// testInvalidCredentials checks that a bearer token ateapi can't verify is +// refused rather than treated as some caller. +func testInvalidCredentials(t *testing.T, ctx context.Context, f *fixture) { + forged := &principal{name: "invalid-token", api: clientWithToken(t, ctx, "not-a-token")} + expectAll(t, ctx, []call{ + {"GetActor", f.bootstrap, getActor(f.actor), allowed}, + {"GetActor", forged, getActor(f.actor), rejected}, + }) +} + +// testTenantCrossing checks that an editor of one atespace can't reach into +// another. +func testTenantCrossing(t *testing.T, ctx context.Context, f *fixture) { + other := f.atespace + "-other" + createAtespace(t, ctx, f.clients, other) + grace := newPrincipal(t, ctx, f.clients, f.atespace, "boundary-grace") + setAtespacePolicy(t, ctx, f.clients, other, binding("editor", grace)) + + expectAll(t, ctx, []call{ + // Control: grace's grant on her own atespace works. + {"ListActors in her atespace", grace, listActors(other), allowed}, + {"GetActor in another atespace", grace, getActor(f.actor), denied}, + {"CreateActor in another atespace", grace, createActorUnnamed(f.atespace, f.template), denied}, + }) +} + +// testPrivilegeEscalation checks that an editor can't rewrite the +// AccessPolicy to make itself owner, while an owner can update it. +func testPrivilegeEscalation(t *testing.T, ctx context.Context, f *fixture) { + owner := newPrincipal(t, ctx, f.clients, f.atespace, "escalate-owner") + editor := newPrincipal(t, ctx, f.clients, f.atespace, "escalate-editor") + current := []*ateapipb.Binding{binding("owner", owner), binding("editor", editor)} + setAtespacePolicy(t, ctx, f.clients, f.atespace, current...) + cleanupAtespacePolicy(t, f.clients, f.atespace) + + // updatePolicy rewrites the policy's bindings, guarded by the uid and + // version the bootstrap owner reads just before, so the update is + // otherwise valid. + ref := &ateapipb.ObjectRef{Name: f.atespace} + updatePolicy := func(bindings ...*ateapipb.Binding) rpc { + return func(ctx context.Context, api *ateclient.Client) error { + cur, err := f.clients.SubstrateAPI.GetAtespaceAccessPolicy(ctx, &ateapipb.GetAtespaceAccessPolicyRequest{Atespace: ref, Name: resources.SingletonName}) + if err != nil { + return err + } + _, err = api.UpdateAtespaceAccessPolicy(ctx, &ateapipb.UpdateAtespaceAccessPolicyRequest{ + Atespace: ref, + AccessPolicy: &ateapipb.AccessPolicy{Metadata: cur.GetMetadata(), Bindings: bindings}, + }) + return err + } + } + + // Wait for both grants to take effect, so the editor's denial below is + // for its role and not for having none yet. + expectAll(t, ctx, []call{ + {"GetActor", owner, getActor(f.actor), allowed}, + {"GetActor", editor, getActor(f.actor), allowed}, + }) + if t.Failed() { + return + } + expectAll(t, ctx, []call{ + {"UpdateAtespaceAccessPolicy to make itself owner", editor, updatePolicy(binding("owner", owner, editor)), denied}, + // Control: the same request with the bindings unchanged, as owner. + {"UpdateAtespaceAccessPolicy unchanged", owner, updatePolicy(current...), allowed}, + }) +} + +// testRevocation checks that removing a binding, downgrading a role and +// deleting the AccessPolicy each take effect on every ateapi replica, with +// no cached grant outliving the change. Steps run in order. +func testRevocation(t *testing.T, ctx context.Context, f *fixture) { + carol := newPrincipal(t, ctx, f.clients, f.atespace, "revoke-carol") + dan := newPrincipal(t, ctx, f.clients, f.atespace, "revoke-dan") + erin := newPrincipal(t, ctx, f.clients, f.atespace, "revoke-erin") + cleanupAtespacePolicy(t, f.clients, f.atespace) + createActor := createActorUnnamed(f.atespace, f.template) + + for _, step := range []struct { + name string + change func(t *testing.T) + expect []call + }{ + { + name: "granted", + change: func(t *testing.T) { + setAtespacePolicy(t, ctx, f.clients, f.atespace, binding("editor", carol, dan), binding("viewer", erin)) + }, + expect: []call{ + {"CreateActor", carol, createActor, passedCheck}, + {"CreateActor", dan, createActor, passedCheck}, + {"CreateActor", erin, createActor, denied}, + }, + }, + { + name: "binding removed", + change: func(t *testing.T) { + setAtespacePolicy(t, ctx, f.clients, f.atespace, binding("editor", dan), binding("viewer", erin)) + }, + expect: []call{ + {"CreateActor", carol, createActor, denied}, + {"GetActor", carol, getActor(f.actor), denied}, + {"CreateActor", dan, createActor, passedCheck}, + }, + }, + { + name: "editor downgraded to viewer", + change: func(t *testing.T) { + setAtespacePolicy(t, ctx, f.clients, f.atespace, binding("viewer", dan, erin)) + }, + expect: []call{ + {"CreateActor", dan, createActor, denied}, + {"GetActor", dan, getActor(f.actor), allowed}, + }, + }, + { + name: "AccessPolicy deleted", + change: func(t *testing.T) { + if err := deleteAtespacePolicy(ctx, f.clients, f.atespace); err != nil { + t.Fatalf("DeleteAtespaceAccessPolicy: %v", err) + } + }, + expect: []call{ + {"GetActor", dan, getActor(f.actor), denied}, + {"GetActor", erin, getActor(f.actor), denied}, + {"GetActor", f.bootstrap, getActor(f.actor), allowed}, + }, + }, + } { + if !t.Run(step.name, func(t *testing.T) { + step.change(t) + expectAll(t, ctx, step.expect) + }) { + // Later steps assume this one's change took effect. + t.FailNow() + } + } +} + +// testAtespaceRecreated checks that grants on an atespace go with it: +// deleting an atespace and creating another with the same name doesn't +// hand the new one's access to the old one's members. +func testAtespaceRecreated(t *testing.T, ctx context.Context, f *fixture) { + name := f.atespace + "-aba" + frank := newPrincipal(t, ctx, f.clients, f.atespace, "aba-frank") + createAtespace(t, ctx, f.clients, name) + setAtespacePolicy(t, ctx, f.clients, name, binding("owner", frank)) + expectAll(t, ctx, []call{{"GetAtespace", frank, getAtespace(name), allowed}}) + if t.Failed() { + return + } + + if err := deleteAtespace(ctx, f.clients, name); err != nil { + t.Fatalf("DeleteAtespace %q: %v", name, err) + } + createAtespace(t, ctx, f.clients, name) + expectAll(t, ctx, []call{ + {"GetAtespace", frank, getAtespace(name), denied}, + {"GetAtespace", f.bootstrap, getAtespace(name), allowed}, + }) +} + +// testAccessPolicyRecreated checks that a deleted and recreated AccessPolicy +// is a new object: it carries none of the old one's bindings, and an update +// guarded by the old one's uid is rejected rather than applied to it. +func testAccessPolicyRecreated(t *testing.T, ctx context.Context, f *fixture) { + gina := newPrincipal(t, ctx, f.clients, f.atespace, "aba-gina") + setAtespacePolicy(t, ctx, f.clients, f.atespace, binding("viewer", gina)) + cleanupAtespacePolicy(t, f.clients, f.atespace) + expectAll(t, ctx, []call{{"GetActor", gina, getActor(f.actor), allowed}}) + if t.Failed() { + return + } + + ref := &ateapipb.ObjectRef{Name: f.atespace} + old, err := f.clients.SubstrateAPI.GetAtespaceAccessPolicy(ctx, &ateapipb.GetAtespaceAccessPolicyRequest{Atespace: ref, Name: resources.SingletonName}) + if err != nil { + t.Fatalf("GetAtespaceAccessPolicy: %v", err) + } + if err := deleteAtespacePolicy(ctx, f.clients, f.atespace); err != nil { + t.Fatalf("DeleteAtespaceAccessPolicy: %v", err) + } + if _, err := f.clients.SubstrateAPI.CreateAtespaceAccessPolicy(ctx, &ateapipb.CreateAtespaceAccessPolicyRequest{ + Atespace: ref, + AccessPolicy: &ateapipb.AccessPolicy{Metadata: &ateapipb.ResourceMetadata{Name: resources.SingletonName}}, + }); err != nil { + t.Fatalf("CreateAtespaceAccessPolicy: %v", err) + } + + // The new policy has none of the old one's bindings. + expectAll(t, ctx, []call{{"GetActor", gina, getActor(f.actor), denied}}) + + // An update guarded by the old policy's uid and version must not apply + // to the new one. + _, err = f.clients.SubstrateAPI.UpdateAtespaceAccessPolicy(ctx, &ateapipb.UpdateAtespaceAccessPolicyRequest{ + Atespace: ref, + AccessPolicy: &ateapipb.AccessPolicy{Metadata: old.GetMetadata(), Bindings: []*ateapipb.Binding{binding("owner", gina)}}, + }) + if status.Code(err) != codes.Aborted { + t.Errorf("UpdateAtespaceAccessPolicy with the deleted policy's uid: got %v, want %v", err, codes.Aborted) + } + cur, err := f.clients.SubstrateAPI.GetAtespaceAccessPolicy(ctx, &ateapipb.GetAtespaceAccessPolicyRequest{Atespace: ref, Name: resources.SingletonName}) + if err != nil { + t.Fatalf("GetAtespaceAccessPolicy: %v", err) + } + if len(cur.GetBindings()) != 0 { + t.Errorf("recreated AccessPolicy bindings = %v, want none", cur.GetBindings()) + } +} + +// testWorkerServiceRequiresAtelet checks that WorkerService rejects every +// caller but atelet. Its RPCs skip the permission checks; each handler +// authenticates atelet by its mTLS certificate, so that is all that guards +// them. A bearer token is rejected even when it is atelet's own +// ServiceAccount's or the bootstrap owner's. +func testWorkerServiceRequiresAtelet(t *testing.T, ctx context.Context, f *fixture) { + ns := e2e.SystemNamespace() + sa := installdefaults.AteletServiceAccount + // Every install creates atelet's ServiceAccount, so a missing one fails + // the test rather than skipping these checks. + if !serviceAccountExists(t, ctx, f.clients, ns, sa) { + t.Fatalf("ServiceAccount %s/%s not found", ns, sa) + } + ateletToken := principalFor(t, ctx, f.clients, ns, sa) + + // The worker named is never created: a handler that let the caller + // through would answer NotFound, not reject it. + worker := &ateapipb.ObjectRef{Name: "authz-missing"} + registerWorker := func(ctx context.Context, api *ateclient.Client) error { + _, err := api.WorkerService().RegisterWorker(ctx, &ateapipb.RegisterWorkerRequest{Worker: worker}) + return err + } + requestSuspend := func(ctx context.Context, api *ateclient.Client) error { + _, err := api.WorkerService().RequestActorSuspend(ctx, &ateapipb.RequestActorSuspendRequest{Worker: worker, Actor: f.actor}) + return err + } + // This handler validates the request before authenticating the caller, + // so the request must be valid. The CSR is parsed only after + // authentication, so it needn't be a real one. + mintWorkerCert := func(ctx context.Context, api *ateclient.Client) error { + _, err := api.WorkerService().MintWorkerActorCertificate(ctx, &ateapipb.MintWorkerActorCertificateRequest{ + Worker: worker, + Actor: f.actor, + ActorUid: "00000000-0000-4000-8000-000000000000", + CertificateSigningRequest: []byte("not a CSR"), + }) + return err + } + // TODO(#2570): also check ReportActorSnapshotState once it is served. Until + // then every caller gets UNIMPLEMENTED, before anything authenticates it. + + expectAll(t, ctx, []call{ + {"RegisterWorker", f.bootstrap, registerWorker, rejected}, + {"RegisterWorker", ateletToken, registerWorker, rejected}, + {"RequestActorSuspend", f.bootstrap, requestSuspend, rejected}, + {"RequestActorSuspend", ateletToken, requestSuspend, rejected}, + {"MintWorkerActorCertificate", f.bootstrap, mintWorkerCert, rejected}, + {"MintWorkerActorCertificate", ateletToken, mintWorkerCert, rejected}, + }) +} diff --git a/internal/e2e/suites/authz/testmain_test.go b/internal/e2e/suites/authz/testmain_test.go new file mode 100644 index 0000000000..61b1896273 --- /dev/null +++ b/internal/e2e/suites/authz/testmain_test.go @@ -0,0 +1,24 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package authz + +import ( + "os" + "testing" + + "go.ate.dev/substrate/internal/e2e" +) + +func TestMain(m *testing.M) { os.Exit(e2e.RunTestMain(m)) } diff --git a/internal/e2e/suites/capabilities/capabilities_test.go b/internal/e2e/suites/capabilities/capabilities_test.go index 39339350af..0709fa87ea 100644 --- a/internal/e2e/suites/capabilities/capabilities_test.go +++ b/internal/e2e/suites/capabilities/capabilities_test.go @@ -22,9 +22,9 @@ import ( "slices" "testing" - "github.com/agent-substrate/substrate/internal/e2e" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // defaultCapabilities mirrors atelet's default set (cmd/atelet/oci.go). It is diff --git a/internal/e2e/suites/capabilities/testmain_test.go b/internal/e2e/suites/capabilities/testmain_test.go index 34b1c0f0a4..19b5579cd5 100644 --- a/internal/e2e/suites/capabilities/testmain_test.go +++ b/internal/e2e/suites/capabilities/testmain_test.go @@ -18,7 +18,7 @@ import ( "os" "testing" - "github.com/agent-substrate/substrate/internal/e2e" + "go.ate.dev/substrate/internal/e2e" ) func TestMain(m *testing.M) { os.Exit(e2e.RunTestMain(m)) } diff --git a/internal/e2e/suites/combinedvolumes/combinedvolumes_test.go b/internal/e2e/suites/combinedvolumes/combinedvolumes_test.go index 236ae0f6b7..c0cc7a39aa 100644 --- a/internal/e2e/suites/combinedvolumes/combinedvolumes_test.go +++ b/internal/e2e/suites/combinedvolumes/combinedvolumes_test.go @@ -30,10 +30,6 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/e2e" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" @@ -41,6 +37,10 @@ import ( "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/tarball" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) diff --git a/internal/e2e/suites/combinedvolumes/testmain_test.go b/internal/e2e/suites/combinedvolumes/testmain_test.go index 0d283d557a..c4e46454a2 100644 --- a/internal/e2e/suites/combinedvolumes/testmain_test.go +++ b/internal/e2e/suites/combinedvolumes/testmain_test.go @@ -18,7 +18,7 @@ import ( "os" "testing" - "github.com/agent-substrate/substrate/internal/e2e" + "go.ate.dev/substrate/internal/e2e" ) func TestMain(m *testing.M) { os.Exit(e2e.RunTestMain(m)) } diff --git a/internal/e2e/suites/credentials/credentials_test.go b/internal/e2e/suites/credentials/credentials_test.go index 52892bcdd2..df8ba0fd28 100644 --- a/internal/e2e/suites/credentials/credentials_test.go +++ b/internal/e2e/suites/credentials/credentials_test.go @@ -29,9 +29,9 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/agent-substrate/substrate/internal/e2e" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // TestKubernetesCredentialInjection uses the Helm values in values.yaml, real @@ -60,7 +60,7 @@ func TestKubernetesCredentialInjection(t *testing.T) { e2e.DeployServerPod(t, ctx, e2e.ServerPod{ Name: "credential-origin", Namespace: namespace, - ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + ImportPath: "go.ate.dev/substrate/internal/e2e/fixtures/testserver", Args: []string{"http", "--authorization-file=/run/token/token"}, Port: 80, TargetPort: 8080, Volumes: []corev1.Volume{ diff --git a/internal/e2e/suites/credentials/testmain_test.go b/internal/e2e/suites/credentials/testmain_test.go index 6b8549a17a..479b9e7fde 100644 --- a/internal/e2e/suites/credentials/testmain_test.go +++ b/internal/e2e/suites/credentials/testmain_test.go @@ -18,7 +18,7 @@ import ( "os" "testing" - "github.com/agent-substrate/substrate/internal/e2e" + "go.ate.dev/substrate/internal/e2e" ) func TestMain(m *testing.M) { os.Exit(e2e.RunTestMain(m)) } diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index 48becbc6a4..69ab16fb91 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -26,15 +26,15 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/ateclient" - "github.com/agent-substrate/substrate/internal/atenet" - "github.com/agent-substrate/substrate/internal/e2e" - "github.com/agent-substrate/substrate/internal/hardware" - "github.com/agent-substrate/substrate/internal/preview" - "github.com/agent-substrate/substrate/internal/resources" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/internal/ateclient" + "go.ate.dev/substrate/internal/atenet" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/hardware" + "go.ate.dev/substrate/internal/preview" + "go.ate.dev/substrate/internal/resources" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/testing/protocmp" diff --git a/internal/e2e/suites/demo/termination_test.go b/internal/e2e/suites/demo/termination_test.go index 763179cc96..e865ded0cc 100644 --- a/internal/e2e/suites/demo/termination_test.go +++ b/internal/e2e/suites/demo/termination_test.go @@ -19,9 +19,11 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/e2e" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -142,7 +144,8 @@ func waitForWorkerRemoved(ctx context.Context, t *testing.T, clients *e2e.Client // TestGracefulWorkerTerminationSuspend exercises the case where a worker pod is // deleted (evicted), and while the container is in its SIGTERM shutdown phase, -// we initiate a suspend. Suspend should succeed. +// we initiate a suspend. Suspend either completes before worker deletion crashes +// the actor, or reports that concurrent crash without overwriting it. func TestGracefulWorkerTerminationSuspend(t *testing.T) { t.Parallel() nsObj := e2e.CreateNamespace(t) @@ -195,6 +198,10 @@ func TestGracefulWorkerTerminationSuspend(t *testing.T) { t.Fatalf("running actor has no bound worker pod: ns=%q name=%q", podNS, podName) } t.Logf("Actor %q bound to worker pod %s/%s", actorID, podNS, podName) + previousSnapshotURI := durableSnapshotURI(running.GetStatus()) + if previousSnapshotURI == "" { + t.Fatal("running actor has no completed durable snapshot") + } // Evict the worker pod. The kubelet sends SIGTERM to worker, which propagates // it into the sandbox; the container hangs for 30s. @@ -207,10 +214,20 @@ func TestGracefulWorkerTerminationSuspend(t *testing.T) { // Suspend the actor. t.Logf("Suspending actor %q during termination", actorID) - if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + _, suspendErr := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, - }); err != nil { - t.Fatalf("failed to suspend Actor: %v", err) + }) + wantState := ateapipb.ActorState_ACTOR_STATE_SUSPENDED + switch status.Code(suspendErr) { + case codes.OK: + case codes.FailedPrecondition, codes.Aborted: + // Worker deletion can crash the actor before finalize reads it, or + // after that read but before its version-guarded update. Both must + // preserve the crash and report failure instead of returning success. + wantState = ateapipb.ActorState_ACTOR_STATE_CRASHED + t.Logf("Suspend reported concurrent worker deletion: %v", suspendErr) + default: + t.Fatalf("failed to suspend Actor: %v", suspendErr) } // The worker record must eventually be removed once the pod is gone. @@ -218,8 +235,8 @@ func TestGracefulWorkerTerminationSuspend(t *testing.T) { t.Fatalf("worker %s not removed after pod deletion: %v", podName, err) } - // Verify the actor lands in ACTOR_STATE_SUSPENDED - waitForActorState(ctx, t, clients, actorID, ateapipb.ActorState_ACTOR_STATE_SUSPENDED) + // The RPC result and persisted state must agree after the worker is gone. + waitForActorState(ctx, t, clients, actorID, wantState) // Verify the pod assignment was cleared. actor, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{ @@ -231,4 +248,15 @@ func TestGracefulWorkerTerminationSuspend(t *testing.T) { if pod := actor.GetStatus().GetWorkerAssignment().GetWorkerPod(); pod != "" { t.Errorf("actor still bound to worker pod %q, expected empty", pod) } + if got := durableSnapshotURI(actor.GetStatus()); got == "" { + t.Error("actor lost its completed durable snapshot") + } else if wantState == ateapipb.ActorState_ACTOR_STATE_CRASHED && got != previousSnapshotURI { + t.Errorf("crashed actor snapshot URI = %q, want previous snapshot %q", got, previousSnapshotURI) + } + if wantState == ateapipb.ActorState_ACTOR_STATE_CRASHED && actor.GetStatus().GetCrash().GetMessage() == "" { + t.Error("crashed actor lost its crash reason") + } + if wantState == ateapipb.ActorState_ACTOR_STATE_SUSPENDED && actor.GetStatus().GetCrash() != nil { + t.Errorf("successful suspend retained a crash record: %v", actor.GetStatus().GetCrash()) + } } diff --git a/internal/e2e/suites/demo/testmain_test.go b/internal/e2e/suites/demo/testmain_test.go index d25a7d361c..7ecf69a5aa 100644 --- a/internal/e2e/suites/demo/testmain_test.go +++ b/internal/e2e/suites/demo/testmain_test.go @@ -18,7 +18,7 @@ import ( "os" "testing" - "github.com/agent-substrate/substrate/internal/e2e" + "go.ate.dev/substrate/internal/e2e" ) func TestMain(m *testing.M) { os.Exit(e2e.RunTestMain(m)) } diff --git a/internal/e2e/suites/demo/updatetemplate_test.go b/internal/e2e/suites/demo/updatetemplate_test.go index e7d8e954bf..38332b6557 100644 --- a/internal/e2e/suites/demo/updatetemplate_test.go +++ b/internal/e2e/suites/demo/updatetemplate_test.go @@ -20,9 +20,9 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/e2e" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // TestUpdateTemplateLifecycle covers repointing a suspended actor at a diff --git a/internal/e2e/suites/egressauthz/actoridentity_test.go b/internal/e2e/suites/egressauthz/actoridentity_test.go index 5543ed931d..e05ea54658 100644 --- a/internal/e2e/suites/egressauthz/actoridentity_test.go +++ b/internal/e2e/suites/egressauthz/actoridentity_test.go @@ -30,8 +30,8 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/agent-substrate/substrate/internal/e2e" - "github.com/agent-substrate/substrate/internal/localca" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/localca" ) // The gateway's front door requires a client certificate signed by the diff --git a/internal/e2e/suites/egressauthz/egressauthz_test.go b/internal/e2e/suites/egressauthz/egressauthz_test.go index e764add3ea..6940da9e28 100644 --- a/internal/e2e/suites/egressauthz/egressauthz_test.go +++ b/internal/e2e/suites/egressauthz/egressauthz_test.go @@ -48,9 +48,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "github.com/agent-substrate/substrate/internal/ateclient" - "github.com/agent-substrate/substrate/internal/e2e" - "github.com/agent-substrate/substrate/internal/portforward" + "go.ate.dev/substrate/internal/ateclient" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/portforward" ) const ( diff --git a/internal/e2e/suites/egressauthz/testmain_test.go b/internal/e2e/suites/egressauthz/testmain_test.go index 8b36a55dc7..439325016e 100644 --- a/internal/e2e/suites/egressauthz/testmain_test.go +++ b/internal/e2e/suites/egressauthz/testmain_test.go @@ -18,7 +18,7 @@ import ( "os" "testing" - "github.com/agent-substrate/substrate/internal/e2e" + "go.ate.dev/substrate/internal/e2e" ) // Setup runs before the suite. The probe is created on first use, in the diff --git a/internal/e2e/suites/egresscredinject/egresscredinject_test.go b/internal/e2e/suites/egresscredinject/egresscredinject_test.go index f7ea9fec93..5d92f13c6d 100644 --- a/internal/e2e/suites/egresscredinject/egresscredinject_test.go +++ b/internal/e2e/suites/egresscredinject/egresscredinject_test.go @@ -31,9 +31,9 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/e2e" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -48,7 +48,7 @@ var probeNamespace string // the egress gateway trusts once hack/setup-e2e-egress-tls-kind.sh has run. var echoServer = e2e.ServiceDNSTLS(e2e.ServerPod{ Name: "echo", - ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + ImportPath: "go.ate.dev/substrate/internal/e2e/fixtures/testserver", Args: []string{"http"}, }) diff --git a/internal/e2e/suites/egresscredinject/testmain_test.go b/internal/e2e/suites/egresscredinject/testmain_test.go index 03e4c0e124..0ee27c2069 100644 --- a/internal/e2e/suites/egresscredinject/testmain_test.go +++ b/internal/e2e/suites/egresscredinject/testmain_test.go @@ -18,7 +18,7 @@ import ( "os" "testing" - "github.com/agent-substrate/substrate/internal/e2e" + "go.ate.dev/substrate/internal/e2e" ) func TestMain(m *testing.M) { os.Exit(e2e.RunTestMain(m)) } diff --git a/internal/e2e/suites/egressmitm/egressmitm_test.go b/internal/e2e/suites/egressmitm/egressmitm_test.go index 672563d3cc..7f69a458ae 100644 --- a/internal/e2e/suites/egressmitm/egressmitm_test.go +++ b/internal/e2e/suites/egressmitm/egressmitm_test.go @@ -32,9 +32,9 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/e2e" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) const probeTemplate = "probe" @@ -204,7 +204,7 @@ func TestActorEgressMITMTrust(t *testing.T) { func originSpec(name string) e2e.ServerPod { return e2e.ServiceDNSTLS(e2e.ServerPod{ Name: name, - ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + ImportPath: "go.ate.dev/substrate/internal/e2e/fixtures/testserver", Args: []string{"http"}, }) } diff --git a/internal/e2e/suites/egressmitm/testmain_test.go b/internal/e2e/suites/egressmitm/testmain_test.go index 0fd7958321..996e88beb6 100644 --- a/internal/e2e/suites/egressmitm/testmain_test.go +++ b/internal/e2e/suites/egressmitm/testmain_test.go @@ -18,7 +18,7 @@ import ( "os" "testing" - "github.com/agent-substrate/substrate/internal/e2e" + "go.ate.dev/substrate/internal/e2e" ) func TestMain(m *testing.M) { os.Exit(e2e.RunTestMain(m)) } diff --git a/internal/e2e/suites/example/testmain_test.go b/internal/e2e/suites/example/testmain_test.go index f221bd77a0..73413a924c 100644 --- a/internal/e2e/suites/example/testmain_test.go +++ b/internal/e2e/suites/example/testmain_test.go @@ -18,7 +18,7 @@ import ( "os" "testing" - "github.com/agent-substrate/substrate/internal/e2e" + "go.ate.dev/substrate/internal/e2e" ) func Setup() { diff --git a/internal/e2e/suites/identity/identity_test.go b/internal/e2e/suites/identity/identity_test.go index 5349ccbf01..8355109d20 100644 --- a/internal/e2e/suites/identity/identity_test.go +++ b/internal/e2e/suites/identity/identity_test.go @@ -23,9 +23,9 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/e2e" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) const probeTemplate = "probe" diff --git a/internal/e2e/suites/identity/testmain_test.go b/internal/e2e/suites/identity/testmain_test.go index 2ccbd931b4..8a9bebc218 100644 --- a/internal/e2e/suites/identity/testmain_test.go +++ b/internal/e2e/suites/identity/testmain_test.go @@ -18,7 +18,7 @@ import ( "os" "testing" - "github.com/agent-substrate/substrate/internal/e2e" + "go.ate.dev/substrate/internal/e2e" ) func TestMain(m *testing.M) { os.Exit(e2e.RunTestMain(m)) } diff --git a/internal/e2e/suites/metrics/metrics_test.go b/internal/e2e/suites/metrics/metrics_test.go index 737e2257e1..590d19fef2 100644 --- a/internal/e2e/suites/metrics/metrics_test.go +++ b/internal/e2e/suites/metrics/metrics_test.go @@ -27,10 +27,10 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/e2e" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -200,7 +200,7 @@ func TestPlatformMetricsEmitted(t *testing.T) { if opVal == "" { crashErrs = append(crashErrs, "ate_actor_operation_name label is missing or empty") } else if ateattr.NormalizeOperationName(opVal) != opVal { - crashErrs = append(crashErrs, fmt.Sprintf("ate_actor_operation_name %q is invalid (must be one of {create, resume, suspend, pause, delete, unknown})", opVal)) + crashErrs = append(crashErrs, fmt.Sprintf("ate_actor_operation_name %q is invalid (must be one of {create, resume, suspend, delete, revert, unknown})", opVal)) } if tmplAtespaceVal == "" { diff --git a/internal/e2e/suites/metrics/testmain_test.go b/internal/e2e/suites/metrics/testmain_test.go index ba011905f1..bad5ee4aab 100644 --- a/internal/e2e/suites/metrics/testmain_test.go +++ b/internal/e2e/suites/metrics/testmain_test.go @@ -18,7 +18,7 @@ import ( "os" "testing" - "github.com/agent-substrate/substrate/internal/e2e" + "go.ate.dev/substrate/internal/e2e" ) func TestMain(m *testing.M) { os.Exit(e2e.RunTestMain(m)) } diff --git a/internal/e2e/suites/multiactor/multiactor_test.go b/internal/e2e/suites/multiactor/multiactor_test.go index 53d8fb1d42..5daf2ccda4 100644 --- a/internal/e2e/suites/multiactor/multiactor_test.go +++ b/internal/e2e/suites/multiactor/multiactor_test.go @@ -22,9 +22,9 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/e2e" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) const multiactorTemplate = "probe-multiactor" diff --git a/internal/e2e/suites/multiactor/testmain_test.go b/internal/e2e/suites/multiactor/testmain_test.go index 16095fa959..f60ff8af5b 100644 --- a/internal/e2e/suites/multiactor/testmain_test.go +++ b/internal/e2e/suites/multiactor/testmain_test.go @@ -18,7 +18,7 @@ import ( "os" "testing" - "github.com/agent-substrate/substrate/internal/e2e" + "go.ate.dev/substrate/internal/e2e" ) func TestMain(m *testing.M) { diff --git a/internal/e2e/suites/networking/accesslog_test.go b/internal/e2e/suites/networking/accesslog_test.go index 3bc5c50b1e..3cf50d67e4 100644 --- a/internal/e2e/suites/networking/accesslog_test.go +++ b/internal/e2e/suites/networking/accesslog_test.go @@ -17,7 +17,7 @@ package networking import ( "testing" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/resources" ) func TestAccessLogField(t *testing.T) { diff --git a/internal/e2e/suites/networking/arbitraryport_test.go b/internal/e2e/suites/networking/arbitraryport_test.go index 0e5325e70e..a77cc6308c 100644 --- a/internal/e2e/suites/networking/arbitraryport_test.go +++ b/internal/e2e/suites/networking/arbitraryport_test.go @@ -25,9 +25,9 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/atenet" - "github.com/agent-substrate/substrate/internal/e2e" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/atenet" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/resources" ) // counterExtraPort is the counter demo's second listener (see @@ -40,7 +40,7 @@ const counterExtraPort = 9090 // other than an actor's primary one via HTTP CONNECT, with the target port // carried in the CONNECT authority (e.g. ":9090") rather than a // separate header -- see ingress.Handler.HandleRequestHeaders and -// atunnel.TargetPortHeader. +// TargetPortHeader in internal/atunnel/ingress. // // CONNECT establishment itself (RouterClient.Connect) succeeds as soon as // atenet-router's connect_terminate listener reinjects the tunnel into diff --git a/internal/e2e/suites/networking/egresspolicy_test.go b/internal/e2e/suites/networking/egresspolicy_test.go index 86d6ec6e98..e3ee1216ce 100644 --- a/internal/e2e/suites/networking/egresspolicy_test.go +++ b/internal/e2e/suites/networking/egresspolicy_test.go @@ -23,8 +23,8 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/e2e" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/resources" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -185,7 +185,7 @@ func TestActorEgressPassthroughPolicyRefresh(t *testing.T) { dataplane := e2e.CurrentAtenetDataplane() origin, address, rootCA := prepareProtocolOrigin(t, ctx, e2e.ServerPod{ Name: "passthrough-origin", - ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + ImportPath: "go.ate.dev/substrate/internal/e2e/fixtures/testserver", Args: []string{"http"}, Port: 80, TargetPort: 8080, diff --git a/internal/e2e/suites/networking/grpcegress_test.go b/internal/e2e/suites/networking/grpcegress_test.go index 97ce9849f2..7c0aff3f50 100644 --- a/internal/e2e/suites/networking/grpcegress_test.go +++ b/internal/e2e/suites/networking/grpcegress_test.go @@ -25,15 +25,15 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/agent-substrate/substrate/internal/e2e" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/resources" ) // grpcEcho is the origin this test deploys: a cleartext-HTTP/2 gRPC server, in // its own namespace, so nothing here depends on the internet. var grpcEcho = e2e.ServerPod{ Name: "grpcecho", - ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + ImportPath: "go.ate.dev/substrate/internal/e2e/fixtures/testserver", Args: []string{"grpc"}, Port: 50051, // A gRPC server answers an HTTP GET with a protocol error, so readiness has diff --git a/internal/e2e/suites/networking/grpcingress_test.go b/internal/e2e/suites/networking/grpcingress_test.go index 2cd02e8522..ce3f1127b5 100644 --- a/internal/e2e/suites/networking/grpcingress_test.go +++ b/internal/e2e/suites/networking/grpcingress_test.go @@ -28,12 +28,12 @@ import ( "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/metadata" - "github.com/agent-substrate/substrate/internal/ateclient" - "github.com/agent-substrate/substrate/internal/atenet" - "github.com/agent-substrate/substrate/internal/e2e" - "github.com/agent-substrate/substrate/internal/portforward" - "github.com/agent-substrate/substrate/internal/proto/grpcechopb" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/ateclient" + "go.ate.dev/substrate/internal/atenet" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/portforward" + "go.ate.dev/substrate/internal/proto/grpcechopb" + "go.ate.dev/substrate/internal/resources" "k8s.io/client-go/kubernetes" ) diff --git a/internal/e2e/suites/networking/microvmclock_test.go b/internal/e2e/suites/networking/microvmclock_test.go index 43433498f0..cbbef4a26e 100644 --- a/internal/e2e/suites/networking/microvmclock_test.go +++ b/internal/e2e/suites/networking/microvmclock_test.go @@ -23,8 +23,8 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/e2e" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/pkg/proto/ateapipb" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -74,7 +74,7 @@ func TestMicroVMClockAfterDelayedRestore(t *testing.T) { t.Logf("clock evidence host_utc_after_delay=%s", time.Now().UTC().Format(time.RFC3339Nano)) target, address, rootCA := prepareProtocolOrigin(t, ctx, e2e.ServerPod{ - Name: "clock-origin", ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + Name: "clock-origin", ImportPath: "go.ate.dev/substrate/internal/e2e/fixtures/testserver", Args: []string{"http", "--body=delayed restore clock"}, Port: 8080, TLSPort: 8443, }, true) logOriginCertificateValidity(t, ctx, target, rootCA, suspendBoundary) diff --git a/internal/e2e/suites/networking/networking_test.go b/internal/e2e/suites/networking/networking_test.go index 5a0eceaefe..48ce302d14 100644 --- a/internal/e2e/suites/networking/networking_test.go +++ b/internal/e2e/suites/networking/networking_test.go @@ -27,12 +27,12 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/atenet" - "github.com/agent-substrate/substrate/internal/e2e" - "github.com/agent-substrate/substrate/internal/localca" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/testcert" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/atenet" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/localca" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/internal/testcert" + "go.ate.dev/substrate/pkg/proto/ateapipb" corev1 "k8s.io/api/core/v1" discoveryv1 "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -87,7 +87,7 @@ func TestActorDirectAccess(t *testing.T) { func egressHTTPTarget() e2e.ServerPod { return e2e.ServerPod{ Name: "egresshttp", - ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + ImportPath: "go.ate.dev/substrate/internal/e2e/fixtures/testserver", Args: []string{"http"}, Port: 80, TargetPort: 8080, @@ -202,7 +202,7 @@ func TestActorEgressMultiplexing(t *testing.T) { func egressHTTPSTarget(name, body string) e2e.ServerPod { return e2e.ServiceDNSTLS(e2e.ServerPod{ Name: name, - ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + ImportPath: "go.ate.dev/substrate/internal/e2e/fixtures/testserver", Args: []string{"http", "--body=" + body}, }) } @@ -242,7 +242,7 @@ func TestActorEgressHTTPS(t *testing.T) { // serves nothing but /healthz, which is all this target is dialed for. var httpTarget = e2e.ServerPod{ Name: "httptarget", - ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + ImportPath: "go.ate.dev/substrate/internal/e2e/fixtures/testserver", Args: []string{"http"}, Port: 8080, } @@ -597,7 +597,7 @@ func TestActorEgressHTTPSNonStandardPort(t *testing.T) { const expectedBody = "https nonstandard origin" target, address, _ := prepareProtocolOrigin(t, ctx, e2e.ServerPod{ Name: "https-origin", - ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + ImportPath: "go.ate.dev/substrate/internal/e2e/fixtures/testserver", Args: []string{"http", "--body=" + expectedBody}, Port: 8080, TLSPort: 8443, diff --git a/internal/e2e/suites/networking/testmain_test.go b/internal/e2e/suites/networking/testmain_test.go index 2d31b9c9f5..9e9d471886 100644 --- a/internal/e2e/suites/networking/testmain_test.go +++ b/internal/e2e/suites/networking/testmain_test.go @@ -18,7 +18,7 @@ import ( "os" "testing" - "github.com/agent-substrate/substrate/internal/e2e" + "go.ate.dev/substrate/internal/e2e" ) func TestMain(m *testing.M) { os.Exit(e2e.RunTestMain(m)) } diff --git a/internal/e2e/suites/networking/websocketegress_test.go b/internal/e2e/suites/networking/websocketegress_test.go index 59f5c2cc36..ab6fa18369 100644 --- a/internal/e2e/suites/networking/websocketegress_test.go +++ b/internal/e2e/suites/networking/websocketegress_test.go @@ -19,7 +19,7 @@ import ( "net/http" "testing" - "github.com/agent-substrate/substrate/internal/e2e" + "go.ate.dev/substrate/internal/e2e" ) // egressWebSocketResponse mirrors observations returned by the actor. The @@ -41,7 +41,7 @@ type egressWebSocketMessage struct { func TestActorEgressWebSocket(t *testing.T) { ctx := t.Context() _, address, _ := prepareProtocolOrigin(t, ctx, e2e.ServerPod{ - Name: "ws-origin", ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + Name: "ws-origin", ImportPath: "go.ate.dev/substrate/internal/e2e/fixtures/testserver", Args: []string{"websocket", "--echo"}, Port: 80, TargetPort: 8080, HealthPath: "/readyz", }, false) _, router, actorRef := prepareProtocolActor(t, ctx, "ws") @@ -69,7 +69,7 @@ func TestActorEgressWebSocket(t *testing.T) { func TestActorEgressSecureWebSocket(t *testing.T) { ctx := t.Context() _, address, _ := prepareProtocolOrigin(t, ctx, e2e.ServerPod{ - Name: "wss-origin", ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + Name: "wss-origin", ImportPath: "go.ate.dev/substrate/internal/e2e/fixtures/testserver", Args: []string{"websocket", "--echo"}, Port: 80, TargetPort: 8080, TLSPort: 443, HealthPath: "/readyz", }, true) _, router, actorRef := prepareProtocolActor(t, ctx, "wss") diff --git a/internal/e2e/suites/networking/websocketingress_test.go b/internal/e2e/suites/networking/websocketingress_test.go index d05153bb72..48dedc97d8 100644 --- a/internal/e2e/suites/networking/websocketingress_test.go +++ b/internal/e2e/suites/networking/websocketingress_test.go @@ -22,10 +22,10 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/atenet" - "github.com/agent-substrate/substrate/internal/e2e" - "github.com/agent-substrate/substrate/internal/resources" "github.com/gorilla/websocket" + "go.ate.dev/substrate/internal/atenet" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/resources" ) // deployWebsocketFixture installs the websocket fixture for the sandbox class diff --git a/internal/e2e/suites/networkpolicy/networkpolicy_test.go b/internal/e2e/suites/networkpolicy/networkpolicy_test.go index 3f9efb1b6d..c357e81cce 100644 --- a/internal/e2e/suites/networkpolicy/networkpolicy_test.go +++ b/internal/e2e/suites/networkpolicy/networkpolicy_test.go @@ -33,12 +33,12 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "github.com/agent-substrate/substrate/internal/ateclient" - "github.com/agent-substrate/substrate/internal/e2e" - "github.com/agent-substrate/substrate/internal/portforward" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/ateclient" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/portforward" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) const ( @@ -283,7 +283,7 @@ func testserverPod(name, namespace string) e2e.ServerPod { return e2e.ServerPod{ Name: name, Namespace: namespace, - ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + ImportPath: "go.ate.dev/substrate/internal/e2e/fixtures/testserver", Args: []string{"http"}, Port: testserverPort, } diff --git a/internal/e2e/suites/networkpolicy/testmain_test.go b/internal/e2e/suites/networkpolicy/testmain_test.go index 6a686e2edb..18a7e30e7b 100644 --- a/internal/e2e/suites/networkpolicy/testmain_test.go +++ b/internal/e2e/suites/networkpolicy/testmain_test.go @@ -18,7 +18,7 @@ import ( "os" "testing" - "github.com/agent-substrate/substrate/internal/e2e" + "go.ate.dev/substrate/internal/e2e" ) func run(m *testing.M) int { diff --git a/internal/e2e/suites/parking/parking_test.go b/internal/e2e/suites/parking/parking_test.go index 0df9c7972c..2cd23ab812 100644 --- a/internal/e2e/suites/parking/parking_test.go +++ b/internal/e2e/suites/parking/parking_test.go @@ -30,9 +30,9 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/e2e" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // parkingAtespace is where deployParkingFixture applies the fixture, and the diff --git a/internal/e2e/suites/parking/testmain_test.go b/internal/e2e/suites/parking/testmain_test.go index 060577f537..d13fb98211 100644 --- a/internal/e2e/suites/parking/testmain_test.go +++ b/internal/e2e/suites/parking/testmain_test.go @@ -18,7 +18,7 @@ import ( "os" "testing" - "github.com/agent-substrate/substrate/internal/e2e" + "go.ate.dev/substrate/internal/e2e" ) func TestMain(m *testing.M) { os.Exit(e2e.RunTestMain(m)) } diff --git a/internal/e2e/suites/sandboxconfig/classdefault_test.go b/internal/e2e/suites/sandboxconfig/classdefault_test.go index 27bba75312..220cf0689c 100644 --- a/internal/e2e/suites/sandboxconfig/classdefault_test.go +++ b/internal/e2e/suites/sandboxconfig/classdefault_test.go @@ -36,8 +36,8 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/e2e" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/internal/e2e" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" corev1 "k8s.io/api/core/v1" eventsv1 "k8s.io/api/events/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/internal/e2e/suites/sandboxconfig/testmain_test.go b/internal/e2e/suites/sandboxconfig/testmain_test.go index e61c70574a..409c439196 100644 --- a/internal/e2e/suites/sandboxconfig/testmain_test.go +++ b/internal/e2e/suites/sandboxconfig/testmain_test.go @@ -18,7 +18,7 @@ import ( "os" "testing" - "github.com/agent-substrate/substrate/internal/e2e" + "go.ate.dev/substrate/internal/e2e" ) func TestMain(m *testing.M) { os.Exit(e2e.RunTestMain(m)) } diff --git a/internal/e2e/suites/security/crossatespace_test.go b/internal/e2e/suites/security/crossatespace_test.go index 0637f3cc33..5d5e1ca58f 100644 --- a/internal/e2e/suites/security/crossatespace_test.go +++ b/internal/e2e/suites/security/crossatespace_test.go @@ -37,10 +37,10 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/e2e" - "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) diff --git a/internal/e2e/suites/security/testmain_test.go b/internal/e2e/suites/security/testmain_test.go index 3f30d513f4..2c1a5b0a14 100644 --- a/internal/e2e/suites/security/testmain_test.go +++ b/internal/e2e/suites/security/testmain_test.go @@ -18,7 +18,7 @@ import ( "os" "testing" - "github.com/agent-substrate/substrate/internal/e2e" + "go.ate.dev/substrate/internal/e2e" ) func TestMain(m *testing.M) { os.Exit(e2e.RunTestMain(m)) } diff --git a/internal/e2e/suites/sizing/sizing_test.go b/internal/e2e/suites/sizing/sizing_test.go index 6142f41321..7cc179b066 100644 --- a/internal/e2e/suites/sizing/sizing_test.go +++ b/internal/e2e/suites/sizing/sizing_test.go @@ -21,9 +21,9 @@ import ( "net/http" "testing" - "github.com/agent-substrate/substrate/internal/e2e" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) const ( diff --git a/internal/e2e/suites/sizing/testmain_test.go b/internal/e2e/suites/sizing/testmain_test.go index 5ba57d803f..fc6380ad29 100644 --- a/internal/e2e/suites/sizing/testmain_test.go +++ b/internal/e2e/suites/sizing/testmain_test.go @@ -18,7 +18,7 @@ import ( "os" "testing" - "github.com/agent-substrate/substrate/internal/e2e" + "go.ate.dev/substrate/internal/e2e" ) func TestMain(m *testing.M) { diff --git a/internal/e2e/template.go b/internal/e2e/template.go index 5339e2b89c..102d49ce5e 100644 --- a/internal/e2e/template.go +++ b/internal/e2e/template.go @@ -19,8 +19,8 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/internal/e2e/template_test.go b/internal/e2e/template_test.go index d870add40b..3211d14f34 100644 --- a/internal/e2e/template_test.go +++ b/internal/e2e/template_test.go @@ -18,8 +18,8 @@ import ( "context" "testing" - "github.com/agent-substrate/substrate/internal/ateclient" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/ateclient" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" ) diff --git a/internal/e2e/trustbundle.go b/internal/e2e/trustbundle.go index 11c8318453..01f0dbc781 100644 --- a/internal/e2e/trustbundle.go +++ b/internal/e2e/trustbundle.go @@ -23,8 +23,8 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/clustertrustbundle" - "github.com/agent-substrate/substrate/internal/localca" + "go.ate.dev/substrate/internal/clustertrustbundle" + "go.ate.dev/substrate/internal/localca" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/internal/e2e/trustbundle_test.go b/internal/e2e/trustbundle_test.go index 7b4fe5d0f1..0bcacff98d 100644 --- a/internal/e2e/trustbundle_test.go +++ b/internal/e2e/trustbundle_test.go @@ -24,8 +24,8 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/localca" - "github.com/agent-substrate/substrate/internal/pemutil" + "go.ate.dev/substrate/internal/localca" + "go.ate.dev/substrate/internal/pemutil" ) // TestAddCA pins what lets the identity suite run alongside the suites doing diff --git a/internal/egresspolicy/egresspolicy.go b/internal/egresspolicy/egresspolicy.go index dd9ae88cf2..4d7e984da7 100644 --- a/internal/egresspolicy/egresspolicy.go +++ b/internal/egresspolicy/egresspolicy.go @@ -33,7 +33,7 @@ import ( "k8s.io/apimachinery/pkg/api/validate/content" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // Destination is what a request is going to, as far as the leg evaluating it diff --git a/internal/egresspolicy/egresspolicy_test.go b/internal/egresspolicy/egresspolicy_test.go index 9488d50e99..a4561754e0 100644 --- a/internal/egresspolicy/egresspolicy_test.go +++ b/internal/egresspolicy/egresspolicy_test.go @@ -19,7 +19,7 @@ import ( "slices" "testing" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) func httpRule(patterns ...string) *ateapipb.EgressRule { diff --git a/internal/hardware/cpufeatures.go b/internal/hardware/cpufeatures.go new file mode 100644 index 0000000000..075f81b78a --- /dev/null +++ b/internal/hardware/cpufeatures.go @@ -0,0 +1,244 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hardware + +import ( + "bufio" + "fmt" + "io" + "maps" + "math/bits" + "os" + "runtime" + "slices" + "strconv" + "strings" +) + +// cpuFeatureTables maps each supported architecture to its CPU feature table +// from cpufeatures_table.go, whose header says where the tables come from +// and what a bit means. +var cpuFeatureTables = map[string]featureTable{ + "amd64": newFeatureTable(amd64CPUFeatures), + "arm64": newFeatureTable(arm64CPUFeatures), +} + +// featureTable is one architecture's CPU feature names by bit, with the +// reverse lookup EncodeCPUFeatures uses and the set of bits that have a +// name, which DecodeCPUFeatures validates against. +type featureTable struct { + names map[int]string + bits []int // the keys of names, ascending + index map[string]int + defined featureMask +} + +// newFeatureTable indexes a table. A bit outside featureMask is a table +// error, so it is a panic rather than a dropped name. +func newFeatureTable(names map[int]string) featureTable { + t := featureTable{ + names: names, + bits: slices.Sorted(maps.Keys(names)), + index: make(map[string]int, len(names)), + } + for bit, name := range names { + if bit < 0 || bit >= featureMaskWords*64 { + panic(fmt.Sprintf("hardware: CPU feature %q has bit %d outside a %d-bit featureMask", name, bit, featureMaskWords*64)) + } + t.defined.set(bit) + t.index[name] = bit + } + return t +} + +// featureMaskWords is the width of a cpu_features bitmask in 64-bit words. +// gVisor's amd64 blocks 0-8 use bits up to 261, and 6 words leave room for +// blocks through 11 within SchemaVersionV1. +const featureMaskWords = 6 + +// featureMask is a decoded cpu_features value: bit i set means the feature at +// bit i of the architecture's table is present. It is a fixed-size array so +// matching on the scheduler's hot path allocates nothing. +type featureMask [featureMaskWords]uint64 + +func (m *featureMask) set(bit int) { + m[bit/64] |= 1 << (bit % 64) +} + +func (m featureMask) has(bit int) bool { + return m[bit/64]&(1<<(bit%64)) != 0 +} + +// subsetOf reports whether every bit set in m is also set in w. +func (m featureMask) subsetOf(w featureMask) bool { + for i := range m { + if m[i]&^w[i] != 0 { + return false + } + } + return true +} + +// firstNotIn is the lowest bit set in m but not in w, or -1 if m is a +// subset of w. +func (m featureMask) firstNotIn(w featureMask) int { + for i := range m { + if extra := m[i] &^ w[i]; extra != 0 { + return i*64 + bits.TrailingZeros64(extra) + } + } + return -1 +} + +// String is the canonical encoding: lowercase hex, no prefix, no leading +// zeros, "0" for the empty mask. Equal sets encode to equal strings, which +// is what proto equality on a recorded runtime compares. +func (m featureMask) String() string { + top := featureMaskWords - 1 + for top > 0 && m[top] == 0 { + top-- + } + var sb strings.Builder + sb.WriteString(strconv.FormatUint(m[top], 16)) + for i := top - 1; i >= 0; i-- { + fmt.Fprintf(&sb, "%016x", m[i]) + } + return sb.String() +} + +// parseFeatureMask parses the canonical encoding and rejects anything else. +func parseFeatureMask(s string) (featureMask, bool) { + var m featureMask + if s == "" || len(s) > featureMaskWords*16 || (len(s) > 1 && s[0] == '0') { + return m, false + } + for i := 0; i < len(s); i++ { + c := s[len(s)-1-i] + var v uint64 + switch { + case '0' <= c && c <= '9': + v = uint64(c - '0') + case 'a' <= c && c <= 'f': + v = uint64(c-'a') + 10 + default: + return featureMask{}, false + } + m[i/16] |= v << (4 * (i % 16)) + } + return m, true +} + +// EncodeCPUFeatures encodes the feature names `runsc cpu-features` printed +// for arch as a cpu_features value. Blank names are skipped; duplicates and +// order do not matter. An unknown name is an error rather than a dropped +// bit: a snapshot encoded without a feature its sandbox used would match +// workers that lack it. +func EncodeCPUFeatures(arch string, features []string) (string, error) { + table, ok := cpuFeatureTables[arch] + if !ok { + return "", fmt.Errorf("unsupported architecture %q for cpu_features", arch) + } + var m featureMask + for _, raw := range features { + name := strings.TrimSpace(raw) + if name == "" { + continue + } + bit, known := table.index[name] + if !known { + return "", fmt.Errorf("unknown %s CPU feature %q", arch, name) + } + m.set(bit) + } + return m.String(), nil +} + +// DecodeCPUFeatures returns the feature names a cpu_features value encodes +// for arch, in bit order. A set bit with no name is an error: the hardware +// reserves it, so no encoder of this schema could have produced the value. +func DecodeCPUFeatures(arch, value string) ([]string, error) { + table, ok := cpuFeatureTables[arch] + if !ok { + return nil, fmt.Errorf("unsupported architecture %q for cpu_features", arch) + } + m, ok := parseFeatureMask(value) + if !ok { + return nil, fmt.Errorf("invalid cpu_features value %q", value) + } + if bit := m.firstNotIn(table.defined); bit >= 0 { + return nil, fmt.Errorf("cpu_features value %q sets bit %d, which is not a known %s feature", value, bit, arch) + } + var out []string + for _, bit := range table.bits { + if m.has(bit) { + out = append(out, table.names[bit]) + } + } + return out, nil +} + +// ProbeCPUIdentity reads /proc/cpuinfo for the host CPU's vendor and model: +// the AttrCPUVendor and AttrCPUModel values of a microVM runtime. +func ProbeCPUIdentity() (vendor, model string, err error) { + f, err := os.Open("/proc/cpuinfo") + if err != nil { + return "", "", fmt.Errorf("while opening /proc/cpuinfo: %w", err) + } + defer f.Close() + return parseCPUInfo(runtime.GOARCH, f) +} + +// parseCPUInfo extracts the vendor and model for arch from /proc/cpuinfo +// content, from the first processor that lists them: +// - amd64: vendor_id (e.g. "GenuineIntel") and ":" +// (e.g. "6:143"). +// - arm64: CPU implementer (e.g. "0x41") and +// "::" (e.g. "0x0:0xd40:1"). +func parseCPUInfo(arch string, r io.Reader) (vendor, model string, err error) { + // The vendor key first, then the keys the model is composed of. + var keys []string + switch arch { + case "amd64": + keys = []string{"vendor_id", "cpu family", "model"} + case "arm64": + keys = []string{"CPU implementer", "CPU variant", "CPU part", "CPU revision"} + default: + return "", "", fmt.Errorf("unsupported architecture %q for CPU identity", arch) + } + fields := make(map[string]string, len(keys)) + scanner := bufio.NewScanner(r) + for len(fields) < len(keys) && scanner.Scan() { + k, v, ok := strings.Cut(scanner.Text(), ":") + if !ok { + continue + } + k = strings.TrimSpace(k) + if _, seen := fields[k]; !seen && slices.Contains(keys, k) { + fields[k] = strings.TrimSpace(v) + } + } + if err := scanner.Err(); err != nil { + return "", "", fmt.Errorf("while reading /proc/cpuinfo: %w", err) + } + values := make([]string, 0, len(keys)) + for _, k := range keys { + v, ok := fields[k] + if !ok || v == "" { + return "", "", fmt.Errorf("missing %q in /proc/cpuinfo", k) + } + values = append(values, v) + } + return values[0], strings.Join(values[1:], ":"), nil +} diff --git a/internal/hardware/cpufeatures_table.go b/internal/hardware/cpufeatures_table.go new file mode 100644 index 0000000000..7fc97ad7f4 --- /dev/null +++ b/internal/hardware/cpufeatures_table.go @@ -0,0 +1,244 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hardware + +// CPU feature tables for the cpu_features compatibility attribute. +// +// `runsc cpu-features` prints the features a gVisor sandbox exposes by name, +// and the control plane needs them as a bitmask it can compare by subset. +// Each table maps a bit to the name gVisor prints for it. A bit is the value +// of the gVisor Feature constant named in the entry's comment: on amd64, +// 32*block + the feature's bit in that block's CPUID register, which the +// hardware fixes; on arm64, the value gVisor declares. A bit therefore never +// moves between gVisor versions, and a bit with no entry is one the hardware +// reserves. +// +// The tables were produced once from gVisor's pkg/cpuid sources at +// gvisor.dev/gvisor@v0.0.0-20261014212059-ddc37c698141: for every constant of +// type Feature in features_amd64.go and features_arm64.go with an entry in +// that file's allFeatures map, the constant's value (evaluated with go/types) +// keyed to the entry's display name. To take features from a newer gVisor, +// add its new constants the same way; existing entries do not change. +// +// TODO: Remove the tables, EncodeCPUFeatures, and DecodeCPUFeatures once +// runsc can print its feature set as this bitmask itself, so atelet records +// the value directly instead of mapping names. The mask format and the +// subset matching stay. +var ( + amd64CPUFeatures = map[int]string{ + 0: "pni", // X86FeatureSSE3 (block 0, bit 0) + 1: "pclmulqdq", // X86FeaturePCLMULDQ (block 0, bit 1) + 2: "dtes64", // X86FeatureDTES64 (block 0, bit 2) + 3: "monitor", // X86FeatureMONITOR (block 0, bit 3) + 4: "ds_cpl", // X86FeatureDSCPL (block 0, bit 4) + 5: "vmx", // X86FeatureVMX (block 0, bit 5) + 6: "smx", // X86FeatureSMX (block 0, bit 6) + 7: "est", // X86FeatureEST (block 0, bit 7) + 8: "tm2", // X86FeatureTM2 (block 0, bit 8) + 9: "ssse3", // X86FeatureSSSE3 (block 0, bit 9) + 10: "cid", // X86FeatureCNXTID (block 0, bit 10) + 11: "sdbg", // X86FeatureSDBG (block 0, bit 11) + 12: "fma", // X86FeatureFMA (block 0, bit 12) + 13: "cx16", // X86FeatureCX16 (block 0, bit 13) + 14: "xtpr", // X86FeatureXTPR (block 0, bit 14) + 15: "pdcm", // X86FeaturePDCM (block 0, bit 15) + 17: "pcid", // X86FeaturePCID (block 0, bit 17) + 18: "dca", // X86FeatureDCA (block 0, bit 18) + 19: "sse4_1", // X86FeatureSSE4_1 (block 0, bit 19) + 20: "sse4_2", // X86FeatureSSE4_2 (block 0, bit 20) + 21: "x2apic", // X86FeatureX2APIC (block 0, bit 21) + 22: "movbe", // X86FeatureMOVBE (block 0, bit 22) + 23: "popcnt", // X86FeaturePOPCNT (block 0, bit 23) + 24: "tsc_deadline_timer", // X86FeatureTSCD (block 0, bit 24) + 25: "aes", // X86FeatureAES (block 0, bit 25) + 26: "xsave", // X86FeatureXSAVE (block 0, bit 26) + 27: "osxsave", // X86FeatureOSXSAVE (block 0, bit 27) + 28: "avx", // X86FeatureAVX (block 0, bit 28) + 29: "f16c", // X86FeatureF16C (block 0, bit 29) + 30: "rdrand", // X86FeatureRDRAND (block 0, bit 30) + 31: "hypervisor", // X86FeatureHypervisor (block 0, bit 31) + 32: "fpu", // X86FeatureFPU (block 1, bit 0) + 33: "vme", // X86FeatureVME (block 1, bit 1) + 34: "de", // X86FeatureDE (block 1, bit 2) + 35: "pse", // X86FeaturePSE (block 1, bit 3) + 36: "tsc", // X86FeatureTSC (block 1, bit 4) + 37: "msr", // X86FeatureMSR (block 1, bit 5) + 38: "pae", // X86FeaturePAE (block 1, bit 6) + 39: "mce", // X86FeatureMCE (block 1, bit 7) + 40: "cx8", // X86FeatureCX8 (block 1, bit 8) + 41: "apic", // X86FeatureAPIC (block 1, bit 9) + 43: "sep", // X86FeatureSEP (block 1, bit 11) + 44: "mtrr", // X86FeatureMTRR (block 1, bit 12) + 45: "pge", // X86FeaturePGE (block 1, bit 13) + 46: "mca", // X86FeatureMCA (block 1, bit 14) + 47: "cmov", // X86FeatureCMOV (block 1, bit 15) + 48: "pat", // X86FeaturePAT (block 1, bit 16) + 49: "pse36", // X86FeaturePSE36 (block 1, bit 17) + 50: "pn", // X86FeaturePSN (block 1, bit 18) + 51: "clflush", // X86FeatureCLFSH (block 1, bit 19) + 53: "dts", // X86FeatureDS (block 1, bit 21) + 54: "acpi", // X86FeatureACPI (block 1, bit 22) + 55: "mmx", // X86FeatureMMX (block 1, bit 23) + 56: "fxsr", // X86FeatureFXSR (block 1, bit 24) + 57: "sse", // X86FeatureSSE (block 1, bit 25) + 58: "sse2", // X86FeatureSSE2 (block 1, bit 26) + 59: "ss", // X86FeatureSS (block 1, bit 27) + 60: "ht", // X86FeatureHTT (block 1, bit 28) + 61: "tm", // X86FeatureTM (block 1, bit 29) + 62: "ia64", // X86FeatureIA64 (block 1, bit 30) + 63: "pbe", // X86FeaturePBE (block 1, bit 31) + 64: "fsgsbase", // X86FeatureFSGSBase (block 2, bit 0) + 65: "tsc_adjust", // X86FeatureTSC_ADJUST (block 2, bit 1) + 67: "bmi1", // X86FeatureBMI1 (block 2, bit 3) + 68: "hle", // X86FeatureHLE (block 2, bit 4) + 69: "avx2", // X86FeatureAVX2 (block 2, bit 5) + 70: "fdp_excptn_only", // X86FeatureFDP_EXCPTN_ONLY (block 2, bit 6) + 71: "smep", // X86FeatureSMEP (block 2, bit 7) + 72: "bmi2", // X86FeatureBMI2 (block 2, bit 8) + 73: "erms", // X86FeatureERMS (block 2, bit 9) + 74: "invpcid", // X86FeatureINVPCID (block 2, bit 10) + 75: "rtm", // X86FeatureRTM (block 2, bit 11) + 76: "cqm", // X86FeatureCQM (block 2, bit 12) + 77: "fpcsds", // X86FeatureFPCSDS (block 2, bit 13) + 78: "mpx", // X86FeatureMPX (block 2, bit 14) + 79: "rdt_a", // X86FeatureRDT (block 2, bit 15) + 80: "avx512f", // X86FeatureAVX512F (block 2, bit 16) + 81: "avx512dq", // X86FeatureAVX512DQ (block 2, bit 17) + 82: "rdseed", // X86FeatureRDSEED (block 2, bit 18) + 83: "adx", // X86FeatureADX (block 2, bit 19) + 84: "smap", // X86FeatureSMAP (block 2, bit 20) + 87: "clfushopt", // X86FeatureCLFLUSHOPT (block 2, bit 23) + 88: "clwb", // X86FeatureCLWB (block 2, bit 24) + 89: "ipt", // X86FeatureIPT (block 2, bit 25) + 90: "avx512pf", // X86FeatureAVX512PF (block 2, bit 26) + 91: "avx512er", // X86FeatureAVX512ER (block 2, bit 27) + 92: "avx512cd", // X86FeatureAVX512CD (block 2, bit 28) + 93: "sha_ni", // X86FeatureSHA (block 2, bit 29) + 94: "avx512bw", // X86FeatureAVX512BW (block 2, bit 30) + 95: "avx512vl", // X86FeatureAVX512VL (block 2, bit 31) + 96: "prefetchwt1", // X86FeaturePREFETCHWT1 (block 3, bit 0) + 97: "avx512vbmi", // X86FeatureAVX512VBMI (block 3, bit 1) + 98: "umip", // X86FeatureUMIP (block 3, bit 2) + 99: "pku", // X86FeaturePKU (block 3, bit 3) + 100: "ospke", // X86FeatureOSPKE (block 3, bit 4) + 101: "waitpkg", // X86FeatureWAITPKG (block 3, bit 5) + 102: "avx512_vbmi2", // X86FeatureAVX512_VBMI2 (block 3, bit 6) + 103: "cet_ss", // X86FeatureCET_SS (block 3, bit 7) + 104: "gfni", // X86FeatureGFNI (block 3, bit 8) + 105: "vaes", // X86FeatureVAES (block 3, bit 9) + 106: "vpclmulqdq", // X86FeatureVPCLMULQDQ (block 3, bit 10) + 107: "avx512_vnni", // X86FeatureAVX512_VNNI (block 3, bit 11) + 108: "avx512_bitalg", // X86FeatureAVX512_BITALG (block 3, bit 12) + 109: "tme", // X86FeatureTME (block 3, bit 13) + 110: "avx512_vpopcntdq", // X86FeatureAVX512_VPOPCNTDQ (block 3, bit 14) + 112: "la57", // X86FeatureLA57 (block 3, bit 16) + 118: "rdpid", // X86FeatureRDPID (block 3, bit 22) + 121: "cldemote", // X86FeatureCLDEMOTE (block 3, bit 25) + 123: "movdiri", // X86FeatureMOVDIRI (block 3, bit 27) + 124: "movdir64b", // X86FeatureMOVDIR64B (block 3, bit 28) + 128: "xsaveopt", // X86FeatureXSAVEOPT (block 4, bit 0) + 129: "xsavec", // X86FeatureXSAVEC (block 4, bit 1) + 130: "xgetbv1", // X86FeatureXGETBV1 (block 4, bit 2) + 131: "xsaves", // X86FeatureXSAVES (block 4, bit 3) + 160: "lahf_lm", // X86FeatureLAHF64 (block 5, bit 0) + 161: "cmp_legacy", // X86FeatureCMP_LEGACY (block 5, bit 1) + 162: "svm", // X86FeatureSVM (block 5, bit 2) + 163: "extapic", // X86FeatureEXTAPIC (block 5, bit 3) + 164: "cr8_legacy", // X86FeatureCR8_LEGACY (block 5, bit 4) + 165: "abm", // X86FeatureLZCNT (block 5, bit 5) + 166: "sse4a", // X86FeatureSSE4A (block 5, bit 6) + 167: "misalignsse", // X86FeatureMISALIGNSSE (block 5, bit 7) + 168: "3dnowprefetch", // X86FeaturePREFETCHW (block 5, bit 8) + 169: "osvw", // X86FeatureOSVW (block 5, bit 9) + 170: "ibs", // X86FeatureIBS (block 5, bit 10) + 171: "xop", // X86FeatureXOP (block 5, bit 11) + 172: "skinit", // X86FeatureSKINIT (block 5, bit 12) + 173: "wdt", // X86FeatureWDT (block 5, bit 13) + 175: "lwp", // X86FeatureLWP (block 5, bit 15) + 176: "fma4", // X86FeatureFMA4 (block 5, bit 16) + 177: "tce", // X86FeatureTCE (block 5, bit 17) + 181: "tbm", // X86FeatureTBM (block 5, bit 21) + 182: "topoext", // X86FeatureTOPOLOGY (block 5, bit 22) + 183: "perfctr_core", // X86FeaturePERFCTR_CORE (block 5, bit 23) + 184: "perfctr_nb", // X86FeaturePERFCTR_NB (block 5, bit 24) + 186: "bpext", // X86FeatureBPEXT (block 5, bit 26) + 187: "ptsc", // X86FeaturePERFCTR_TSC (block 5, bit 27) + 188: "perfctr_llc", // X86FeaturePERFCTR_LLC (block 5, bit 28) + 189: "mwaitx", // X86FeatureMWAITX (block 5, bit 29) + 190: "ad_mask_extn", // X86FeatureADMSKEXTN (block 5, bit 30) + 203: "syscall", // X86FeatureSYSCALL (block 6, bit 11) + 212: "nx", // X86FeatureNX (block 6, bit 20) + 214: "mmxext", // X86FeatureMMXEXT (block 6, bit 22) + 217: "fxsr_opt", // X86FeatureFXSR_OPT (block 6, bit 25) + 218: "pdpe1gb", // X86FeatureGBPAGES (block 6, bit 26) + 219: "rdtscp", // X86FeatureRDTSCP (block 6, bit 27) + 221: "lm", // X86FeatureLM (block 6, bit 29) + 222: "3dnowext", // X86Feature3DNOWEXT (block 6, bit 30) + 223: "3dnow", // X86Feature3DNOW (block 6, bit 31) + 226: "avx512_4vnniw", // X86FeatureAVX512_4VNNIW (block 7, bit 2) + 227: "avx512_4fmaps", // X86FeatureAVX512_4FMAPS (block 7, bit 3) + 228: "fsrm", // X86FeatureFSRM (block 7, bit 4) + 232: "avx512_vp2intersect", // X86FeatureAVX512_VP2INTERSECT (block 7, bit 8) + 233: "srbds_ctrl", // X86FeatureSRBDS_CTRL (block 7, bit 9) + 234: "md_clear", // X86FeatureMD_CLEAR (block 7, bit 10) + 235: "rtm_always_abort", // X86FeatureRTM_ALWAYS_ABORT (block 7, bit 11) + 237: "tsx_force_abort", // X86FeatureTSX_FORCE_ABORT (block 7, bit 13) + 238: "serialize", // X86FeatureSERIALIZE (block 7, bit 14) + 239: "hybrid_cpu", // X86FeatureHYBRID_CPU (block 7, bit 15) + 240: "tsxldtrk", // X86FeatureTSXLDTRK (block 7, bit 16) + 242: "pconfig", // X86FeaturePCONFIG (block 7, bit 18) + 243: "arch_lbr", // X86FeatureARCH_LBR (block 7, bit 19) + 244: "ibt", // X86FeatureIBT (block 7, bit 20) + 246: "amx_bf16", // X86FeatureAMX_BF16 (block 7, bit 22) + 247: "avx512_fp16", // X86FeatureAVX512_FP16 (block 7, bit 23) + 248: "amx_tile", // X86FeatureAMX_TILE (block 7, bit 24) + 249: "amx_int8", // X86FeatureAMX_INT8 (block 7, bit 25) + 250: "spec_ctrl", // X86FeatureSPEC_CTRL (block 7, bit 26) + 251: "intel_stibp", // X86FeatureINTEL_STIBP (block 7, bit 27) + 252: "flush_l1d", // X86FeatureFLUSH_L1D (block 7, bit 28) + 253: "arch_capabilities", // X86FeatureARCH_CAPABILITIES (block 7, bit 29) + 254: "core_capabilities", // X86FeatureCORE_CAPABILITIES (block 7, bit 30) + 255: "spec_ctrl_ssbd", // X86FeatureSPEC_CTRL_SSBD (block 7, bit 31) + 260: "avx_vnni", // X86FeatureAVX_VNNI (block 8, bit 4) + 261: "avx512_bf16", // X86FeatureAVX512_BF16 (block 8, bit 5) + } + + arm64CPUFeatures = map[int]string{ + 0: "fp", // ARM64FeatureFP + 1: "asimd", // ARM64FeatureASIMD + 2: "evtstrm", // ARM64FeatureEVTSTRM + 3: "aes", // ARM64FeatureAES + 4: "pmull", // ARM64FeaturePMULL + 5: "sha1", // ARM64FeatureSHA1 + 6: "sha2", // ARM64FeatureSHA2 + 7: "crc32", // ARM64FeatureCRC32 + 8: "atomics", // ARM64FeatureATOMICS + 9: "fphp", // ARM64FeatureFPHP + 10: "asimdhp", // ARM64FeatureASIMDHP + 11: "cpuid", // ARM64FeatureCPUID + 12: "asimdrdm", // ARM64FeatureASIMDRDM + 13: "jscvt", // ARM64FeatureJSCVT + 14: "fcma", // ARM64FeatureFCMA + 15: "lrcpc", // ARM64FeatureLRCPC + 16: "dcpop", // ARM64FeatureDCPOP + 17: "sha3", // ARM64FeatureSHA3 + 18: "sm3", // ARM64FeatureSM3 + 19: "sm4", // ARM64FeatureSM4 + 20: "asimddp", // ARM64FeatureASIMDDP + 21: "sha512", // ARM64FeatureSHA512 + 22: "sve", // ARM64FeatureSVE + 23: "asimdfhm", // ARM64FeatureASIMDFHM + } +) diff --git a/internal/hardware/cpufeatures_test.go b/internal/hardware/cpufeatures_test.go new file mode 100644 index 0000000000..51b2ad6fd4 --- /dev/null +++ b/internal/hardware/cpufeatures_test.go @@ -0,0 +1,333 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hardware + +import ( + "slices" + "strconv" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestEncodeDecodeCPUFeatures(t *testing.T) { + for _, arch := range []string{"amd64", "arm64"} { + t.Run(arch+"/all_features_roundtrip", func(t *testing.T) { + table := cpuFeatureTables[arch] + all := make([]string, 0, len(table.bits)) + for _, bit := range table.bits { + all = append(all, table.names[bit]) + } + mask, err := EncodeCPUFeatures(arch, all) + if err != nil { + t.Fatalf("EncodeCPUFeatures(%q, all) error: %v", arch, err) + } + got, err := DecodeCPUFeatures(arch, mask) + if err != nil { + t.Fatalf("DecodeCPUFeatures(%q, %q) error: %v", arch, mask, err) + } + if diff := cmp.Diff(all, got); diff != "" { + t.Errorf("round-trip mismatch (-want +got):\n%s", diff) + } + }) + + t.Run(arch+"/empty_roundtrip", func(t *testing.T) { + mask, err := EncodeCPUFeatures(arch, nil) + if err != nil { + t.Fatalf("EncodeCPUFeatures(%q, nil) error: %v", arch, err) + } + if mask != "0" { + t.Errorf("EncodeCPUFeatures(%q, nil) = %q, want \"0\"", arch, mask) + } + got, err := DecodeCPUFeatures(arch, mask) + if err != nil { + t.Fatalf("DecodeCPUFeatures(%q, %q) error: %v", arch, mask, err) + } + if len(got) != 0 { + t.Errorf("DecodeCPUFeatures(%q, \"0\") = %v, want empty", arch, got) + } + }) + } + + t.Run("duplicates_and_order_normalized", func(t *testing.T) { + m1, err := EncodeCPUFeatures("amd64", []string{"avx2", "pni", "avx2", ""}) + if err != nil { + t.Fatal(err) + } + m2, err := EncodeCPUFeatures("amd64", []string{"pni", "avx2"}) + if err != nil { + t.Fatal(err) + } + if m1 != m2 { + t.Errorf("EncodeCPUFeatures with duplicates/reordering = %q, want %q", m1, m2) + } + got, err := DecodeCPUFeatures("amd64", m1) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff([]string{"pni", "avx2"}, got); diff != "" { + t.Errorf("decoded features mismatch (-want +got):\n%s", diff) + } + }) + + t.Run("errors", func(t *testing.T) { + if _, err := EncodeCPUFeatures("riscv64", []string{"pni"}); err == nil { + t.Error("EncodeCPUFeatures with unsupported arch succeeded") + } + if _, err := EncodeCPUFeatures("amd64", []string{"not_a_real_feature"}); err == nil { + t.Error("EncodeCPUFeatures with unknown feature succeeded") + } + if _, err := DecodeCPUFeatures("riscv64", "0"); err == nil { + t.Error("DecodeCPUFeatures with unsupported arch succeeded") + } + for _, bad := range []string{"", "0x1f", "1F", "07", "xyz", strings.Repeat("f", featureMaskWords*16+1)} { + if _, err := DecodeCPUFeatures("amd64", bad); err == nil { + t.Errorf("DecodeCPUFeatures(%q) succeeded, want a non-canonical value rejected", bad) + } + } + // Bit 16 is CPUID.1:ECX bit 16, which the hardware reserves, so it + // sits inside the table's range with no name; and one past the + // highest named bit has none either. Decoding must name the bit. + amd64 := cpuFeatureTables["amd64"] + lastBit := amd64.bits[len(amd64.bits)-1] + for _, bit := range []int{16, lastBit + 1} { + if _, named := amd64.names[bit]; named { + t.Fatalf("test assumes amd64 bit %d has no name, but it is %q", bit, amd64.names[bit]) + } + var m featureMask + m.set(0) + m.set(bit) + _, err := DecodeCPUFeatures("amd64", m.String()) + if err == nil || !strings.Contains(err.Error(), strconv.Itoa(bit)) { + t.Errorf("DecodeCPUFeatures with unnamed bit %d set = %v, want an error naming the bit", bit, err) + } + } + }) + + t.Run("canonical_hex", func(t *testing.T) { + // Bits 0, 1, 2 and 64: one word boundary crossed, low word zero-padded. + var m featureMask + for _, bit := range []int{0, 1, 2, 64} { + m.set(bit) + } + if got, want := m.String(), "1"+strings.Repeat("0", 15)+"7"; got != want { + t.Errorf("featureMask.String() = %q, want %q", got, want) + } + back, ok := parseFeatureMask(m.String()) + if !ok || back != m { + t.Errorf("parseFeatureMask(%q) = %v, %v; want the original mask", m.String(), back, ok) + } + }) +} + +// TestCPUFeatureTables checks structural invariants of each architecture's +// feature table and pins anchor bits to the values of gVisor's Feature +// constants (32*block + CPUID bit on amd64). A moved anchor means +// cpufeatures_table.go no longer follows the hardware's numbering and stored +// snapshots would decode differently than they were encoded. +func TestCPUFeatureTables(t *testing.T) { + for _, tc := range []struct { + arch, name string + bit int + }{ + {"amd64", "pni", 0}, + {"amd64", "pcid", 17}, // after the reserved bit 16 + {"amd64", "osxsave", 27}, // listed after bit 31 in gVisor's map + {"amd64", "fpu", 32}, // block 1, bit 0 + {"amd64", "avx2", 2*32 + 5}, // block 2 + {"amd64", "syscall", 6*32 + 11}, // sparse block 6 + {"amd64", "avx512_bf16", 8*32 + 5}, + {"arm64", "fp", 0}, + {"arm64", "asimdfhm", 23}, + } { + if got, ok := cpuFeatureTables[tc.arch].index[tc.name]; !ok || got != tc.bit { + t.Errorf("%s %q at bit %d (present=%v), want %d", tc.arch, tc.name, got, ok, tc.bit) + } + } + for arch, table := range cpuFeatureTables { + t.Run(arch, func(t *testing.T) { + if len(table.bits) == 0 { + t.Fatalf("%s table is empty", arch) + } + if !slices.IsSorted(table.bits) || len(table.bits) != len(table.names) { + t.Errorf("%s table bits %v are not the sorted keys of its %d names", arch, table.bits, len(table.names)) + } + var defined featureMask + seen := make(map[string]int, len(table.names)) + for bit, name := range table.names { + defined.set(bit) // newFeatureTable panics at init on a bit outside the mask + if other, dup := seen[name]; dup { + t.Errorf("%s table lists %q at both %d and %d", arch, name, other, bit) + } + seen[name] = bit + if name != strings.TrimSpace(name) || name == "" { + t.Errorf("%s table bit %d is %q, want a trimmed non-empty name", arch, bit, name) + } + if got := table.index[name]; got != bit { + t.Errorf("%s index[%q] = %d, want %d", arch, name, got, bit) + } + } + if defined != table.defined { + t.Errorf("%s table's defined mask %v does not match its names", arch, table.defined) + } + for name, bit := range table.index { + if _, ok := table.names[bit]; !ok { + t.Errorf("%s index maps %q to bit %d, which has no name", arch, name, bit) + } + } + }) + } +} + +// A table entry the mask cannot hold is a panic at package init rather than a +// dropped name. +func TestNewFeatureTableRejectsBitOutsideMask(t *testing.T) { + defer func() { + if recover() == nil { + t.Error("newFeatureTable with a bit outside the mask did not panic") + } + }() + newFeatureTable(map[int]string{featureMaskWords * 64: "beyond"}) +} + +func TestParseCPUInfo(t *testing.T) { + tests := []struct { + name string + arch string + cpuinfo string + wantVendor string + wantModel string + wantErr bool + }{ + { + name: "amd64 Intel", + arch: "amd64", + cpuinfo: `processor : 0 +vendor_id : GenuineIntel +cpu family : 6 +model : 143 +model name : Intel(R) Xeon(R) Platinum 8481C +`, + wantVendor: "GenuineIntel", + wantModel: "6:143", + }, + { + name: "arm64 Neoverse", + arch: "arm64", + cpuinfo: `processor : 0 +BogoMIPS : 2000.00 +Features : fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics +CPU implementer : 0x41 +CPU architecture: 8 +CPU variant : 0x0 +CPU part : 0xd40 +CPU revision : 1 +`, + wantVendor: "0x41", + wantModel: "0x0:0xd40:1", + }, + { + name: "missing fields", + arch: "amd64", + cpuinfo: "vendor_id : GenuineIntel\n", + wantErr: true, + }, + { + name: "unsupported arch", + arch: "riscv64", + cpuinfo: "vendor_id : test\n", + wantErr: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + vendor, model, err := parseCPUInfo(tc.arch, strings.NewReader(tc.cpuinfo)) + if (err != nil) != tc.wantErr { + t.Fatalf("parseCPUInfo() error = %v, wantErr %v", err, tc.wantErr) + } + if vendor != tc.wantVendor || model != tc.wantModel { + t.Errorf("parseCPUInfo() = (%q, %q), want (%q, %q)", vendor, model, tc.wantVendor, tc.wantModel) + } + }) + } +} + +// FuzzCPUFeatures checks, from real `runsc cpu-features` output and whatever +// the fuzzer derives from it, that a feature list round-trips through +// EncodeCPUFeatures and DecodeCPUFeatures, that the encoding is canonical, +// and that subsetOf agrees with set inclusion on the names. +func FuzzCPUFeatures(f *testing.F) { + const ( + broadwellCSV = "fpu,vme,de,pse,tsc,msr,pae,mce,cx8,apic,sep,mtrr,pge,mca,cmov,pat,pse36,clflush,mmx,fxsr,sse,sse2,ss,ht,syscall,nx,pdpe1gb,rdtscp,lm,pni,pclmulqdq,ssse3,fma,cx16,pcid,sse4_1,sse4_2,x2apic,movbe,popcnt,aes,xsave,osxsave,avx,f16c,rdrand,hypervisor,lahf_lm,abm,3dnowprefetch,fsgsbase,tsc_adjust,bmi1,hle,avx2,smep,bmi2,erms,invpcid,rtm,rdseed,adx,smap,xsaveopt,md_clear,arch_capabilities" + sapphireRapidsCSV = "fpu,vme,de,pse,tsc,msr,pae,mce,cx8,apic,sep,mtrr,pge,mca,cmov,pat,pse36,clflush,mmx,fxsr,sse,sse2,ss,ht,syscall,nx,pdpe1gb,rdtscp,lm,pni,pclmulqdq,ssse3,fma,cx16,pcid,sse4_1,sse4_2,x2apic,movbe,popcnt,tsc_deadline_timer,aes,xsave,osxsave,avx,f16c,rdrand,hypervisor,lahf_lm,abm,3dnowprefetch,fsgsbase,tsc_adjust,bmi1,hle,avx2,smep,bmi2,erms,invpcid,rtm,avx512f,avx512dq,rdseed,adx,smap,clfushopt,clwb,avx512cd,sha_ni,avx512bw,avx512vl,xsaveopt,xsavec,xgetbv1,xsaves,avx512vbmi,umip,pku,ospke,waitpkg,avx512_vbmi2,gfni,vaes,vpclmulqdq,avx512_vnni,avx512_bitalg,avx512_vpopcntdq,la57,rdpid,cldemote,movdiri,movdir64b,fsrm,md_clear,serialize,tsxldtrk,amx_bf16,avx512_fp16,amx_tile,amx_int8,spec_ctrl,intel_stibp,flush_l1d,arch_capabilities,spec_ctrl_ssbd,avx_vnni,avx512_bf16" + amdMilanCSV = "fpu,vme,de,pse,tsc,msr,pae,mce,cx8,apic,sep,mtrr,pge,mca,cmov,pat,pse36,clflush,mmx,fxsr,sse,sse2,ht,syscall,nx,mmxext,fxsr_opt,pdpe1gb,rdtscp,lm,pni,pclmulqdq,ssse3,fma,cx16,pcid,sse4_1,sse4_2,x2apic,movbe,popcnt,aes,xsave,osxsave,avx,f16c,rdrand,hypervisor,lahf_lm,cmp_legacy,svm,cr8_legacy,abm,sse4a,misalignsse,3dnowprefetch,osvw,topoext,perfctr_core,fsgsbase,tsc_adjust,bmi1,avx2,smep,bmi2,erms,invpcid,rdseed,adx,smap,clfushopt,clwb,sha_ni,xsaveopt,xsavec,xgetbv1,xsaves,umip,pku,ospke,vaes,vpclmulqdq,rdpid,fsrm" + neoverseN1CSV = "fp,asimd,evtstrm,aes,pmull,sha1,sha2,crc32,atomics,fphp,asimdhp,cpuid,asimdrdm,lrcpc,dcpop,asimddp" + neoverseV2CSV = "fp,asimd,evtstrm,aes,pmull,sha1,sha2,crc32,atomics,fphp,asimdhp,cpuid,asimdrdm,jscvt,fcma,lrcpc,dcpop,sha3,sm3,sm4,asimddp,sha512,sve,asimdfhm" + ) + // A seed the tables cannot encode would only exercise the early return + // below, so a table change that drops one of these names fails here + // rather than quietly weakening the corpus. + for _, seed := range []struct{ arch, csv string }{ + {"amd64", broadwellCSV}, {"amd64", sapphireRapidsCSV}, {"amd64", amdMilanCSV}, + {"arm64", neoverseN1CSV}, {"arm64", neoverseV2CSV}, + } { + if _, err := EncodeCPUFeatures(seed.arch, strings.Split(seed.csv, ",")); err != nil { + f.Fatalf("seed does not encode: %v", err) + } + } + f.Add("amd64", broadwellCSV, sapphireRapidsCSV) + f.Add("amd64", sapphireRapidsCSV, broadwellCSV) + f.Add("amd64", broadwellCSV, amdMilanCSV) + f.Add("arm64", neoverseN1CSV, neoverseV2CSV) + f.Add("arm64", neoverseV2CSV, neoverseN1CSV) + // Linux-synthesized flags runsc never prints; encoding rejects them. + f.Add("amd64", "fpu,sse2,constant_tsc,rep_good,nopl", broadwellCSV) + + f.Fuzz(func(t *testing.T, arch, snapCSV, workerCSV string) { + snapRaw, workerRaw := strings.Split(snapCSV, ","), strings.Split(workerCSV, ",") + snapMask, snapErr := EncodeCPUFeatures(arch, snapRaw) + workerMask, workerErr := EncodeCPUFeatures(arch, workerRaw) + if snapErr != nil || workerErr != nil { + return + } + decoded, err := DecodeCPUFeatures(arch, snapMask) + if err != nil { + t.Fatalf("DecodeCPUFeatures(%q, %q) failed: %v", arch, snapMask, err) + } + if reencoded, err := EncodeCPUFeatures(arch, decoded); err != nil || reencoded != snapMask { + t.Fatalf("round-trip EncodeCPUFeatures(%q, %v) = (%q, %v), want %q", arch, decoded, reencoded, err, snapMask) + } + workerDecoded, err := DecodeCPUFeatures(arch, workerMask) + if err != nil { + t.Fatalf("DecodeCPUFeatures(%q, %q) failed: %v", arch, workerMask, err) + } + wantSubset := true + for _, feat := range decoded { + if !slices.Contains(workerDecoded, feat) { + wantSubset = false + break + } + } + mSnap, okSnap := parseFeatureMask(snapMask) + mWorker, okWorker := parseFeatureMask(workerMask) + if !okSnap || !okWorker || mSnap.String() != snapMask || mWorker.String() != workerMask { + t.Fatalf("non-canonical mask produced: snap=%q worker=%q", snapMask, workerMask) + } + if got := mSnap.subsetOf(mWorker); got != wantSubset { + t.Fatalf("subsetOf(%q, %q) = %v, want %v", snapMask, workerMask, got, wantSubset) + } + }) +} diff --git a/internal/hardware/hardware.go b/internal/hardware/hardware.go index 827f59dd62..4511b90707 100644 --- a/internal/hardware/hardware.go +++ b/internal/hardware/hardware.go @@ -19,8 +19,8 @@ package hardware import ( "runtime" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) const ( @@ -35,15 +35,26 @@ const ( // digest, computed by atelet. AttrSandboxRuntimeHash = "sandbox_runtime_hash" - // TODO: Add AttrCPUFeatures ("cpu_features") and other compatibility - // attributes as snapshot compatibility expands. + // AttrCPUFeatures is the set of CPU features `runsc cpu-features` reports + // for a gVisor runtime, encoded by EncodeCPUFeatures as a hex bitmask over + // the architecture's feature table. A snapshot's set must be a subset of + // the worker's. + AttrCPUFeatures = "cpu_features" + + // AttrCPUVendor identifies the host CPU vendor (e.g. "GenuineIntel", + // "AuthenticAMD", or arm64 implementer "0x41") reported by microVM + // runtimes. + AttrCPUVendor = "cpu_vendor" + + // AttrCPUModel identifies the host CPU family/model (e.g. "6:143" on amd64 + // or "0x0:0xd40:1" on arm64) reported by microVM runtimes. + AttrCPUModel = "cpu_model" ) -// ProbeHost inspects the current host and returns its v1 VersionedSandboxCompat -// in atelet wire format. -// -// TODO: Probe and populate cpu_features and other host hardware attributes -// (e.g. via CPUID on amd64 and MIDR_EL1 on arm64). +// ProbeHost returns a minimal v1 VersionedSandboxCompat with the current +// host's architecture in atelet wire format, for synthetic workers in tests +// and benchmarks. Production workers report the attributes atelet writes to +// sandbox-runtimes.json instead. func ProbeHost() *ateletpb.VersionedSandboxCompat { return &ateletpb.VersionedSandboxCompat{ SchemaVersion: SchemaVersionV1, @@ -56,47 +67,92 @@ func ProbeHost() *ateletpb.VersionedSandboxCompat { } } -// Matches reports whether a worker's SandboxRuntime can restore a snapshot -// stamped with snap. A nil snapshot SandboxRuntime imposes no constraint. -// Otherwise the sandbox classes must be equal and MatchesCompat must hold. -func Matches(worker, snap *ateapipb.SandboxRuntime) bool { - if snap == nil { +// Requirement is what a worker must run to restore a snapshot: the +// SandboxRuntime the snapshot was stamped with, prepared for matching against +// many workers. The scheduler asks it once per candidate, so the snapshot's +// AttrCPUFeatures is parsed here, once, rather than on every comparison. The +// zero Requirement, like NewRequirement(nil), requires nothing. +type Requirement struct { + snap *ateapipb.SandboxRuntime + compat *ateapipb.VersionedSandboxCompat // snap's version + // cpuFeatures is compat's AttrCPUFeatures parsed, when it has one. + // badCPUFeatures records one that is not a canonical mask, which no + // worker satisfies. + cpuFeatures featureMask + badCPUFeatures bool +} + +// NewRequirement prepares snap for matching. A nil snap requires nothing. +func NewRequirement(snap *ateapipb.SandboxRuntime) Requirement { + r := compatRequirement(snap.GetVersion()) + r.snap = snap + return r +} + +func compatRequirement(compat *ateapipb.VersionedSandboxCompat) Requirement { + r := Requirement{compat: compat} + for _, a := range compat.GetAttributes() { + if a.GetKey() == AttrCPUFeatures { + var ok bool + r.cpuFeatures, ok = parseFeatureMask(a.GetValue()) + r.badCPUFeatures = !ok + break + } + } + return r +} + +// Runtime returns the SandboxRuntime the requirement was prepared from, or +// nil when it requires nothing. +func (r Requirement) Runtime() *ateapipb.SandboxRuntime { + return r.snap +} + +// Matches reports whether a worker's SandboxRuntime can restore the +// snapshot. A requirement of nothing is met by any worker. Otherwise the +// sandbox classes must be equal and the compat must match. +func (r Requirement) Matches(worker *ateapipb.SandboxRuntime) bool { + if r.snap == nil { return true } - if worker == nil || worker.GetSandboxClass() != snap.GetSandboxClass() { + if worker == nil || worker.GetSandboxClass() != r.snap.GetSandboxClass() { return false } - return MatchesCompat(worker.GetVersion(), snap.GetVersion()) + return r.matchesCompat(worker.GetVersion()) } -// MatchesCompat reports whether worker can restore a snapshot stamped with -// snap: the same non-empty schema version, and every attribute on snap -// present on worker with an equal value. Keys only the worker has are -// ignored, so a worker can start reporting a new key under the same schema -// version without invalidating the snapshots taken before it; removing or -// recomputing a key is a new schema version, under which older snapshots no -// longer match. -func MatchesCompat(worker, snap *ateapipb.VersionedSandboxCompat) bool { - if worker == nil || snap == nil { +// matchesCompat reports whether worker can restore the snapshot: the same +// non-empty schema version, and every attribute on the snapshot satisfied by +// worker. AttrCPUFeatures is satisfied when the snapshot's feature bitmask is +// a subset of the worker's; every other attribute must be present on worker +// with an equal value. Keys only the worker has are ignored, so a worker can +// start reporting a new key under the same schema version without +// invalidating the snapshots taken before it; removing or recomputing a key +// is a new schema version, under which older snapshots no longer match. +func (r Requirement) matchesCompat(worker *ateapipb.VersionedSandboxCompat) bool { + if worker == nil || r.compat == nil { return false } - if worker.GetSchemaVersion() == "" || worker.GetSchemaVersion() != snap.GetSchemaVersion() { + if worker.GetSchemaVersion() == "" || worker.GetSchemaVersion() != r.compat.GetSchemaVersion() { + return false + } + if r.badCPUFeatures { return false } wAttrs := worker.GetAttributes() - sAttrs := snap.GetAttributes() // API validation bounds attributes to at most 32 entries with unique keys, // so the nested scan is small and avoids map allocation on the hot path. - for _, sa := range sAttrs { + for _, sa := range r.compat.GetAttributes() { found := false for _, wa := range wAttrs { - if wa.GetKey() == sa.GetKey() { - if wa.GetValue() != sa.GetValue() { - return false - } - found = true - break + if wa.GetKey() != sa.GetKey() { + continue + } + if !r.attributeSatisfied(sa.GetKey(), wa.GetValue(), sa.GetValue()) { + return false } + found = true + break } if !found { return false @@ -105,42 +161,78 @@ func MatchesCompat(worker, snap *ateapipb.VersionedSandboxCompat) bool { return true } -// MatchingRuntime returns the SandboxRuntime from status that can restore a -// snapshot stamped with snap (checking DefaultRuntime first, then -// RestorableRuntimes in order), or nil if none matches. When snap is nil, -// DefaultRuntime is returned. -func MatchingRuntime(status *ateapipb.WorkerStatus, snap *ateapipb.SandboxRuntime) *ateapipb.SandboxRuntime { - if snap == nil { - return status.GetDefaultRuntime() +// attributeSatisfied reports whether a worker's value for key satisfies the +// snapshot's. AttrCPUFeatures is a set the worker's must include, compared +// against the mask parsed by NewRequirement; every other attribute must be +// equal. +func (r Requirement) attributeSatisfied(key, workerVal, snapVal string) bool { + if key == AttrCPUFeatures { + w, ok := parseFeatureMask(workerVal) + return ok && r.cpuFeatures.subsetOf(w) } - if def := status.GetDefaultRuntime(); Matches(def, snap) { + return workerVal == snapVal +} + +// MatchingRuntime returns the SandboxRuntime from status that can restore +// the snapshot (checking DefaultRuntime first, then RestorableRuntimes in +// order), or nil if none matches. A requirement of nothing returns +// DefaultRuntime. +func (r Requirement) MatchingRuntime(status *ateapipb.WorkerStatus) *ateapipb.SandboxRuntime { + if def := status.GetDefaultRuntime(); r.snap == nil || r.Matches(def) { return def } for _, rt := range status.GetRestorableRuntimes() { - if Matches(rt, snap) { + if r.Matches(rt) { return rt } } return nil } -// WorkerMatches reports whether a worker with status can restore a snapshot -// stamped with snap. A nil snapshot SandboxRuntime imposes no constraint. -func WorkerMatches(status *ateapipb.WorkerStatus, snap *ateapipb.SandboxRuntime) bool { - if snap == nil { - return true - } - return MatchingRuntime(status, snap) != nil +// WorkerMatches reports whether a worker with status can restore the +// snapshot. A requirement of nothing is met by any worker. +func (r Requirement) WorkerMatches(status *ateapipb.WorkerStatus) bool { + return r.snap == nil || r.MatchingRuntime(status) != nil } // RuntimeFor returns the runtime a worker with status runs an actor under -// whose snapshot is stamped with snap: the matching runtime when it can -// restore the snapshot, else the DefaultRuntime the guest boots fresh under. -// This is what an assignment records and what the actor's next checkpoint -// is stamped with. -func RuntimeFor(status *ateapipb.WorkerStatus, snap *ateapipb.SandboxRuntime) *ateapipb.SandboxRuntime { - if rt := MatchingRuntime(status, snap); rt != nil { +// whose snapshot is stamped with the required runtime: the matching runtime +// when it can restore the snapshot, else the DefaultRuntime the guest boots +// fresh under. This is what an assignment records and what the actor's next +// checkpoint is stamped with. +func (r Requirement) RuntimeFor(status *ateapipb.WorkerStatus) *ateapipb.SandboxRuntime { + if rt := r.MatchingRuntime(status); rt != nil { return rt } return status.GetDefaultRuntime() } + +// Matches reports whether a worker's SandboxRuntime can restore a snapshot +// stamped with snap, for one comparison; a caller holding one snap against +// many workers should prepare a Requirement once. A nil snap imposes no +// constraint. +func Matches(worker, snap *ateapipb.SandboxRuntime) bool { + return NewRequirement(snap).Matches(worker) +} + +// MatchesCompat reports whether worker can restore a snapshot stamped with +// snap, for one comparison. Unlike Matches, a nil snap is a stamp missing its +// identity and matches nothing. +func MatchesCompat(worker, snap *ateapipb.VersionedSandboxCompat) bool { + return compatRequirement(snap).matchesCompat(worker) +} + +// MatchingRuntime is NewRequirement(snap).MatchingRuntime(status). +func MatchingRuntime(status *ateapipb.WorkerStatus, snap *ateapipb.SandboxRuntime) *ateapipb.SandboxRuntime { + return NewRequirement(snap).MatchingRuntime(status) +} + +// WorkerMatches is NewRequirement(snap).WorkerMatches(status). +func WorkerMatches(status *ateapipb.WorkerStatus, snap *ateapipb.SandboxRuntime) bool { + return NewRequirement(snap).WorkerMatches(status) +} + +// RuntimeFor is NewRequirement(snap).RuntimeFor(status). +func RuntimeFor(status *ateapipb.WorkerStatus, snap *ateapipb.SandboxRuntime) *ateapipb.SandboxRuntime { + return NewRequirement(snap).RuntimeFor(status) +} diff --git a/internal/hardware/hardware_test.go b/internal/hardware/hardware_test.go index 652313c7ef..ee50f280dc 100644 --- a/internal/hardware/hardware_test.go +++ b/internal/hardware/hardware_test.go @@ -18,7 +18,7 @@ import ( "runtime" "testing" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) func TestProbeHost(t *testing.T) { @@ -44,28 +44,28 @@ func TestMatches(t *testing.T) { } amd64GVisor := runtimeWith("gvisor", "v1", &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "amd64"}, - &ateapipb.AttributeEntry{Key: "cpu_features", Value: "GenuineIntel"}, + &ateapipb.AttributeEntry{Key: AttrCPUFeatures, Value: "7"}, ) amd64GVisorReordered := runtimeWith("gvisor", "v1", - &ateapipb.AttributeEntry{Key: "cpu_features", Value: "GenuineIntel"}, + &ateapipb.AttributeEntry{Key: AttrCPUFeatures, Value: "7"}, &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "amd64"}, ) arm64GVisor := runtimeWith("gvisor", "v1", &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "arm64"}, - &ateapipb.AttributeEntry{Key: "cpu_features", Value: "GenuineIntel"}, + &ateapipb.AttributeEntry{Key: AttrCPUFeatures, Value: "7"}, ) amd64MicroVM := runtimeWith("microvm", "v1", &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "amd64"}, - &ateapipb.AttributeEntry{Key: "cpu_features", Value: "GenuineIntel"}, + &ateapipb.AttributeEntry{Key: AttrCPUFeatures, Value: "7"}, ) amd64GVisorV2 := runtimeWith("gvisor", "v2", &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "amd64"}, - &ateapipb.AttributeEntry{Key: "cpu_features", Value: "GenuineIntel"}, + &ateapipb.AttributeEntry{Key: AttrCPUFeatures, Value: "7"}, ) amd64GVisorExtraAttr := runtimeWith("gvisor", "v1", &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "amd64"}, - &ateapipb.AttributeEntry{Key: "cpu_features", Value: "GenuineIntel"}, - &ateapipb.AttributeEntry{Key: "gvisor_asset_hash", Value: "abc"}, + &ateapipb.AttributeEntry{Key: AttrCPUFeatures, Value: "7"}, + &ateapipb.AttributeEntry{Key: AttrSandboxRuntimeHash, Value: "sha256:abc"}, ) tests := []struct { @@ -100,8 +100,24 @@ func TestMatchesCompat(t *testing.T) { } amd64 := &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "amd64"} arm64 := &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "arm64"} - cpu := &ateapipb.AttributeEntry{Key: "cpu_features", Value: "GenuineIntel"} - hash := &ateapipb.AttributeEntry{Key: "gvisor_asset_hash", Value: "abc"} + hash := &ateapipb.AttributeEntry{Key: AttrSandboxRuntimeHash, Value: "sha256:abc"} + cpuBits := func(val string) *ateapipb.AttributeEntry { + return &ateapipb.AttributeEntry{Key: AttrCPUFeatures, Value: val} + } + + // Build >64-bit masks so the subset check spans mask words. + highSnap, err := EncodeCPUFeatures("amd64", []string{"pni", "avx2", "avx512_bf16"}) + if err != nil { + t.Fatal(err) + } + highWorkerSuperset, err := EncodeCPUFeatures("amd64", []string{"pni", "sse2", "avx2", "avx_vnni", "avx512_bf16"}) + if err != nil { + t.Fatal(err) + } + highWorkerMissingOne, err := EncodeCPUFeatures("amd64", []string{"pni", "sse2", "avx2", "avx_vnni"}) + if err != nil { + t.Fatal(err) + } tests := []struct { name string @@ -109,10 +125,19 @@ func TestMatchesCompat(t *testing.T) { want bool }{ {"equal", compat("v1", amd64), compat("v1", amd64), true}, - {"key only the worker has is ignored", compat("v1", amd64, cpu), compat("v1", amd64), true}, - {"key only the snapshot has fails", compat("v1", amd64), compat("v1", amd64, cpu), false}, - {"key only the snapshot has fails even at equal count", compat("v1", amd64, cpu), compat("v1", amd64, hash), false}, + {"key only the worker has is ignored", compat("v1", amd64, cpuBits("7")), compat("v1", amd64), true}, + {"key only the snapshot has fails", compat("v1", amd64), compat("v1", amd64, cpuBits("7")), false}, + {"key only the snapshot has fails even at equal count", compat("v1", amd64, cpuBits("7")), compat("v1", amd64, hash), false}, {"value differs", compat("v1", arm64), compat("v1", amd64), false}, + {"cpu_features exact match", compat("v1", amd64, cpuBits(highSnap)), compat("v1", amd64, cpuBits(highSnap)), true}, + {"cpu_features worker is superset of snapshot", compat("v1", amd64, cpuBits(highWorkerSuperset)), compat("v1", amd64, cpuBits(highSnap)), true}, + {"cpu_features worker missing a snapshot feature bit", compat("v1", amd64, cpuBits(highWorkerMissingOne)), compat("v1", amd64, cpuBits(highSnap)), false}, + {"cpu_features empty snapshot set matches any worker", compat("v1", amd64, cpuBits("7")), compat("v1", amd64, cpuBits("0")), true}, + {"cpu_features empty worker set matches nothing", compat("v1", amd64, cpuBits("0")), compat("v1", amd64, cpuBits("7")), false}, + {"cpu_features invalid worker bitmask fails", compat("v1", amd64, cpuBits("not-a-number")), compat("v1", amd64, cpuBits("7")), false}, + {"cpu_features invalid snapshot bitmask fails", compat("v1", amd64, cpuBits("7")), compat("v1", amd64, cpuBits("-1")), false}, + {"cpu_features non-canonical bitmask fails", compat("v1", amd64, cpuBits("07")), compat("v1", amd64, cpuBits("7")), false}, + {"cpu_features uppercase hex fails", compat("v1", amd64, cpuBits("1F")), compat("v1", amd64, cpuBits("7")), false}, {"schema version differs", compat("v2", amd64), compat("v1", amd64), false}, {"both schema versions empty", compat("", amd64), compat("", amd64), false}, {"no attributes on either", compat("v1"), compat("v1"), true}, @@ -131,6 +156,60 @@ func TestMatchesCompat(t *testing.T) { } } +// A Requirement is prepared once and asked of many workers, so what it keeps +// of the snapshot must hold across comparisons, and the zero value must +// require nothing. +func TestRequirement(t *testing.T) { + rt := func(cpu string) *ateapipb.SandboxRuntime { + return &ateapipb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: SchemaVersionV1, + Attributes: []*ateapipb.AttributeEntry{ + {Key: AttrArchitecture, Value: "amd64"}, + {Key: AttrCPUFeatures, Value: cpu}, + }, + }, + } + } + snap := rt("7") + req := NewRequirement(snap) + if req.Runtime() != snap { + t.Errorf("Runtime() = %v, want the prepared runtime", req.Runtime()) + } + for _, tc := range []struct { + name string + worker *ateapipb.SandboxRuntime + want bool + }{ + {"worker superset", rt("f"), true}, + {"worker missing a bit", rt("3"), false}, + {"worker mask invalid", rt("zz"), false}, + {"worker superset after misses", rt("7"), true}, + } { + if got := req.Matches(tc.worker); got != tc.want { + t.Errorf("%s: Matches() = %v, want %v", tc.name, got, tc.want) + } + } + status := &ateapipb.WorkerStatus{DefaultRuntime: rt("3"), RestorableRuntimes: []*ateapipb.SandboxRuntime{rt("f")}} + if got := req.MatchingRuntime(status); got != status.RestorableRuntimes[0] { + t.Errorf("MatchingRuntime() = %v, want the restorable superset", got) + } + + // A snapshot mask that does not parse is remembered as matching nothing. + if NewRequirement(rt("zz")).Matches(rt("f")) { + t.Error("Matches() with an invalid snapshot mask = true, want false") + } + + var none Requirement + if none.Runtime() != nil || !none.Matches(nil) || !none.WorkerMatches(nil) { + t.Error("the zero Requirement must require nothing") + } + if got := none.RuntimeFor(status); got != status.DefaultRuntime { + t.Errorf("zero Requirement RuntimeFor() = %v, want DefaultRuntime", got) + } +} + func TestMatchingRuntimeAndWorkerMatches(t *testing.T) { rt := func(class, arch, hash string) *ateapipb.SandboxRuntime { attrs := []*ateapipb.AttributeEntry{{Key: AttrArchitecture, Value: arch}} diff --git a/internal/imagecache/README.md b/internal/imagecache/README.md index 4d9e5cc8db..e366534174 100644 --- a/internal/imagecache/README.md +++ b/internal/imagecache/README.md @@ -99,10 +99,12 @@ layer diffIDs in order — layers shared by N images exist once. confinement (path traversal and symlink/hardlink escapes are refused), "later entry wins" within a layer, read-only-dir handling that works without `CAP_DAC_OVERRIDE`, and creation of parent directories that the - layer tar omits (they may exist only in lower layers). Whiteout entries - (`.wh.*`) are **not** written into the tree — overlayfs whiteouts are - char devices atelet cannot create — they are recorded in - `whiteouts.json` for the consumer to materialize. + layer tar omits (they may exist only in lower layers). Character and + block device nodes are skipped (atelet runs without `CAP_MKNOD` and the + runtime mounts `/dev`). Whiteout entries (`.wh.*`) are **not** written + into the tree — overlayfs whiteouts are char devices atelet cannot + create — they are recorded in `whiteouts.json` for the consumer to + materialize. 5. **Record**: the image config + diffID list is written under the requested digest (and the per-platform child digest for multi-arch refs). diff --git a/internal/imagecache/bundle_linux_test.go b/internal/imagecache/bundle_linux_test.go index 3d35206047..82901031c8 100644 --- a/internal/imagecache/bundle_linux_test.go +++ b/internal/imagecache/bundle_linux_test.go @@ -22,11 +22,12 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" "golang.org/x/sys/unix" - "github.com/agent-substrate/substrate/internal/roottest" + "go.ate.dev/substrate/internal/roottest" ) // writeLayer builds a layer dir (fs/ tree + whiteouts.json) as the store's @@ -254,18 +255,27 @@ func TestSetupBundleRootfs_MountAndUnmount(t *testing.T) { // limit. Needs CAP_SYS_ADMIN. func TestSetupBundleRootfs_ManyLayers(t *testing.T) { roottest.Require(t, "mount/unmount") - // Digest-length dir names so each path matches production length (~114 - // bytes); 64 of them comfortably exceed the page that motivated this. + // Overlayfs caps a chain at 500 lower layers, and production-length + // paths (~114 bytes) cannot outgrow a 64 KiB page within that cap. So + // each digest-named layer dir is padded until its path fills the 255 + // bytes one fsconfig(2) string value carries, and the layer count + // follows the page size: 17 layers on 4 KiB pages, 257 on 64 KiB pages. pool := filepath.Join(t.TempDir(), "sha256") - const n = 64 + const lowerLen = 255 + pad := lowerLen - len(filepath.Join(pool, strings.Repeat("0", 64), layerFSDirName)) + if pad < 0 { + t.Fatalf("temp dir %q is too long to pad a lowerdir path to %d bytes", pool, lowerLen) + } + pageSize := os.Getpagesize() + n := pageSize/(lowerLen+1) + 1 layers := make([]string, n) joined := 0 for i := range layers { - layers[i] = filepath.Join(pool, fmt.Sprintf("%064d", i)) + layers[i] = filepath.Join(pool, fmt.Sprintf("%064d", i)+strings.Repeat("p", pad)) writeLayer(t, layers[i], map[string]string{fmt.Sprintf("from-layer-%02d.txt", i): "x"}, nil) joined += len(layers[i]) + len("/fs") + 1 } - if pageSize := os.Getpagesize(); joined <= pageSize { + if joined <= pageSize { t.Fatalf("test layers join to %d bytes, not exceeding the %d-byte page this test guards against", joined, pageSize) } diff --git a/internal/imagecache/imagecache.go b/internal/imagecache/imagecache.go index d6601ef2c0..edcaaff37b 100644 --- a/internal/imagecache/imagecache.go +++ b/internal/imagecache/imagecache.go @@ -70,7 +70,7 @@ import ( "golang.org/x/sync/errgroup" "golang.org/x/sync/singleflight" - "github.com/agent-substrate/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/ateattr" ) const ( diff --git a/internal/imagecache/metrics.go b/internal/imagecache/metrics.go index 6e014b4f77..4b51f85551 100644 --- a/internal/imagecache/metrics.go +++ b/internal/imagecache/metrics.go @@ -24,7 +24,7 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" - "github.com/agent-substrate/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/ateattr" ) const requestsMetric = "ate.imagecache.requests" diff --git a/internal/imagecache/metrics_test.go b/internal/imagecache/metrics_test.go index 2280d5d586..45156586ce 100644 --- a/internal/imagecache/metrics_test.go +++ b/internal/imagecache/metrics_test.go @@ -29,7 +29,7 @@ import ( sdkmetric "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/metricdata" - "github.com/agent-substrate/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/ateattr" ) // newMeteredStore opens a store against a local ManualReader-backed provider, diff --git a/internal/imagecache/mountinfo.go b/internal/imagecache/mountinfo.go index 0f874fa5af..dca27a73f8 100644 --- a/internal/imagecache/mountinfo.go +++ b/internal/imagecache/mountinfo.go @@ -20,28 +20,45 @@ package imagecache import ( "bufio" + "bytes" "fmt" "io" + "math" "path/filepath" "strings" ) +// mountinfo octal-escapes space, tab, newline and backslash in paths (\040, +// \011, \012, \134); these decode and encode that form. +var ( + mountPathUnescaper = strings.NewReplacer(`\040`, " ", `\011`, "\t", `\012`, "\n", `\134`, `\`) + mountPathEscaper = strings.NewReplacer(" ", `\040`, "\t", `\011`, "\n", `\012`, `\`, `\134`) +) + // mountPointsIn returns the mount points at or below dir found in r, which // must be in /proc/[pid]/mountinfo format: // // ID parentID major:minor root MOUNTPOINT options... +// +// mountinfo can run to several MB on a dense worker and is read on every +// teardown, so the loop allocates only for matching lines. func mountPointsIn(r io.Reader, dir string) ([]string, error) { - dir = filepath.Clean(dir) + // Compare in mountinfo's escaped form, so only matches are decoded. The + // escaping is per character and leaves '/' alone, so a prefix match on + // the escaped path is a prefix match on the decoded one. + esc := []byte(mountPathEscaper.Replace(filepath.Clean(dir))) var points []string scanner := bufio.NewScanner(r) + // No line-length cap: an overlay's record lists every lowerdir path, and + // a long chain outgrows the 64 KiB default token size. + scanner.Buffer(nil, math.MaxInt) for scanner.Scan() { - fields := strings.Fields(scanner.Text()) - if len(fields) < 5 { + mp, ok := mountinfoField(scanner.Bytes(), 4) + if !ok || !bytes.HasPrefix(mp, esc) { continue } - mp := unescapeMountPath(fields[4]) - if mp == dir || strings.HasPrefix(mp, dir+"/") { - points = append(points, mp) + if rest := mp[len(esc):]; len(rest) == 0 || rest[0] == '/' { + points = append(points, unescapeMountPath(string(mp))) } } if err := scanner.Err(); err != nil { @@ -50,12 +67,27 @@ func mountPointsIn(r io.Reader, dir string) ([]string, error) { return points, nil } +// mountinfoField returns the nth (0-based) field of line. The kernel +// separates fields with a single space. +func mountinfoField(line []byte, n int) ([]byte, bool) { + for ; n > 0; n-- { + i := bytes.IndexByte(line, ' ') + if i < 0 { + return nil, false + } + line = line[i+1:] + } + if i := bytes.IndexByte(line, ' '); i >= 0 { + line = line[:i] + } + return line, len(line) > 0 +} + // unescapeMountPath decodes the octal escapes mountinfo uses for space, tab, // newline, and backslash. func unescapeMountPath(s string) string { if !strings.Contains(s, `\`) { return s } - r := strings.NewReplacer(`\040`, " ", `\011`, "\t", `\012`, "\n", `\134`, `\`) - return r.Replace(s) + return mountPathUnescaper.Replace(s) } diff --git a/internal/imagecache/mountinfo_test.go b/internal/imagecache/mountinfo_test.go index cc7dfa492c..42f2dd6be5 100644 --- a/internal/imagecache/mountinfo_test.go +++ b/internal/imagecache/mountinfo_test.go @@ -15,6 +15,7 @@ package imagecache import ( + "bufio" "slices" "strings" "testing" @@ -58,6 +59,50 @@ func TestMountPointsIn_ExactDirMatch(t *testing.T) { } } +// dir is compared in mountinfo's escaped form; a dir with a space must match +// its own mounts and not a longer sibling. +func TestMountPointsIn_EscapedDir(t *testing.T) { + mountinfo := strings.Join([]string{ + `40 21 8:1 / /mnt/odd\040name rw - ext4 /dev/sda1 rw`, + `41 21 8:1 / /mnt/odd\040name/rootfs rw - overlay overlay rw`, + `42 21 8:1 / /mnt/odd\040name2/rootfs rw - overlay overlay rw`, + ``, + }, "\n") + + got, err := mountPointsIn(strings.NewReader(mountinfo), "/mnt/odd name") + if err != nil { + t.Fatalf("mountPointsIn: %v", err) + } + want := []string{"/mnt/odd name", "/mnt/odd name/rootfs"} + if !slices.Equal(got, want) { + t.Errorf("mount points = %v, want %v", got, want) + } +} + +// An overlay's record lists every lowerdir path, so one line can be longer +// than bufio.Scanner's default token size; the lines after it must still be +// read. +func TestMountPointsIn_LongLine(t *testing.T) { + long := `40 21 8:1 / /mnt/target/rootfs rw - overlay overlay rw,lowerdir=` + strings.Repeat("/l", 40_000) + if len(long) <= bufio.MaxScanTokenSize { + t.Fatalf("test line is %d bytes, not exceeding bufio's %d-byte default", len(long), bufio.MaxScanTokenSize) + } + mountinfo := strings.Join([]string{ + long, + `41 21 8:1 / /mnt/target/volumes/agent rw - overlay overlay rw`, + ``, + }, "\n") + + got, err := mountPointsIn(strings.NewReader(mountinfo), "/mnt/target") + if err != nil { + t.Fatalf("mountPointsIn: %v", err) + } + want := []string{"/mnt/target/rootfs", "/mnt/target/volumes/agent"} + if !slices.Equal(got, want) { + t.Errorf("mount points = %v, want %v", got, want) + } +} + func TestUnescapeMountPath(t *testing.T) { tests := []struct{ in, want string }{ {`/plain/path`, `/plain/path`}, diff --git a/internal/imagecache/unpack.go b/internal/imagecache/unpack.go index 9b95ccd462..7d9b94ecd1 100644 --- a/internal/imagecache/unpack.go +++ b/internal/imagecache/unpack.go @@ -127,6 +127,27 @@ func unpackLayer(ctx context.Context, tarData io.Reader, root *os.Root) (*whiteo } } + // replaceExisting removes whatever is at name ("later entry wins" within a + // layer) and drops any deferred dir mode/mtime restore for name and its + // subtree so the restore pass does not chmod a removed directory. + replaceExisting := func(name, before string) error { + if _, err := root.Lstat(name); err == nil { + if err := root.RemoveAll(name); err != nil { + return fmt.Errorf("while replacing existing path at %q before %s: %w", name, before, err) + } + prefix := name + "/" + for d := range dirModes { + if d == name || strings.HasPrefix(d, prefix) { + delete(dirModes, d) + delete(dirModTimes, d) + } + } + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("while checking existing path at %q before %s: %w", name, before, err) + } + return nil + } + tarReader := tar.NewReader(tarData) for { hdr, err := tarReader.Next() @@ -160,6 +181,16 @@ func unpackLayer(ctx context.Context, tarData io.Reader, root *os.Root) (*whiteo continue } + // Skip device nodes like rootless containerd does: an image must never + // grant device access, and the runtime provides /dev. + if hdr.Typeflag == tar.TypeChar || hdr.Typeflag == tar.TypeBlock { + if err := replaceExisting(name, "device node"); err != nil { + return nil, err + } + slog.DebugContext(ctx, "Skipping device node in image layer", slog.String("name", name)) + continue + } + mode := hdr.FileInfo().Mode().Perm() // A layer tar routinely omits entries for parent directories that @@ -181,12 +212,8 @@ func unpackLayer(ctx context.Context, tarData io.Reader, root *os.Root) (*whiteo // 1. If it's a symlink, we don't write through it (security vulnerability / incorrectness). // 2. If it's a hardlink, we unlink it instead of truncating the shared inode. // 3. If it's a directory, we recursively remove it so we can write the file. - if _, err := root.Lstat(name); err == nil { - if err := root.RemoveAll(name); err != nil { - return nil, fmt.Errorf("while replacing existing path at %q before regular file: %w", name, err) - } - } else if !errors.Is(err, os.ErrNotExist) { - return nil, fmt.Errorf("while checking existing path at %q before regular file: %w", name, err) + if err := replaceExisting(name, "regular file"); err != nil { + return nil, err } // Stream directly from tarReader to target file to avoid buffering in memory. @@ -236,15 +263,13 @@ func unpackLayer(ctx context.Context, tarData io.Reader, root *os.Root) (*whiteo continue } } - // Root.RemoveAll removes the symlink entry itself; it does NOT - // traverse and remove the directory the symlink points to. - // That's the desired semantic here — replace this path's - // entry without touching whatever the prior symlink targeted. - if err := root.RemoveAll(name); err != nil { - return nil, fmt.Errorf("while replacing existing path at %q before symlink: %w", name, err) - } - } else if !errors.Is(err, os.ErrNotExist) { - return nil, fmt.Errorf("while checking existing path at %q before symlink: %w", name, err) + } + // Root.RemoveAll removes the symlink entry itself; it does NOT + // traverse and remove the directory the symlink points to. + // That's the desired semantic here — replace this path's + // entry without touching whatever the prior symlink targeted. + if err := replaceExisting(name, "symlink"); err != nil { + return nil, err } if err := root.Symlink(hdr.Linkname, name); err != nil { return nil, fmt.Errorf("while creating symlink src=%q target=%q: %w", name, hdr.Linkname, err) @@ -261,12 +286,8 @@ func unpackLayer(ctx context.Context, tarData io.Reader, root *os.Root) (*whiteo return nil, fmt.Errorf("invalid hardlink target for %q: empty", name) } // Same "later entry wins" handling as TypeSymlink: replace existing entry. - if _, err := root.Lstat(name); err == nil { - if err := root.RemoveAll(name); err != nil { - return nil, fmt.Errorf("while replacing existing path at %q before hardlink: %w", name, err) - } - } else if !errors.Is(err, os.ErrNotExist) { - return nil, fmt.Errorf("while checking existing path at %q before hardlink: %w", name, err) + if err := replaceExisting(name, "hardlink"); err != nil { + return nil, err } if err := root.Link(linkname, name); err != nil { return nil, fmt.Errorf("while creating hardlink src=%q target=%q: %w", name, linkname, err) @@ -276,8 +297,8 @@ func unpackLayer(ctx context.Context, tarData io.Reader, root *os.Root) (*whiteo default: tfStr := string([]byte{hdr.Typeflag}) - slog.ErrorContext(ctx, "Unhandled tar entry typeflag", slog.String("typeflag", tfStr), slog.Any("hdr", hdr)) - return nil, fmt.Errorf("unhandled tar entry typeflag %q", tfStr) + slog.ErrorContext(ctx, "Unhandled tar entry typeflag", slog.String("typeflag", tfStr), slog.String("name", name), slog.Any("hdr", hdr)) + return nil, fmt.Errorf("unhandled tar entry typeflag %q at %q", tfStr, name) } } diff --git a/internal/imagecache/unpack_test.go b/internal/imagecache/unpack_test.go index d6f8c375a9..746d4d328b 100644 --- a/internal/imagecache/unpack_test.go +++ b/internal/imagecache/unpack_test.go @@ -18,6 +18,7 @@ import ( "archive/tar" "bytes" "context" + "errors" "os" "path/filepath" "slices" @@ -33,6 +34,8 @@ type tarEntry struct { body string linkname string modTime time.Time + devmajor int64 + devminor int64 } func defaultMode(typeflag byte) int64 { @@ -62,6 +65,8 @@ func buildTar(t *testing.T, entries []tarEntry) []byte { Size: int64(len(e.body)), Linkname: e.linkname, ModTime: e.modTime, + Devmajor: e.devmajor, + Devminor: e.devminor, } if err := tw.WriteHeader(hdr); err != nil { t.Fatalf("tar.WriteHeader(%+v): %v", hdr, err) @@ -627,25 +632,47 @@ func TestUnpackLayer_HardlinkEscape(t *testing.T) { } } -func TestUnpackLayer_RejectSpecialFiles(t *testing.T) { - tests := []struct { - name string - typeflag byte - }{ - {name: "char device", typeflag: tar.TypeChar}, - {name: "block device", typeflag: tar.TypeBlock}, - {name: "fifo", typeflag: tar.TypeFifo}, +func TestUnpackLayer_SkipsDeviceNodes(t *testing.T) { + dir, wh, err := runUnpack(t, []tarEntry{ + {name: "dev/", typeflag: tar.TypeDir}, + {name: "dev/console", typeflag: tar.TypeChar, devmajor: 5, devminor: 1}, + {name: "dev/sda", typeflag: tar.TypeBlock, devmajor: 8}, + {name: "dev/zero", typeflag: tar.TypeChar}, + // Undeclared parent of a skipped device node must not be created or + // recorded in ImplicitDirs. + {name: "undeclared/tty0", typeflag: tar.TypeChar, devmajor: 4}, + // A device node at an existing path removes the prior entry (including + // a declared directory subtree) without breaking the mode-restore pass. + {name: "dev/replaced", typeflag: tar.TypeReg, body: "stale"}, + {name: "dev/replaced", typeflag: tar.TypeChar, devmajor: 1, devminor: 3}, + {name: "dev/subdir/", typeflag: tar.TypeDir}, + {name: "dev/subdir/nested/", typeflag: tar.TypeDir}, + {name: "dev/subdir", typeflag: tar.TypeChar, devmajor: 1, devminor: 5}, + {name: "etc/hostname", typeflag: tar.TypeReg, body: "x"}, + }) + if err != nil { + t.Fatalf("unpackLayer: %v", err) } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - _, _, err := runUnpack(t, []tarEntry{{name: "weird", typeflag: tc.typeflag}}) - if err == nil { - t.Fatalf("unpackLayer succeeded, want unhandled-typeflag error") - } - if !strings.Contains(err.Error(), "unhandled tar entry typeflag") { - t.Errorf("error = %q, want it to mention 'unhandled tar entry typeflag'", err.Error()) - } - }) + for _, name := range []string{"dev/console", "dev/sda", "dev/zero", "undeclared", "undeclared/tty0", "dev/replaced", "dev/subdir"} { + if _, err := os.Lstat(filepath.Join(dir, name)); !errors.Is(err, os.ErrNotExist) { + t.Errorf("Lstat(%q) = %v, want not exist", name, err) + } + } + if want := []string{"etc"}; !slices.Equal(wh.ImplicitDirs, want) { + t.Errorf("ImplicitDirs = %v, want %v", wh.ImplicitDirs, want) + } + if _, err := os.Stat(filepath.Join(dir, "etc/hostname")); err != nil { + t.Errorf("entry after device nodes not extracted: %v", err) + } +} + +func TestUnpackLayer_RejectFifo(t *testing.T) { + _, _, err := runUnpack(t, []tarEntry{{name: "weird", typeflag: tar.TypeFifo}}) + if err == nil { + t.Fatalf("unpackLayer succeeded, want unhandled-typeflag error") + } + if want := `unhandled tar entry typeflag "6" at "weird"`; !strings.Contains(err.Error(), want) { + t.Errorf("error = %q, want it to contain %q", err.Error(), want) } } diff --git a/internal/installdefaults/installdefaults.go b/internal/installdefaults/installdefaults.go index d0e6980c4e..3ebe646191 100644 --- a/internal/installdefaults/installdefaults.go +++ b/internal/installdefaults/installdefaults.go @@ -51,6 +51,10 @@ const ( EgressServiceAccount = "atenet-egress" ControllerServiceAccount = "ate-controller" + // WorkerServiceAccount is what worker pods run as when their WorkerPool + // names none; atecontroller creates it in the pool's namespace. + WorkerServiceAccount = "ateworker" + // PodNamespaceEnv is the conventional env var name for the namespace // a pod is running in, exposed via Kubernetes' downward API. PodNamespaceEnv = "POD_NAMESPACE" @@ -76,6 +80,13 @@ func ActorJWTIssuer(namespace string) string { }).String() } +// ClientPrincipal returns the principal ID ate-api-server sees for a bearer +// token minted from ClientServiceAccount in namespace: the Kubernetes +// ServiceAccount username in the token's subject. +func ClientPrincipal(namespace string) string { + return "system:serviceaccount:" + namespace + ":" + ClientServiceAccount +} + // SPIFFEID returns the SPIFFE ID that Pod certificates for serviceAccount in // namespace carry. Peers authenticate by comparing against this exact string. func SPIFFEID(namespace, serviceAccount string) string { diff --git a/internal/installdefaults/installdefaults_test.go b/internal/installdefaults/installdefaults_test.go index 165265abc2..e15cc7f128 100644 --- a/internal/installdefaults/installdefaults_test.go +++ b/internal/installdefaults/installdefaults_test.go @@ -16,6 +16,14 @@ package installdefaults import "testing" +// ate-api-server matches bootstrap owners on this exact string, which is the +// username Kubernetes puts in a ServiceAccount token's subject. +func TestClientPrincipal(t *testing.T) { + if got, want := ClientPrincipal("ate-system"), "system:serviceaccount:ate-system:ate-client"; got != want { + t.Errorf("ClientPrincipal(ate-system) = %q, want %q", got, want) + } +} + func TestAteletSPIFFEID(t *testing.T) { tests := []struct { name string diff --git a/internal/linuxsys/doc.go b/internal/linuxsys/doc.go new file mode 100644 index 0000000000..1d52cd25e2 --- /dev/null +++ b/internal/linuxsys/doc.go @@ -0,0 +1,17 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package linuxsys adapts Linux-specific syscalls and socket options for the +// rest of Substrate. +package linuxsys diff --git a/internal/atunnel/original_dst_format_linux_test.go b/internal/linuxsys/original_dst_format_linux_test.go similarity index 99% rename from internal/atunnel/original_dst_format_linux_test.go rename to internal/linuxsys/original_dst_format_linux_test.go index 069f481df9..2bebc5bb71 100644 --- a/internal/atunnel/original_dst_format_linux_test.go +++ b/internal/linuxsys/original_dst_format_linux_test.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package atunnel +package linuxsys import ( "encoding/binary" diff --git a/internal/atunnel/original_dst_linux.go b/internal/linuxsys/original_dst_linux.go similarity index 85% rename from internal/atunnel/original_dst_linux.go rename to internal/linuxsys/original_dst_linux.go index 0c2d44bef9..5544aaaef3 100644 --- a/internal/atunnel/original_dst_linux.go +++ b/internal/linuxsys/original_dst_linux.go @@ -14,7 +14,7 @@ //go:build linux -package atunnel +package linuxsys import ( "encoding/binary" @@ -35,18 +35,18 @@ const ip6tSOOriginalDst = 80 func TCPOriginalDestination(conn net.Conn) (string, error) { tcpConn, ok := conn.(*net.TCPConn) if !ok { - return "", fmt.Errorf("atunnel: original destination requires a TCP connection, got %T", conn) + return "", fmt.Errorf("linuxsys: original destination requires a TCP connection, got %T", conn) } rawConn, err := tcpConn.SyscallConn() if err != nil { - return "", fmt.Errorf("atunnel: acquiring TCP syscall connection: %w", err) + return "", fmt.Errorf("linuxsys: acquiring TCP syscall connection: %w", err) } // The IPv6 option is only meaningful on an AF_INET6 socket: on AF_INET the // kernel returns EOPNOTSUPP, which would mask the real IPv4 error. A // v4-mapped local address still means an IPv4 flow, so To4 is the test. local, ok := tcpConn.LocalAddr().(*net.TCPAddr) if !ok { - return "", fmt.Errorf("atunnel: original destination requires a TCP local address, got %T", tcpConn.LocalAddr()) + return "", fmt.Errorf("linuxsys: original destination requires a TCP local address, got %T", tcpConn.LocalAddr()) } isIPv6 := local.IP.To4() == nil @@ -59,14 +59,14 @@ func TCPOriginalDestination(conn net.Conn) (string, error) { } destination, sockoptErr = originalIPv4Destination(fd) }); err != nil { - return "", fmt.Errorf("atunnel: accessing TCP socket: %w", err) + return "", fmt.Errorf("linuxsys: accessing TCP socket: %w", err) } if sockoptErr != nil { family := "IPv4" if isIPv6 { family = "IPv6" } - return "", fmt.Errorf("atunnel: reading original %s TCP destination: %w", family, sockoptErr) + return "", fmt.Errorf("linuxsys: reading original %s TCP destination: %w", family, sockoptErr) } return destination, nil } @@ -108,7 +108,7 @@ func formatOriginalDestination(ip []byte, rawPort uint16) (string, error) { portBytes := (*[2]byte)(unsafe.Pointer(&rawPort)) port := binary.BigEndian.Uint16(portBytes[:]) if port == 0 { - return "", fmt.Errorf("atunnel: original TCP destination has port zero") + return "", fmt.Errorf("linuxsys: original TCP destination has port zero") } return net.JoinHostPort(net.IP(ip).String(), strconv.Itoa(int(port))), nil } diff --git a/internal/atunnel/original_dst_linux_test.go b/internal/linuxsys/original_dst_linux_test.go similarity index 99% rename from internal/atunnel/original_dst_linux_test.go rename to internal/linuxsys/original_dst_linux_test.go index 1e71bc95f4..a234b8a08f 100644 --- a/internal/atunnel/original_dst_linux_test.go +++ b/internal/linuxsys/original_dst_linux_test.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package atunnel +package linuxsys import ( "context" @@ -33,8 +33,8 @@ import ( "github.com/vishvananda/netlink" "golang.org/x/sys/unix" - "github.com/agent-substrate/substrate/internal/netns" - "github.com/agent-substrate/substrate/internal/roottest" + "go.ate.dev/substrate/internal/netns" + "go.ate.dev/substrate/internal/roottest" ) var testNetNSSequence uint64 diff --git a/internal/localjwtauthority/jwks.go b/internal/localjwtauthority/jwks.go index 6dd29d5e3c..8f857ba972 100644 --- a/internal/localjwtauthority/jwks.go +++ b/internal/localjwtauthority/jwks.go @@ -42,6 +42,27 @@ type jwk struct { E string `json:"e,omitempty"` } +// JWKS returns the JWK set, as JSON, that publishes the pool's public keys for +// signature verification. It carries no private material, so it is what a +// verifier that cannot use discovery is given instead of the pool itself. +func (p *ConcretePool) JWKS() ([]byte, error) { + keys, err := p.VerificationKeys() + if err != nil { + return nil, err + } + return JWKS(keys) +} + +// PublicJWKS returns the JWK set for a serialized pool, for writers that hold +// the pool only as bytes. +func PublicJWKS(poolBytes []byte) ([]byte, error) { + pool, err := Unmarshal(poolBytes) + if err != nil { + return nil, fmt.Errorf("while reading the pool: %w", err) + } + return pool.JWKS() +} + // JWKS returns the JWK set, as JSON, that publishes keys for signature // verification, sorted by key ID. Each key's algorithm must fit its type: // ES256 for a P-256 EC key, or RS256, RS384, or RS512 for an RSA key. diff --git a/internal/localjwtauthority/jwks_filesource_test.go b/internal/localjwtauthority/jwks_filesource_test.go new file mode 100644 index 0000000000..66350d6dff --- /dev/null +++ b/internal/localjwtauthority/jwks_filesource_test.go @@ -0,0 +1,102 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package localjwtauthority + +import ( + "os" + "path/filepath" + "testing" + "time" + + "go.ate.dev/substrate/internal/actoridjwt" + "go.ate.dev/substrate/internal/oidcjwt" +) + +const fileSourceTestIssuer = "https://idp.ate-system.svc" + +func writeJWKS(t *testing.T, path string, pool *ConcretePool) { + t.Helper() + b, err := pool.JWKS() + if err != nil { + t.Fatalf("JWKS: %v", err) + } + if err := os.WriteFile(path, b, 0o600); err != nil { + t.Fatalf("write JWKS: %v", err) + } +} + +func signTestJWT(t *testing.T, pool *ConcretePool, now time.Time) string { + t.Helper() + jwt, err := pool.SignJWT(&actoridjwt.Claims{ + Kind: actoridjwt.KindWorkerForActor, + Issuer: fileSourceTestIssuer, + Subject: "worker-for-actor/a/b", + Audiences: []string{"aud"}, + Expiration: now.Add(time.Hour), + NotBefore: now.Add(-5 * time.Minute), + IssuedAt: now, + Substrate: actoridjwt.SubstrateClaims{Atespace: "a", ActorName: "b", ActorUID: "u"}, + }) + if err != nil { + t.Fatalf("SignJWT: %v", err) + } + return jwt +} + +// TestJWKS_FeedsFileKeySource proves the pool's published JWKS is what the +// verifier's file source needs: tokens the pool signs verify against it, +// across a rotation, and tokens from a stranger do not. +func TestJWKS_FeedsFileKeySource(t *testing.T) { + first, err := GenerateAuthority("ES256", "first") + if err != nil { + t.Fatal(err) + } + pool := &ConcretePool{Authorities: []*Authority{first}, ActiveForSigning: "first"} + path := filepath.Join(t.TempDir(), "jwks.json") + writeJWKS(t, path, pool) + + now := time.Now() + verifier := oidcjwt.NewVerifier(fileSourceTestIssuer, []string{"aud"}, oidcjwt.FileKeySource(path), oidcjwt.WithSubstrateClaims()) + claims, err := verifier.Verify(t.Context(), signTestJWT(t, pool, now), now) + if err != nil { + t.Fatalf("Verify: %v", err) + } + if claims.Subject != "worker-for-actor/a/b" { + t.Errorf("sub = %q", claims.Subject) + } + if claims.Kind != actoridjwt.KindWorkerForActor || claims.Substrate == nil || claims.Substrate.Atespace != "a" { + t.Errorf("Substrate claims = %+v, want worker-for-actor", claims.SubstrateClaims) + } + + next, err := GenerateAuthority("ES256", "next") + if err != nil { + t.Fatal(err) + } + pool.Authorities = append(pool.Authorities, next) + pool.ActiveForSigning = "next" + writeJWKS(t, path, pool) + if _, err := verifier.Verify(t.Context(), signTestJWT(t, pool, now), now); err != nil { + t.Fatalf("Verify after rotation: %v", err) + } + + stranger, err := GenerateAuthority("ES256", "stranger") + if err != nil { + t.Fatal(err) + } + strangerPool := &ConcretePool{Authorities: []*Authority{stranger}, ActiveForSigning: "stranger"} + if _, err := verifier.Verify(t.Context(), signTestJWT(t, strangerPool, now.Add(2*time.Minute)), now.Add(2*time.Minute)); err == nil { + t.Fatal("Verify of a token from an unknown authority succeeded, want error") + } +} diff --git a/internal/localjwtauthority/localjwtauthority.go b/internal/localjwtauthority/localjwtauthority.go index 6e3fc5adc2..d15ad84c7c 100644 --- a/internal/localjwtauthority/localjwtauthority.go +++ b/internal/localjwtauthority/localjwtauthority.go @@ -31,7 +31,7 @@ import ( "sync" "time" - "github.com/agent-substrate/substrate/internal/actoridjwt" + "go.ate.dev/substrate/internal/actoridjwt" ) // Pool is the interface for a JWT signing pool. diff --git a/internal/localjwtauthority/localjwtauthority_test.go b/internal/localjwtauthority/localjwtauthority_test.go index bc7f538a2f..937ad27608 100644 --- a/internal/localjwtauthority/localjwtauthority_test.go +++ b/internal/localjwtauthority/localjwtauthority_test.go @@ -29,7 +29,7 @@ import ( "github.com/google/go-cmp/cmp" - "github.com/agent-substrate/substrate/internal/actoridjwt" + "go.ate.dev/substrate/internal/actoridjwt" ) func TestRefreshingPool(t *testing.T) { diff --git a/internal/netns/netns_linux_test.go b/internal/netns/netns_linux_test.go index a2a38b5321..313a4bdd76 100644 --- a/internal/netns/netns_linux_test.go +++ b/internal/netns/netns_linux_test.go @@ -22,8 +22,8 @@ import ( "os" "testing" - "github.com/agent-substrate/substrate/internal/roottest" "github.com/vishvananda/netlink" + "go.ate.dev/substrate/internal/roottest" "golang.org/x/sys/unix" ) diff --git a/internal/netns/sysctl_root_linux_test.go b/internal/netns/sysctl_root_linux_test.go index 81be17442f..12bb4f50f1 100644 --- a/internal/netns/sysctl_root_linux_test.go +++ b/internal/netns/sysctl_root_linux_test.go @@ -24,7 +24,7 @@ import ( "syscall" "testing" - "github.com/agent-substrate/substrate/internal/roottest" + "go.ate.dev/substrate/internal/roottest" "golang.org/x/sys/unix" ) diff --git a/internal/nodepath/nodepath.go b/internal/nodepath/nodepath.go index 5743f40310..2eb3431cbd 100644 --- a/internal/nodepath/nodepath.go +++ b/internal/nodepath/nodepath.go @@ -58,7 +58,7 @@ func AteletOTLPSocketPath() string { // WorkersDir is the parent of every per-worker directory, so listing it is how // a scraper with no prior knowledge discovers the node's workers. atelet -// creates WorkerPath(podUID) once it writes the pod's sandbox versions file and +// creates WorkerPath(podUID) once it writes the pod's sandbox runtimes file and // removes it when the pod is deleted; the worker also MkdirAlls it at boot. A // directory may exist before its worker listens on its socket. func WorkersDir() string { @@ -76,11 +76,20 @@ func WorkerSocketPath(podUID string) string { ) } -// SandboxVersionsFileName is the base name of the file in WorkerPath(podUID) -// where atelet keeps the sandbox versions of a worker's pool, a -// sandboxversions.File. A runtime it lists may not have its assets on the +// SandboxRuntimesFileName is the base name of the file in WorkerPath(podUID) +// where atelet keeps the sandbox runtimes of a worker's pool, a +// sandboxruntimes.File. A runtime it lists may not have its assets on the // node yet. -const SandboxVersionsFileName = "sandbox-versions.json" +const SandboxRuntimesFileName = "sandbox-runtimes.json" + +// WorkerSandboxRuntimesPath is the path of the sandbox runtimes file atelet +// writes for the worker pod with the given UID. +func WorkerSandboxRuntimesPath(podUID string) string { + return filepath.Join( + WorkerPath(podUID), + SandboxRuntimesFileName, + ) +} // ActorNetNSName names an actor's sandbox network namespace. func ActorNetNSName(actorUID string) string { diff --git a/internal/objectstore/gcs_test.go b/internal/objectstore/gcs_test.go index b83107353e..dfd3e8a9d5 100644 --- a/internal/objectstore/gcs_test.go +++ b/internal/objectstore/gcs_test.go @@ -24,8 +24,8 @@ import ( "testing" "cloud.google.com/go/storage" - "github.com/agent-substrate/substrate/internal/objectstore" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/internal/objectstore" "google.golang.org/api/option" ) diff --git a/internal/objectstore/objectstore.go b/internal/objectstore/objectstore.go index 59675e157b..490b730554 100644 --- a/internal/objectstore/objectstore.go +++ b/internal/objectstore/objectstore.go @@ -31,8 +31,8 @@ import ( "strings" "cloud.google.com/go/storage" - "github.com/agent-substrate/substrate/internal/resources" "github.com/aws/smithy-go" + "go.ate.dev/substrate/internal/resources" "golang.org/x/sync/errgroup" ) diff --git a/internal/objectstore/objectstore_test.go b/internal/objectstore/objectstore_test.go index f361741a50..75117b17d3 100644 --- a/internal/objectstore/objectstore_test.go +++ b/internal/objectstore/objectstore_test.go @@ -23,12 +23,12 @@ import ( "testing" "cloud.google.com/go/storage" - "github.com/agent-substrate/substrate/internal/objectstore" - "github.com/agent-substrate/substrate/internal/objectstore/objectstoretest" - "github.com/agent-substrate/substrate/internal/resources" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + "go.ate.dev/substrate/internal/objectstore" + "go.ate.dev/substrate/internal/objectstore/objectstoretest" + "go.ate.dev/substrate/internal/resources" ) const testLocation = "gs://bucket/root" diff --git a/internal/objectstore/objectstoretest/objectstoretest.go b/internal/objectstore/objectstoretest/objectstoretest.go index d0b5e371ce..6d8fde50ba 100644 --- a/internal/objectstore/objectstoretest/objectstoretest.go +++ b/internal/objectstore/objectstoretest/objectstoretest.go @@ -24,8 +24,8 @@ import ( "sync" "testing" - "github.com/agent-substrate/substrate/internal/objectstore" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/objectstore" + "go.ate.dev/substrate/internal/resources" ) // Fake is an in-memory object store. It is safe for concurrent use, as the diff --git a/internal/objectstore/s3_test.go b/internal/objectstore/s3_test.go index 7698a3ddb3..b8d7f93dae 100644 --- a/internal/objectstore/s3_test.go +++ b/internal/objectstore/s3_test.go @@ -25,12 +25,12 @@ import ( "sync" "testing" - "github.com/agent-substrate/substrate/internal/objectstore" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/smithy-go" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/internal/objectstore" ) // s3Request is one call the SDK made, reduced to what the tests assert on. diff --git a/internal/objectstoreplugin/actorscope/actorscope.go b/internal/objectstoreplugin/actorscope/actorscope.go new file mode 100644 index 0000000000..89e2e0a85d --- /dev/null +++ b/internal/objectstoreplugin/actorscope/actorscope.go @@ -0,0 +1,122 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package actorscope derives the object-storage prefixes an actor's transfers +// may touch from the actor record, with the access each needs. It is the +// cloud-neutral half of the credential exchange. A provider turns the grants +// into a scoped credential, for GCS a Credential Access Boundary. +package actorscope + +import ( + "fmt" + "sort" + + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" +) + +// Access is what a grant permits under its prefix. +type Access int + +const ( + // ReadOnly covers the tag or golden snapshot the actor was created from. + ReadOnly Access = iota + 1 + // ReadWrite covers the actor's own snapshots. + ReadWrite +) + +func (a Access) String() string { + switch a { + case ReadOnly: + return "ro" + case ReadWrite: + return "rw" + } + return fmt.Sprintf("Access(%d)", int(a)) +} + +// Grant is one prefix and the access allowed under it. Prefix is the actor's +// own prefix or a tag's prefix, as the control plane built it, so a provider +// derives its bucket and path with the same code the rest of Substrate uses. +type Grant struct { + Prefix resources.StoragePrefix + Access Access +} + +// FromActor derives the grants from status.snapshots. Every snapshot the +// control plane writes for the actor carries a URI the control plane built, so +// the storage location and the tag prefixes come from those URIs, parsed with +// the same code that built them, and the actor's own prefix is built with the +// helper the control plane places snapshots with. +// +// - Every durable URI names the storage location it sits under. For each +// location seen, the actor gets ReadWrite on its own prefix there, +// /atespaces//actors/. So the grant exists from +// the first generation on, before the actor has uploaded anything, and +// a repointed actor keeps both locations. +// - Each TAG-owned generation with a COMPLETED durable URI grants ReadOnly +// on the tag's prefix. That is the golden or tag the actor was created +// from, listed until the actor's own first snapshot completes. +// +// The status is taken as ate-api-server wrote it. An owner this code does not +// know is an error, so a new kind is never silently dropped. Local-only +// generations have no URI and contribute nothing. +func FromActor(actor *ateapipb.Actor) ([]Grant, error) { + meta := actor.GetMetadata() + self := resources.ActorSnapshotOwner(meta.GetAtespace(), meta.GetUid()) + seen := map[Grant]bool{} + var grants []Grant + add := func(prefix resources.StoragePrefix, access Access) { + g := Grant{Prefix: prefix, Access: access} + if !seen[g] { + seen[g] = true + grants = append(grants, g) + } + } + for _, gen := range actor.GetStatus().GetSnapshots() { + durable := gen.GetDurableSnapshot() + uri := durable.GetObject().GetSnapshotUri() + if uri == "" { + continue + } + parsed, err := resources.ParseSnapshotURI(uri) + if err != nil { + return nil, fmt.Errorf("snapshot generation %d of actor %s/%s: %w", gen.GetGeneration(), meta.GetAtespace(), meta.GetName(), err) + } + own, err := self.Prefix(parsed.Location()) + if err != nil { + return nil, fmt.Errorf("snapshot generation %d of actor %s/%s: %w", gen.GetGeneration(), meta.GetAtespace(), meta.GetName(), err) + } + add(own, ReadWrite) + switch gen.GetOwner() { + case ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR: + // Its own prefix is already granted above. + case ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG: + if durable.GetStatus() != ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + continue + } + add(parsed.OwnerPrefix(), ReadOnly) + default: + return nil, fmt.Errorf("snapshot generation %d of actor %s/%s has unknown owner %v", gen.GetGeneration(), meta.GetAtespace(), meta.GetName(), gen.GetOwner()) + } + } + // ReadWrite first, then by prefix, so equal inputs give equal output. + sort.SliceStable(grants, func(i, j int) bool { + if grants[i].Access != grants[j].Access { + return grants[i].Access > grants[j].Access + } + return grants[i].Prefix.String() < grants[j].Prefix.String() + }) + return grants, nil +} diff --git a/internal/objectstoreplugin/actorscope/actorscope_test.go b/internal/objectstoreplugin/actorscope/actorscope_test.go new file mode 100644 index 0000000000..021c053d22 --- /dev/null +++ b/internal/objectstoreplugin/actorscope/actorscope_test.go @@ -0,0 +1,158 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package actorscope + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" +) + +const ( + loc = "gs://snaps/ate-demo" + atespace = "team-a" + uid = "9f1e0b2a-9e4f-4c3d-8a6b-5f2e1d0c9b8a" + tagUID = "7a3c0b2a-9e4f-4c3d-8a6b-5f2e1d0c9b8a" +) + +func actorURI(location, name string) string { + return location + "/atespaces/" + atespace + "/actors/" + uid + "/snapshots/" + name +} + +func gen(n int32, owner ateapipb.SnapshotOwner, status ateapipb.SnapshotStorageStatus, uri string) *ateapipb.Snapshot { + s := &ateapipb.Snapshot{Generation: n, Owner: owner} + if uri != "" { + s.DurableSnapshot = &ateapipb.SnapshotStorage{Status: status, Object: &ateapipb.ObjectSnapshot{SnapshotUri: uri}} + } else { + s.Locality = "node-1" + } + return s +} + +func actorWith(snaps ...*ateapipb.Snapshot) *ateapipb.Actor { + return &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: atespace, Name: "bot", Uid: uid}, + Status: &ateapipb.ActorStatus{Snapshots: snaps}, + } +} + +// prefix builds the StoragePrefix a grant carries, from the string form the +// control plane records. +func prefix(uri string) resources.StoragePrefix { + p, err := resources.ParseStoragePrefix(uri) + if err != nil { + panic(err) + } + return p +} + +var ( + tagURI = loc + "/atespaces/ate-golden/tags/" + tagUID + tagGrant = Grant{Prefix: prefix(tagURI), Access: ReadOnly} + actorGrant = Grant{Prefix: prefix(loc + "/atespaces/" + atespace + "/actors/" + uid), Access: ReadWrite} + completed = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + pending = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + ownerActor = ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR + ownerTag = ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG +) + +func TestFromActor(t *testing.T) { + for _, tc := range []struct { + name string + actor *ateapipb.Actor + want []Grant + }{ + { + name: "no generations", + actor: actorWith(), + want: nil, + }, + { + name: "tag only, before the first suspend", + actor: actorWith(gen(1, ownerTag, completed, tagURI)), + want: []Grant{actorGrant, tagGrant}, + }, + { + name: "tag plus pending actor generation, during the first suspend", + actor: actorWith( + gen(1, ownerTag, completed, tagURI), + gen(2, ownerActor, pending, actorURI(loc, "c0de")), + ), + want: []Grant{actorGrant, tagGrant}, + }, + { + name: "actor only, after the first suspend", + actor: actorWith(gen(2, ownerActor, completed, actorURI(loc, "c0de"))), + want: []Grant{actorGrant}, + }, + { + name: "local-only generation adds nothing", + actor: actorWith( + gen(2, ownerActor, completed, actorURI(loc, "c0de")), + gen(3, ownerActor, completed, ""), + ), + want: []Grant{actorGrant}, + }, + { + name: "two actor generations under one prefix give one grant", + actor: actorWith( + gen(2, ownerActor, completed, actorURI(loc, "c0de")), + gen(3, ownerActor, pending, actorURI(loc, "d00d")), + ), + want: []Grant{actorGrant}, + }, + { + name: "a repointed actor keeps both locations", + actor: actorWith( + gen(2, ownerActor, completed, actorURI(loc, "c0de")), + gen(3, ownerActor, pending, actorURI("gs://other-bucket/root", "d00d")), + ), + want: []Grant{ + {Prefix: prefix("gs://other-bucket/root/atespaces/" + atespace + "/actors/" + uid), Access: ReadWrite}, + actorGrant, + }, + }, + { + name: "a pending tag copy is not yet readable, but names the location", + actor: actorWith(gen(1, ownerTag, pending, tagURI)), + want: []Grant{actorGrant}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := FromActor(tc.actor) + if err != nil { + t.Fatalf("FromActor: %v", err) + } + if diff := cmp.Diff(tc.want, got, cmp.AllowUnexported(resources.StoragePrefix{})); diff != "" { + t.Errorf("grants (-want +got):\n%s", diff) + } + }) + } +} + +func TestFromActor_RefusesUnknownOwner(t *testing.T) { + if _, err := FromActor(actorWith(gen(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_UNSPECIFIED, completed, tagURI))); err == nil { + t.Fatal("a generation with an unspecified owner was accepted") + } +} + +func TestFromActor_RefusesMalformedURI(t *testing.T) { + if _, err := FromActor(actorWith(gen(2, ownerActor, completed, "gs://snaps/not/a/snapshot"))); err == nil { + t.Fatal("a URI outside the snapshot layout was accepted") + } +} diff --git a/internal/objectstoreplugin/asset.go b/internal/objectstoreplugin/asset.go index 9be7eff19e..36bd0dc7de 100644 --- a/internal/objectstoreplugin/asset.go +++ b/internal/objectstoreplugin/asset.go @@ -20,9 +20,9 @@ import ( "io" "strings" - "github.com/agent-substrate/substrate/pkg/assetfetch" - "github.com/agent-substrate/substrate/pkg/objectstorage" - objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "go.ate.dev/substrate/pkg/assetfetch" + "go.ate.dev/substrate/pkg/objectstorage" + objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) diff --git a/internal/objectstoreplugin/asset_test.go b/internal/objectstoreplugin/asset_test.go index 92778f8840..3ab28d6d84 100644 --- a/internal/objectstoreplugin/asset_test.go +++ b/internal/objectstoreplugin/asset_test.go @@ -26,9 +26,9 @@ import ( "sync/atomic" "testing" - "github.com/agent-substrate/substrate/internal/pluginsocket" - "github.com/agent-substrate/substrate/pkg/assetfetch" - objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "go.ate.dev/substrate/internal/pluginsocket" + "go.ate.dev/substrate/pkg/assetfetch" + objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" diff --git a/internal/objectstoreplugin/control.go b/internal/objectstoreplugin/control.go index 69b46787f4..6541584e08 100644 --- a/internal/objectstoreplugin/control.go +++ b/internal/objectstoreplugin/control.go @@ -17,9 +17,9 @@ package objectstoreplugin import ( "context" - "github.com/agent-substrate/substrate/internal/objectstore" - "github.com/agent-substrate/substrate/internal/resources" - objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "go.ate.dev/substrate/internal/objectstore" + "go.ate.dev/substrate/internal/resources" + objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) diff --git a/internal/objectstoreplugin/gcsboundary/boundary.go b/internal/objectstoreplugin/gcsboundary/boundary.go new file mode 100644 index 0000000000..cec0bc8600 --- /dev/null +++ b/internal/objectstoreplugin/gcsboundary/boundary.go @@ -0,0 +1,85 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gcsboundary turns actor grants into the Credential Access Boundary +// rules a downscoped GCS token carries. One rule per grant, each limited to its +// bucket and an object-name prefix. +package gcsboundary + +import ( + "fmt" + "strings" + + "golang.org/x/oauth2/google/downscope" + + "go.ate.dev/substrate/internal/objectstore" + "go.ate.dev/substrate/internal/objectstoreplugin/actorscope" + "go.ate.dev/substrate/internal/resources" +) + +const ( + roleObjectViewer = "inRole:roles/storage.objectViewer" + // objectAdmin, not objectCreator: an upload overwrites on retry and deletes + // parts after a compose. + roleObjectAdmin = "inRole:roles/storage.objectAdmin" +) + +// FromGrants builds the rules. Each grant's prefix must be a gs:// prefix. The +// condition's path ends in "/", so a match cannot spill into a sibling whose +// name merely starts the same way. No objectListPrefix term is added: the node +// plugin never lists objects. downscope.NewTokenSource enforces the rule +// count, between one and ten, see +// https://docs.cloud.google.com/iam/docs/downscoping-short-lived-credentials. +func FromGrants(grants []actorscope.Grant) ([]downscope.AccessBoundaryRule, error) { + var rules []downscope.AccessBoundaryRule + for _, g := range grants { + bucket, path, err := splitGCS(g.Prefix) + if err != nil { + return nil, err + } + var role string + switch g.Access { + case actorscope.ReadOnly: + role = roleObjectViewer + case actorscope.ReadWrite: + role = roleObjectAdmin + default: + return nil, fmt.Errorf("grant on %s has unknown access %v", g.Prefix, g.Access) + } + rules = append(rules, downscope.AccessBoundaryRule{ + AvailableResource: "//storage.googleapis.com/projects/_/buckets/" + bucket, + AvailablePermissions: []string{role}, + Condition: &downscope.AvailabilityCondition{ + Expression: fmt.Sprintf("resource.name.startsWith('projects/_/buckets/%s/objects/%s')", celEscape(bucket), celEscape(path)), + }, + }) + } + return rules, nil +} + +// splitGCS returns the bucket and object path of a gs:// prefix, the path with +// its trailing slash. +func splitGCS(prefix resources.StoragePrefix) (bucket, path string, err error) { + if !strings.HasPrefix(prefix.String(), "gs://") { + return "", "", fmt.Errorf("prefix %q is not a gs:// prefix", prefix) + } + return objectstore.BucketPrefix(prefix) +} + +// celEscaper quotes the two characters that would end or alter a CEL +// single-quoted string literal. A storage location may contain a quote, since +// Substrate accepts one, so it is escaped rather than refused. +var celEscaper = strings.NewReplacer(`\`, `\\`, `'`, `\'`) + +func celEscape(s string) string { return celEscaper.Replace(s) } diff --git a/internal/objectstoreplugin/gcsboundary/boundary_test.go b/internal/objectstoreplugin/gcsboundary/boundary_test.go new file mode 100644 index 0000000000..d5cd3597eb --- /dev/null +++ b/internal/objectstoreplugin/gcsboundary/boundary_test.go @@ -0,0 +1,93 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsboundary + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "golang.org/x/oauth2/google/downscope" + + "go.ate.dev/substrate/internal/objectstoreplugin/actorscope" + "go.ate.dev/substrate/internal/resources" +) + +func prefix(t *testing.T, uri string) resources.StoragePrefix { + t.Helper() + p, err := resources.ParseStoragePrefix(uri) + if err != nil { + t.Fatal(err) + } + return p +} + +func TestFromGrants(t *testing.T) { + grants := []actorscope.Grant{ + {Prefix: prefix(t, "gs://snaps/ate-demo/atespaces/team-a/actors/9f1e"), Access: actorscope.ReadWrite}, + {Prefix: prefix(t, "gs://snaps/ate-demo/atespaces/ate-golden/tags/7a3c"), Access: actorscope.ReadOnly}, + } + b, err := FromGrants(grants) + if err != nil { + t.Fatalf("FromGrants: %v", err) + } + want := []downscope.AccessBoundaryRule{ + { + AvailableResource: "//storage.googleapis.com/projects/_/buckets/snaps", + AvailablePermissions: []string{"inRole:roles/storage.objectAdmin"}, + Condition: &downscope.AvailabilityCondition{Expression: "resource.name.startsWith('projects/_/buckets/snaps/objects/ate-demo/atespaces/team-a/actors/9f1e/')"}, + }, + { + AvailableResource: "//storage.googleapis.com/projects/_/buckets/snaps", + AvailablePermissions: []string{"inRole:roles/storage.objectViewer"}, + Condition: &downscope.AvailabilityCondition{Expression: "resource.name.startsWith('projects/_/buckets/snaps/objects/ate-demo/atespaces/ate-golden/tags/7a3c/')"}, + }, + } + if diff := cmp.Diff(want, b); diff != "" { + t.Errorf("rules (-want +got):\n%s", diff) + } +} + +func TestFromGrants_Refuses(t *testing.T) { + for _, tc := range []struct { + name string + grants []actorscope.Grant + }{ + {"not gs", []actorscope.Grant{{Prefix: prefix(t, "s3://b/p"), Access: actorscope.ReadOnly}}}, + {"zero prefix", []actorscope.Grant{{Prefix: resources.StoragePrefix{}, Access: actorscope.ReadOnly}}}, + {"unknown access", []actorscope.Grant{{Prefix: prefix(t, "gs://b/p"), Access: 0}}}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, err := FromGrants(tc.grants); err == nil { + t.Fatal("accepted, want error") + } + }) + } +} + +func TestFromGrants_EscapesTheCELLiteral(t *testing.T) { + // Substrate accepts a quote in a storage location, so the condition must + // escape it rather than let it end the string literal. + rules, err := FromGrants([]actorscope.Grant{{Prefix: prefix(t, `gs://snaps/tenant's/atespaces/team-a/actors/9f1e`), Access: actorscope.ReadWrite}}) + if err != nil { + t.Fatalf("FromGrants: %v", err) + } + want := `resource.name.startsWith('projects/_/buckets/snaps/objects/tenant\'s/atespaces/team-a/actors/9f1e/')` + if got := rules[0].Condition.Expression; got != want { + t.Errorf("expression = %s, want %s", got, want) + } + if got := celEscape(`a\b'c`); got != `a\\b\'c` { + t.Errorf("celEscape = %s", got) + } +} diff --git a/internal/objectstoreplugin/gcsissuer/issuer.go b/internal/objectstoreplugin/gcsissuer/issuer.go new file mode 100644 index 0000000000..e48c88b909 --- /dev/null +++ b/internal/objectstoreplugin/gcsissuer/issuer.go @@ -0,0 +1,225 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gcsissuer mints GCS tokens downscoped to an actor's grants. It is the +// GCS half of the control plugin's credential exchange: actorscope decides what +// an actor may touch, this package turns that into a credential GCS enforces. +// +// The source credential comes from the default credential chain, which on GKE +// with Workload Identity is the federated token. The downscope package +// exchanges it at STS with a Credential Access Boundary and returns an access +// token GCS limits to the boundary. STS reports no lifetime for that token, so +// the package falls back to the source token's expiry, and the two expire +// together. +package gcsissuer + +import ( + "context" + "errors" + "fmt" + "net/http" + "slices" + "time" + + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "golang.org/x/oauth2/google/downscope" + "k8s.io/utils/lru" + + "go.ate.dev/substrate/internal/objectstoreplugin/actorscope" + "go.ate.dev/substrate/internal/objectstoreplugin/gcsboundary" +) + +const ( + // cloudPlatformScope is the scope the source token is requested with. The + // boundary narrows it to the actor's prefixes. + cloudPlatformScope = "https://www.googleapis.com/auth/cloud-platform" + + // minServeLifetime is the least remaining lifetime a token is cached and + // served with, so a transfer never starts on a token about to expire. There + // is no mid-transfer refresh. + minServeLifetime = 20 * time.Minute + // defaultCacheSize bounds the per-actor cache. A miss costs an exchange, + // never correctness. + defaultCacheSize = 100_000 + // stsTimeout bounds one exchange. The downscope package does not take a + // per-call context, so the client's timeout is the only bound. + stsTimeout = 30 * time.Second +) + +// Token is a downscoped GCS access token and when it expires. +type Token struct { + AccessToken string + Expiry time.Time +} + +// Issuer exchanges the pod's credential for per-actor downscoped tokens and +// caches them by actor UID. +type Issuer struct { + source oauth2.TokenSource + client *http.Client + now func() time.Time + cache *lru.Cache +} + +// entry is a cached token with the grants it was minted for, so a request can +// tell whether the token covers it. +type entry struct { + token Token + grants []actorscope.Grant +} + +// Option configures an Issuer. +type Option func(*Issuer) + +// WithTokenSource overrides the source credential, for tests. +func WithTokenSource(ts oauth2.TokenSource) Option { return func(i *Issuer) { i.source = ts } } + +// WithHTTPClient overrides the client used for STS, for tests. +func WithHTTPClient(c *http.Client) Option { return func(i *Issuer) { i.client = c } } + +// WithClock overrides the clock, for tests. +func WithClock(now func() time.Time) Option { return func(i *Issuer) { i.now = now } } + +// WithCacheSize bounds the token cache. A size below one is treated as one, +// since lru.New(0) would mean no bound at all. +func WithCacheSize(n int) Option { + return func(i *Issuer) { i.cache = lru.New(max(n, 1)) } +} + +// New returns an Issuer whose source credential is the default credential +// chain, unless an option replaces it. The credential keeps refreshing for the +// Issuer's whole life, so ctx is detached from cancellation. +func New(ctx context.Context, opts ...Option) (*Issuer, error) { + i := &Issuer{ + client: &http.Client{Timeout: stsTimeout}, + now: time.Now, + cache: lru.New(defaultCacheSize), + } + for _, o := range opts { + o(i) + } + if i.source == nil { + creds, err := google.FindDefaultCredentialsWithParams(context.WithoutCancel(ctx), google.CredentialsParams{ + Scopes: []string{cloudPlatformScope}, + }) + if err != nil { + return nil, fmt.Errorf("while loading the default credential: %w", err) + } + i.source = creds.TokenSource + } + return i, nil +} + +// Issue returns a token downscoped to grants for the actor. A cached token is +// returned while it covers the grants and has at least minServeLifetime left, +// unless force is set, which mints regardless and is what the node plugin +// sends after GCS refused a token. A minted token with less than +// minServeLifetime left is handed out anyway, since a short token beats a +// failed transfer, but it is not cached. That happens for a stretch at the +// end of every hour of the source credential's life: the downscoped token +// expires with its source, and the GKE metadata server serves the same cached +// source token until shortly before it expires, so no early refresh helps. In +// that stretch every exchange reaches STS and the node plugin re-exchanges +// once if a transfer outlives its token. +// +// The cache is keyed by actor UID and each entry remembers its grants. Grants +// narrow when the tag an actor was created from drops out after its first +// completed suspend, and the cached token still covers that, a hit. They +// widen when a never-suspended actor is repointed to another storage +// location, and the cached token lacks it, a mint. So the common path costs +// nothing and a repoint never hands out a token GCS would refuse. +func (i *Issuer) Issue(ctx context.Context, actorUID string, grants []actorscope.Grant, force bool) (Token, error) { + if len(grants) == 0 { + // Checked before the cache: an empty set is covered by any cached + // token, but an actor with no grants must get nothing. + return Token{}, errors.New("actor has no storage grants") + } + if !force { + if v, ok := i.cache.Get(actorUID); ok { + if e := v.(entry); covers(e.grants, grants) && e.token.Expiry.Sub(i.now()) >= minServeLifetime { + return e.token, nil + } + } + } + tok, err := i.mint(ctx, grants) + if err != nil { + return Token{}, err + } + if tok.Expiry.Sub(i.now()) >= minServeLifetime { + i.cache.Add(actorUID, entry{token: tok, grants: slices.Clone(grants)}) + } + return tok, nil +} + +// covers reports whether a token minted with the cached grants can serve a +// request for the requested grants: each requested prefix must appear in the +// cached grants with the same or higher access. Prefixes are compared as +// written, since both sides build them with the same helper. +func covers(cached, requested []actorscope.Grant) bool { + for _, r := range requested { + if !slices.ContainsFunc(cached, func(c actorscope.Grant) bool { + return c.Prefix == r.Prefix && c.Access >= r.Access + }) { + return false + } + } + return true +} + +func (i *Issuer) mint(ctx context.Context, grants []actorscope.Grant) (Token, error) { + rules, err := gcsboundary.FromGrants(grants) + if err != nil { + return Token{}, fmt.Errorf("while building the access boundary: %w", err) + } + // The downscope package reads its HTTP client from the context but sends + // the STS request without one, so the caller's deadline is attached by the + // transport instead. + ctx = context.WithValue(ctx, oauth2.HTTPClient, &http.Client{ + Transport: contextTransport{ctx: ctx, base: i.client.Transport}, + Timeout: i.client.Timeout, + }) + ts, err := downscope.NewTokenSource(ctx, downscope.DownscopingConfig{RootSource: i.source, Rules: rules}) + if err != nil { + return Token{}, fmt.Errorf("while building the access boundary: %w", err) + } + tok, err := ts.Token() + if err != nil { + return Token{}, fmt.Errorf("while exchanging at STS: %w", err) + } + if tok.AccessToken == "" { + return Token{}, errors.New("STS returned no access token") + } + if tok.Expiry.IsZero() { + // Without an expiry the node plugin could not know when to stop using + // the token. + return Token{}, errors.New("downscoped token has no expiry") + } + return Token{AccessToken: tok.AccessToken, Expiry: tok.Expiry}, nil +} + +// contextTransport sends every request with ctx, so a cancelled or expired +// caller stops an STS exchange in flight instead of waiting out the timeout. +type contextTransport struct { + ctx context.Context + base http.RoundTripper +} + +func (t contextTransport) RoundTrip(req *http.Request) (*http.Response, error) { + base := t.base + if base == nil { + base = http.DefaultTransport + } + return base.RoundTrip(req.WithContext(t.ctx)) +} diff --git a/internal/objectstoreplugin/gcsissuer/issuer_test.go b/internal/objectstoreplugin/gcsissuer/issuer_test.go new file mode 100644 index 0000000000..ea07e37baf --- /dev/null +++ b/internal/objectstoreplugin/gcsissuer/issuer_test.go @@ -0,0 +1,363 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsissuer + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google/downscope" + + "go.ate.dev/substrate/internal/objectstoreplugin/actorscope" + "go.ate.dev/substrate/internal/objectstoreplugin/gcsboundary" + "go.ate.dev/substrate/internal/resources" +) + +func prefix(uri string) resources.StoragePrefix { + p, err := resources.ParseStoragePrefix(uri) + if err != nil { + panic(err) + } + return p +} + +var ( + rwGrant = actorscope.Grant{Prefix: prefix("gs://snaps/root/atespaces/team-a/actors/9f1e"), Access: actorscope.ReadWrite} + roGrant = actorscope.Grant{Prefix: prefix("gs://snaps/root/atespaces/ate-golden/tags/7a3c"), Access: actorscope.ReadOnly} + testGrants = []actorscope.Grant{rwGrant, roGrant} +) + +// fakeSTS records the last exchange request and answers with a token. +type fakeSTS struct { + t *testing.T + calls int + status int + body string + lastURL string + last map[string]string +} + +func (f *fakeSTS) handler(w http.ResponseWriter, r *http.Request) { + f.calls++ + if err := r.ParseForm(); err != nil { + f.t.Errorf("parse form: %v", err) + } + f.last = map[string]string{} + for k := range r.PostForm { + f.last[k] = r.PostForm.Get(k) + } + if f.status != 0 { + w.WriteHeader(f.status) + } + body := f.body + if body == "" { + body = `{"access_token":"downscoped-` + strings.Repeat("x", 8) + `","issued_token_type":"urn:ietf:params:oauth:token-type:access_token","token_type":"Bearer"}` + } + _, _ = w.Write([]byte(body)) +} + +// redirect sends every request to the fake STS, recording where it was bound. +type redirect struct { + sts *fakeSTS + target *url.URL +} + +func (r redirect) RoundTrip(req *http.Request) (*http.Response, error) { + r.sts.lastURL = req.URL.String() + req = req.Clone(req.Context()) + req.URL.Scheme, req.URL.Host = r.target.Scheme, r.target.Host + return http.DefaultTransport.RoundTrip(req) +} + +func newTestIssuer(t *testing.T, sts *fakeSTS, src oauth2.TokenSource, now func() time.Time) *Issuer { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(sts.handler)) + t.Cleanup(srv.Close) + target, _ := url.Parse(srv.URL) + client := &http.Client{Transport: redirect{sts: sts, target: target}} + i, err := New(t.Context(), WithTokenSource(src), WithHTTPClient(client), WithClock(now), WithCacheSize(2)) + if err != nil { + t.Fatalf("New: %v", err) + } + return i +} + +func staticSource(expiry time.Time) oauth2.TokenSource { + return oauth2.StaticTokenSource(&oauth2.Token{AccessToken: "source-token", Expiry: expiry}) +} + +func TestIssue_ExchangesWithTheBoundary(t *testing.T) { + now := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC) + expiry := now.Add(time.Hour) + sts := &fakeSTS{t: t} + i := newTestIssuer(t, sts, staticSource(expiry), func() time.Time { return now }) + + tok, err := i.Issue(t.Context(), "9f1e", testGrants, false) + if err != nil { + t.Fatalf("Issue: %v", err) + } + if !strings.HasPrefix(tok.AccessToken, "downscoped-") { + t.Errorf("AccessToken = %q", tok.AccessToken) + } + if !tok.Expiry.Equal(expiry) { + t.Errorf("Expiry = %v, want the source token's %v", tok.Expiry, expiry) + } + if sts.lastURL != "https://sts.googleapis.com/v1/token" { + t.Errorf("exchanged at %s", sts.lastURL) + } + for k, want := range map[string]string{ + "grant_type": "urn:ietf:params:oauth:grant-type:token-exchange", + "subject_token_type": "urn:ietf:params:oauth:token-type:access_token", + "requested_token_type": "urn:ietf:params:oauth:token-type:access_token", + "subject_token": "source-token", + } { + if got := sts.last[k]; got != want { + t.Errorf("form %s = %q, want %q", k, got, want) + } + } + var options struct { + AccessBoundary struct { + AccessBoundaryRules []downscope.AccessBoundaryRule `json:"accessBoundaryRules"` + } `json:"accessBoundary"` + } + if err := json.Unmarshal([]byte(sts.last["options"]), &options); err != nil { + t.Fatalf("options is not a boundary: %v", err) + } + want, _ := gcsboundary.FromGrants(testGrants) + if diff := cmp.Diff(want, options.AccessBoundary.AccessBoundaryRules); diff != "" { + t.Errorf("rules on the wire (-want +got):\n%s", diff) + } +} + +func TestIssue_CachesPerActorUntilNearExpiry(t *testing.T) { + now := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC) + clock := now + sts := &fakeSTS{t: t} + i := newTestIssuer(t, sts, staticSource(now.Add(time.Hour)), func() time.Time { return clock }) + + first, err := i.Issue(t.Context(), "9f1e", testGrants, false) + if err != nil { + t.Fatal(err) + } + second, err := i.Issue(t.Context(), "9f1e", testGrants, false) + if err != nil { + t.Fatal(err) + } + if sts.calls != 1 || second != first { + t.Fatalf("second Issue made %d STS calls in total and returned a different token", sts.calls) + } + if _, err := i.Issue(t.Context(), "other", testGrants, false); err != nil { + t.Fatal(err) + } + if sts.calls != 2 { + t.Fatalf("a different actor should exchange, STS calls = %d", sts.calls) + } + // With less than minServeLifetime left the cached token is not served. + clock = now.Add(time.Hour - minServeLifetime + time.Second) + if _, err := i.Issue(t.Context(), "9f1e", testGrants, false); err != nil { + t.Fatal(err) + } + if sts.calls != 3 { + t.Fatalf("a token with under %v left should be re-minted, STS calls = %d", minServeLifetime, sts.calls) + } +} + +func TestIssue_ForceBypassesTheCache(t *testing.T) { + now := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC) + sts := &fakeSTS{t: t} + i := newTestIssuer(t, sts, staticSource(now.Add(time.Hour)), func() time.Time { return now }) + for _, force := range []bool{false, true} { + if _, err := i.Issue(t.Context(), "9f1e", testGrants, force); err != nil { + t.Fatal(err) + } + } + if sts.calls != 2 { + t.Fatalf("force should exchange again, STS calls = %d", sts.calls) + } +} + +func TestIssue_CacheIsBounded(t *testing.T) { + now := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC) + sts := &fakeSTS{t: t} + i := newTestIssuer(t, sts, staticSource(now.Add(time.Hour)), func() time.Time { return now }) + for _, uid := range []string{"a", "b", "c", "a"} { + if _, err := i.Issue(t.Context(), uid, testGrants, false); err != nil { + t.Fatal(err) + } + } + // Size 2: c evicted a, so the last call for a exchanged again. + if sts.calls != 4 || i.cache.Len() != 2 { + t.Fatalf("STS calls = %d, cache len = %d, want 4 and 2", sts.calls, i.cache.Len()) + } +} + +func TestIssue_ServesTheCachedTokenWhenItCoversTheGrants(t *testing.T) { + now := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC) + rwOther := actorscope.Grant{Prefix: prefix("gs://other/root/atespaces/team-a/actors/9f1e"), Access: actorscope.ReadWrite} + roOnOwn := actorscope.Grant{Prefix: rwGrant.Prefix, Access: actorscope.ReadOnly} + for _, tc := range []struct { + name string + first []actorscope.Grant + second []actorscope.Grant + wantMints int + }{ + {"same grants", testGrants, testGrants, 1}, + {"tag dropped after the first suspend", testGrants, []actorscope.Grant{rwGrant}, 1}, + {"read-only covered by read-write", testGrants, []actorscope.Grant{roOnOwn}, 1}, + {"repoint adds a location", testGrants, []actorscope.Grant{rwGrant, rwOther, roGrant}, 2}, + {"read-write not covered by read-only", []actorscope.Grant{roOnOwn}, []actorscope.Grant{rwGrant}, 2}, + } { + t.Run(tc.name, func(t *testing.T) { + sts := &fakeSTS{t: t} + i := newTestIssuer(t, sts, staticSource(now.Add(time.Hour)), func() time.Time { return now }) + for _, grants := range [][]actorscope.Grant{tc.first, tc.second} { + if _, err := i.Issue(t.Context(), "9f1e", grants, false); err != nil { + t.Fatal(err) + } + } + if sts.calls != tc.wantMints { + t.Fatalf("STS calls = %d, want %d", sts.calls, tc.wantMints) + } + }) + } +} + +func TestIssue_RefusesEmptyGrantsEvenWhenCached(t *testing.T) { + // An actor that lost its generations derives no grants. The cached token + // would cover an empty request, so the refusal has to come first. + now := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC) + sts := &fakeSTS{t: t} + i := newTestIssuer(t, sts, staticSource(now.Add(time.Hour)), func() time.Time { return now }) + if _, err := i.Issue(t.Context(), "9f1e", testGrants, false); err != nil { + t.Fatal(err) + } + if _, err := i.Issue(t.Context(), "9f1e", nil, false); err == nil { + t.Fatal("Issue with no grants served the cached token") + } + if sts.calls != 1 { + t.Fatalf("STS calls = %d, want 1", sts.calls) + } +} + +func TestIssue_StopsWhenTheCallerGivesUp(t *testing.T) { + // STS hangs. A cancelled caller must get an error at once, not after the + // client timeout. + now := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case <-r.Context().Done(): + case <-time.After(3 * time.Second): + } + })) + t.Cleanup(srv.Close) + target, _ := url.Parse(srv.URL) + client := &http.Client{Transport: redirect{sts: &fakeSTS{t: t}, target: target}, Timeout: time.Minute} + i, err := New(t.Context(), WithTokenSource(staticSource(now.Add(time.Hour))), WithHTTPClient(client), WithClock(func() time.Time { return now })) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) + defer cancel() + start := time.Now() + if _, err := i.Issue(ctx, "9f1e", testGrants, false); err == nil { + t.Fatal("Issue succeeded against a hanging STS") + } + if took := time.Since(start); took > 2*time.Second { + t.Fatalf("Issue took %v to notice the cancelled context", took) + } +} + +func TestWithCacheSize_ZeroIsBounded(t *testing.T) { + now := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC) + sts := &fakeSTS{t: t} + srv := httptest.NewServer(http.HandlerFunc(sts.handler)) + t.Cleanup(srv.Close) + target, _ := url.Parse(srv.URL) + i, err := New(t.Context(), WithTokenSource(staticSource(now.Add(time.Hour))), WithHTTPClient(&http.Client{Transport: redirect{sts: sts, target: target}}), WithClock(func() time.Time { return now }), WithCacheSize(0)) + if err != nil { + t.Fatal(err) + } + for _, uid := range []string{"a", "b"} { + if _, err := i.Issue(t.Context(), uid, testGrants, false); err != nil { + t.Fatal(err) + } + } + if i.cache.Len() != 1 { + t.Fatalf("cache len = %d, want 1", i.cache.Len()) + } +} + +func TestIssue_DoesNotCacheAShortLivedToken(t *testing.T) { + // The source credential can hand out a token with less than the floor + // left. It is returned, so the transfer can try, but not cached. + now := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC) + sts := &fakeSTS{t: t} + i := newTestIssuer(t, sts, staticSource(now.Add(minServeLifetime-time.Minute)), func() time.Time { return now }) + for range 2 { + tok, err := i.Issue(t.Context(), "9f1e", testGrants, false) + if err != nil { + t.Fatal(err) + } + if tok.AccessToken == "" { + t.Fatal("no token returned") + } + } + if sts.calls != 2 || i.cache.Len() != 0 { + t.Fatalf("STS calls = %d, cache len = %d, want 2 and 0", sts.calls, i.cache.Len()) + } +} + +func TestIssue_Errors(t *testing.T) { + now := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC) + clock := func() time.Time { return now } + for _, tc := range []struct { + name string + sts *fakeSTS + source oauth2.TokenSource + grants []actorscope.Grant + want string + }{ + {"source without expiry", &fakeSTS{t: t}, staticSource(time.Time{}), testGrants, "no expiry"}, + {"STS refuses", &fakeSTS{t: t, status: 400, body: `{"error":"invalid_grant"}`}, staticSource(now.Add(time.Hour)), testGrants, "400"}, + {"STS answers without a token", &fakeSTS{t: t, body: `{}`}, staticSource(now.Add(time.Hour)), testGrants, "no access token"}, + {"no grants", &fakeSTS{t: t}, staticSource(now.Add(time.Hour)), nil, "no storage grants"}, + {"too many grants", &fakeSTS{t: t}, staticSource(now.Add(time.Hour)), manyGrants(11), "greater than 10"}, + } { + t.Run(tc.name, func(t *testing.T) { + i := newTestIssuer(t, tc.sts, tc.source, clock) + _, err := i.Issue(t.Context(), "9f1e", tc.grants, false) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("err = %v, want it to mention %q", err, tc.want) + } + }) + } +} + +func manyGrants(n int) []actorscope.Grant { + var gs []actorscope.Grant + for i := range n { + gs = append(gs, actorscope.Grant{Prefix: prefix(fmt.Sprintf("gs://b/p%d", i)), Access: actorscope.ReadOnly}) + } + return gs +} diff --git a/internal/objectstoreplugin/node.go b/internal/objectstoreplugin/node.go index 99df8b87fc..712d3110e0 100644 --- a/internal/objectstoreplugin/node.go +++ b/internal/objectstoreplugin/node.go @@ -25,13 +25,13 @@ import ( "context" "errors" "fmt" - "io" "os" "path/filepath" + "slices" - "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/pkg/objectstorage" - objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/objectstorage" + objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" "golang.org/x/sync/errgroup" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -47,17 +47,23 @@ type NodePlugin struct { objectstorev1.UnimplementedNodeProviderServer gcsClient objectstorage.ObjectStorage + uploader *objectstorage.Uploader // root confines every local path a caller names. root string } // NewNodePlugin returns a NodePlugin that reads and writes local files only -// below root. -func NewNodePlugin(gcsClient objectstorage.ObjectStorage, root string) (*NodePlugin, error) { +// below root. All upload requests share uploadMemory bytes of reservations for +// compression and backend buffers. +func NewNodePlugin(gcsClient objectstorage.ObjectStorage, root string, uploadMemory int64) (*NodePlugin, error) { if !filepath.IsAbs(root) { return nil, fmt.Errorf("root %q is not an absolute path", root) } - return &NodePlugin{gcsClient: gcsClient, root: filepath.Clean(root)}, nil + uploader, err := objectstorage.NewUploader(gcsClient, uploadMemory) + if err != nil { + return nil, err + } + return &NodePlugin{gcsClient: gcsClient, uploader: uploader, root: filepath.Clean(root)}, nil } // FetchSnapshot downloads the requested snapshot files into write_path. @@ -119,38 +125,47 @@ func (p *NodePlugin) UploadSnapshot(ctx context.Context, req *objectstorev1.Uplo } defer root.Close() + upload := func(ctx context.Context, fileName string) error { + local, err := root.Open(fileName) + if err != nil { + return fmt.Errorf("while opening %s in snapshot directory: %w", fileName, err) + } + defer local.Close() + info, err := local.Stat() + if err != nil { + return fmt.Errorf("while inspecting %s in snapshot directory: %w", fileName, err) + } + if !info.Mode().IsRegular() { + return fmt.Errorf("snapshot file %s is not a regular file", fileName) + } + if fileName == manifestFile { + return p.uploadManifest(ctx, uri, local) + } + objectURI, err := uri.ObjectURI(fileName + ".zstd") + if err != nil { + return fmt.Errorf("while addressing %s in GCS: %w", fileName, err) + } + if err := p.uploader.SendFile(ctx, objectURI, local); err != nil { + return fmt.Errorf("while uploading %s to GCS: %w", fileName, err) + } + return nil + } g, gCtx := errgroup.WithContext(ctx) for _, fileName := range req.GetFiles() { - g.Go(func() error { - local, err := root.Open(fileName) - if err != nil { - return fmt.Errorf("while opening %s in snapshot directory: %w", fileName, err) - } - defer local.Close() - info, err := local.Stat() - if err != nil { - return fmt.Errorf("while inspecting %s in snapshot directory: %w", fileName, err) - } - if !info.Mode().IsRegular() { - return fmt.Errorf("snapshot file %s is not a regular file", fileName) - } - if fileName == manifestFile { - return p.uploadManifest(gCtx, uri, local) - } - - objectURI, err := uri.ObjectURI(fileName + ".zstd") - if err != nil { - return fmt.Errorf("while addressing %s in GCS: %w", fileName, err) - } - if err := objectstorage.SendFileToGCSWithZstd(gCtx, p.gcsClient, objectURI, local); err != nil { - return fmt.Errorf("while uploading %s to GCS: %w", fileName, err) - } - return nil - }) + if fileName != manifestFile { + g.Go(func() error { return upload(gCtx, fileName) }) + } } if err := g.Wait(); err != nil { return nil, toStatus(err) } + // A manifest commits the snapshot only after every requested data file has + // succeeded. Use ctx: errgroup cancels gCtx when Wait returns. + if slices.Contains(req.GetFiles(), manifestFile) { + if err := upload(ctx, manifestFile); err != nil { + return nil, toStatus(err) + } + } return &objectstorev1.UploadSnapshotResponse{}, nil } @@ -172,15 +187,11 @@ func (p *NodePlugin) fetchManifest(ctx context.Context, uri resources.SnapshotUR // uploadManifest uploads the manifest from local uncompressed. func (p *NodePlugin) uploadManifest(ctx context.Context, uri resources.SnapshotURI, local *os.File) error { - manifest, err := io.ReadAll(local) - if err != nil { - return fmt.Errorf("while reading %s in snapshot directory: %w", manifestFile, err) - } manifestURI, err := uri.ObjectURI(manifestFile) if err != nil { return fmt.Errorf("while addressing snapshot manifest in GCS: %w", err) } - if err := objectstorage.SendBytesToGCS(ctx, p.gcsClient, manifestURI, manifest); err != nil { + if err := p.uploader.Send(ctx, manifestURI, local); err != nil { return fmt.Errorf("while uploading snapshot manifest: %w", err) } return nil diff --git a/internal/objectstoreplugin/objectstoreplugintest/objectstoreplugintest.go b/internal/objectstoreplugin/objectstoreplugintest/objectstoreplugintest.go index c7012105fb..fa7a019d64 100644 --- a/internal/objectstoreplugin/objectstoreplugintest/objectstoreplugintest.go +++ b/internal/objectstoreplugin/objectstoreplugintest/objectstoreplugintest.go @@ -20,9 +20,9 @@ package objectstoreplugintest import ( "context" - "github.com/agent-substrate/substrate/internal/objectstore" - "github.com/agent-substrate/substrate/internal/objectstoreplugin" - objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "go.ate.dev/substrate/internal/objectstore" + "go.ate.dev/substrate/internal/objectstoreplugin" + objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/grpc" ) diff --git a/internal/objectstoreplugin/plugin_test.go b/internal/objectstoreplugin/plugin_test.go index d9b0b12121..855701e74b 100644 --- a/internal/objectstoreplugin/plugin_test.go +++ b/internal/objectstoreplugin/plugin_test.go @@ -27,9 +27,9 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/pluginsocket" - "github.com/agent-substrate/substrate/pkg/objectstorage" - objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "go.ate.dev/substrate/internal/pluginsocket" + "go.ate.dev/substrate/pkg/objectstorage" + objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/health" @@ -124,7 +124,7 @@ func serve(t *testing.T, backend *memObjects, root string) *grpc.ClientConn { if err != nil { t.Fatal(err) } - node, err := NewNodePlugin(backend, root) + node, err := NewNodePlugin(backend, root, objectstorage.DefaultUploadMemory) if err != nil { t.Fatal(err) } diff --git a/internal/objectstoreplugin/status.go b/internal/objectstoreplugin/status.go index 472229fa60..60ba1382ac 100644 --- a/internal/objectstoreplugin/status.go +++ b/internal/objectstoreplugin/status.go @@ -20,8 +20,8 @@ import ( "fmt" "log/slog" - "github.com/agent-substrate/substrate/internal/objectstore" - "github.com/agent-substrate/substrate/pkg/objectstorage" + "go.ate.dev/substrate/internal/objectstore" + "go.ate.dev/substrate/pkg/objectstorage" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) diff --git a/internal/objectstoreplugin/status_test.go b/internal/objectstoreplugin/status_test.go index 6cf45397c7..e85c55c289 100644 --- a/internal/objectstoreplugin/status_test.go +++ b/internal/objectstoreplugin/status_test.go @@ -23,7 +23,7 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/pkg/objectstorage" + "go.ate.dev/substrate/pkg/objectstorage" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) diff --git a/internal/objectstoreplugin/upload_test.go b/internal/objectstoreplugin/upload_test.go new file mode 100644 index 0000000000..cbec1a68d8 --- /dev/null +++ b/internal/objectstoreplugin/upload_test.go @@ -0,0 +1,200 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstoreplugin + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "testing/synctest" + + objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type slowSnapshotStore struct { + *memObjects + release <-chan struct{} + active atomic.Int64 + peak atomic.Int64 + manifests atomic.Int64 + fail bool +} + +func (s *slowSnapshotStore) PutObject(ctx context.Context, bucket, object string, r io.Reader) error { + if strings.HasSuffix(object, "/"+manifestFile) { + s.manifests.Add(1) + return s.memObjects.PutObject(ctx, bucket, object, r) + } + n := s.active.Add(1) + defer s.active.Add(-1) + for old := s.peak.Load(); n > old && !s.peak.CompareAndSwap(old, n); old = s.peak.Load() { + } + select { + case <-s.release: + case <-ctx.Done(): + return ctx.Err() + } + if s.fail { + return errors.New("upload failed") + } + return s.memObjects.PutObject(ctx, bucket, object, r) +} + +func snapshotUploadFiles(t *testing.T, dir string, n int) []string { + t.Helper() + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + files := []string{manifestFile} + for i := range n { + files = append(files, fmt.Sprintf("file%d", i)) + } + for _, name := range files { + if err := os.WriteFile(filepath.Join(dir, name), []byte("snapshot payload"), 0o600); err != nil { + t.Fatal(err) + } + } + return files +} + +func TestUploadBudgetSharedAcrossSnapshots(t *testing.T) { + root := t.TempDir() + const snapshots, filesPerSnapshot = 10, 8 + requests := make([]*objectstorev1.UploadSnapshotRequest, snapshots) + for i := range snapshots { + dir := filepath.Join(root, fmt.Sprintf("src%d", i)) + requests[i] = &objectstorev1.UploadSnapshotRequest{ + SnapshotUri: fmt.Sprintf("%s%d", testURI, i), LocalPath: dir, + Files: snapshotUploadFiles(t, dir, filesPerSnapshot), + } + } + synctest.Test(t, func(t *testing.T) { + gate := make(chan struct{}) + store := &slowSnapshotStore{memObjects: newMemObjects(), release: gate} + // This backend has no SDK buffers. Two compression reservations fit. + node, err := NewNodePlugin(store, root, 128<<20) + if err != nil { + t.Fatal(err) + } + done := make(chan error, snapshots) + for _, req := range requests { + go func() { _, err := node.UploadSnapshot(t.Context(), req); done <- err }() + } + synctest.Wait() + if got := store.active.Load(); got != 2 { + t.Errorf("active uploads = %d, want 2 across all requests", got) + } + if got := store.manifests.Load(); got != 0 { + t.Errorf("committed %d manifests before data finished", got) + } + close(gate) + synctest.Wait() + for range snapshots { + if err := <-done; err != nil { + t.Fatal(err) + } + } + if got := store.peak.Load(); got != 2 { + t.Errorf("peak uploads = %d, want 2", got) + } + if got := len(store.keys()); got != snapshots*(filesPerSnapshot+1) { + t.Fatalf("uploaded %d objects", got) + } + // Existing callers also send manifests in a separate RPC. Verify a + // combined request's stored data still downloads through the same reader. + dst := filepath.Join(root, "restored") + if err := os.Mkdir(dst, 0o700); err != nil { + t.Fatal(err) + } + if _, err := node.FetchSnapshot(t.Context(), &objectstorev1.FetchSnapshotRequest{ + SnapshotUri: requests[0].SnapshotUri, WritePath: dst, Files: requests[0].Files, + }); err != nil { + t.Fatal(err) + } + for _, name := range requests[0].Files { + data, err := os.ReadFile(filepath.Join(dst, name)) + if err != nil || string(data) != "snapshot payload" { + t.Fatalf("%s: %q, %v", name, data, err) + } + } + }) +} + +func TestUploadFailureDoesNotCommitManifest(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "src") + files := snapshotUploadFiles(t, dir, 3) + gate := make(chan struct{}) + close(gate) + store := &slowSnapshotStore{memObjects: newMemObjects(), release: gate, fail: true} + node, err := NewNodePlugin(store, root, 64<<20) + if err != nil { + t.Fatal(err) + } + if _, err := node.UploadSnapshot(t.Context(), &objectstorev1.UploadSnapshotRequest{ + SnapshotUri: testURI, LocalPath: dir, Files: files, + }); status.Code(err) != codes.Internal { + t.Fatalf("got %v, want upload error", err) + } + if got := store.manifests.Load(); got != 0 { + t.Fatalf("committed %d manifests after data failed", got) + } + store.fail = false + if _, err := node.UploadSnapshot(t.Context(), &objectstorev1.UploadSnapshotRequest{ + SnapshotUri: testURI, LocalPath: dir, Files: files, + }); err != nil { + t.Fatalf("retry after failure: %v", err) + } +} + +func TestUploadCanceledWaiterReturnsCanceled(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "src") + files := snapshotUploadFiles(t, dir, 1) + synctest.Test(t, func(t *testing.T) { + gate := make(chan struct{}) + store := &slowSnapshotStore{memObjects: newMemObjects(), release: gate} + node, err := NewNodePlugin(store, root, 64<<20) + if err != nil { + t.Fatal(err) + } + req := &objectstorev1.UploadSnapshotRequest{SnapshotUri: testURI, LocalPath: dir, Files: files} + first := make(chan error, 1) + go func() { _, err := node.UploadSnapshot(t.Context(), req); first <- err }() + synctest.Wait() + ctx, cancel := context.WithCancel(t.Context()) + waiting := make(chan error, 1) + go func() { _, err := node.UploadSnapshot(ctx, req); waiting <- err }() + synctest.Wait() + cancel() + synctest.Wait() + if err := <-waiting; status.Code(err) != codes.Canceled { + t.Errorf("waiting upload: %v", err) + } + close(gate) + synctest.Wait() + if err := <-first; err != nil { + t.Fatal(err) + } + }) +} diff --git a/internal/ocispec/gvisor.go b/internal/ocispec/gvisor.go index 135e44ae7e..cde27bccc8 100644 --- a/internal/ocispec/gvisor.go +++ b/internal/ocispec/gvisor.go @@ -18,9 +18,9 @@ import ( "path/filepath" "slices" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/sizing" "github.com/opencontainers/runtime-spec/specs-go" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/sizing" ) // PauseContainer is the name of the sandbox root container. The underscore diff --git a/internal/ocispec/microvm.go b/internal/ocispec/microvm.go index 33b307020c..5fcdb231c1 100644 --- a/internal/ocispec/microvm.go +++ b/internal/ocispec/microvm.go @@ -21,9 +21,9 @@ import ( "slices" "strings" - "github.com/agent-substrate/substrate/internal/imagecache" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" "github.com/opencontainers/runtime-spec/specs-go" + "go.ate.dev/substrate/internal/imagecache" + "go.ate.dev/substrate/internal/proto/ateworkerpb" ) // Sub-share names inside the micro-VM virtio-fs share. diff --git a/internal/ocispec/microvm_test.go b/internal/ocispec/microvm_test.go index c791bbf25f..905a0f792e 100644 --- a/internal/ocispec/microvm_test.go +++ b/internal/ocispec/microvm_test.go @@ -17,8 +17,8 @@ package ocispec import ( "testing" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" "github.com/opencontainers/runtime-spec/specs-go" + "go.ate.dev/substrate/internal/proto/ateworkerpb" ) // The device allowlist and CPU shares from guestResources are the proven-good diff --git a/internal/ocispec/ocispec.go b/internal/ocispec/ocispec.go index 7c40caa656..b6a7c18914 100644 --- a/internal/ocispec/ocispec.go +++ b/internal/ocispec/ocispec.go @@ -23,10 +23,10 @@ import ( "path" "path/filepath" - "github.com/agent-substrate/substrate/internal/imagecache" - "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" "github.com/opencontainers/runtime-spec/specs-go" + "go.ate.dev/substrate/internal/imagecache" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/proto/ateworkerpb" ) // specFile is the OCI spec file name within a bundle. diff --git a/internal/ocispec/ocispec_test.go b/internal/ocispec/ocispec_test.go index 4fec74c022..8b945c4476 100644 --- a/internal/ocispec/ocispec_test.go +++ b/internal/ocispec/ocispec_test.go @@ -21,10 +21,10 @@ import ( "slices" "testing" - "github.com/agent-substrate/substrate/internal/imagecache" - "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" "github.com/opencontainers/runtime-spec/specs-go" + "go.ate.dev/substrate/internal/imagecache" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/proto/ateworkerpb" ) // mountFor returns the spec's mount at destination dest, or fails. diff --git a/internal/ocispec/parity_test.go b/internal/ocispec/parity_test.go index 809c67ea03..e448625b07 100644 --- a/internal/ocispec/parity_test.go +++ b/internal/ocispec/parity_test.go @@ -20,9 +20,9 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/sizing" "github.com/opencontainers/runtime-spec/specs-go" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/sizing" ) const testActorUID = "actor_uid" diff --git a/internal/oidcjwt/oidcjwt.go b/internal/oidcjwt/oidcjwt.go index d2e87982ad..fccd1788ea 100644 --- a/internal/oidcjwt/oidcjwt.go +++ b/internal/oidcjwt/oidcjwt.go @@ -23,16 +23,20 @@ import ( "crypto/rsa" "encoding/base64" "encoding/json" + "errors" "fmt" "hash" "io" "log/slog" "math/big" "net/http" + "os" "slices" "strings" "sync" "time" + + "go.ate.dev/substrate/internal/actoridjwt" ) // KeyAndID wraps a crypto.PublicKey and its JWK key ID. @@ -60,6 +64,12 @@ type parseClaims struct { // Kubernetes bound token claims. BoundClaims parseBoundClaims `json:"kubernetes.io,omitempty"` + // Substrate's own claims, under the claim name for the token's kind. Kept + // raw until the signature is verified, so an unverified payload shapes no + // parsing beyond this struct. + Actor json.RawMessage `json:"actor.ate.dev,omitempty"` + WorkerForActor json.RawMessage `json:"worker-for-actor.ate.dev,omitempty"` + // Kubernetes legacy token claims. LegacyNamespace string `json:"kubernetes.io/serviceaccount/namespace,omitempty"` LegacySecretName string `json:"kubernetes.io/serviceaccount/secret.name,omitempty"` @@ -82,7 +92,7 @@ type parseBoundObjectReference struct { } // Claims contains standard JWT claims and, when present, Kubernetes -// ServiceAccount claims. +// ServiceAccount claims and Substrate's own claims. type Claims struct { // Claims from RFC7519 Issuer string @@ -94,6 +104,21 @@ type Claims struct { JTI string KubernetesClaims + SubstrateClaims +} + +// SubstrateClaims are the claims the actor-id JWT issuer adds to the tokens it +// mints. Kind says which claim the token carried, KindActor for the actor's +// own token or KindWorkerForActor for a Substrate component acting on the +// actor's behalf, and Substrate holds it. A token carrying neither has +// KindUnspecified and a nil Substrate. A token carrying both is refused. +// +// Only a Verifier built with WithSubstrateClaims fills these in. Any issuer can +// put the claim names in a token, so a verifier for another issuer leaves them +// alone, whatever they hold. +type SubstrateClaims struct { + Kind actoridjwt.Kind + Substrate *actoridjwt.WireSubstrateClaims } // KubernetesClaims contains claims added to Kubernetes ServiceAccount tokens. @@ -119,13 +144,38 @@ var ( defaultHTTPClient = &http.Client{Timeout: 10 * time.Second} ) +// KeySource supplies the public keys a Verifier checks signatures with. The +// Verifier calls Keys on first use, every jwksRefreshInterval, and once per +// jwksRefreshRetryInterval when a token names a key ID it does not have, so a +// source may read its backing store on every call. +type KeySource interface { + Keys(ctx context.Context) ([]*KeyAndID, error) +} + +type httpKeySource struct { + issuer string + client *http.Client +} + +func (s httpKeySource) Keys(ctx context.Context) ([]*KeyAndID, error) { + return discoverKeysForIssuer(ctx, s.client, s.issuer) +} + +// HTTPKeySource fetches the issuer's keys through OpenID Connect Discovery: +// /.well-known/openid-configuration, then its jwks_uri. +func HTTPKeySource(issuer string, client *http.Client) KeySource { + return httpKeySource{issuer: issuer, client: client} +} + // Verifier verifies JWTs from one trusted OIDC issuer and caches its signing // keys. Keys are refreshed periodically and, at a bounded rate, when a token // names an unknown key ID. type Verifier struct { - issuer string - audiences []string - httpClient *http.Client + issuer string + audiences []string + source KeySource + // substrate says whether Verify decodes Substrate's claims. + substrate bool mu sync.RWMutex keys []*KeyAndID @@ -134,10 +184,50 @@ type Verifier struct { lastUnknownKeyRefresh time.Time } +type fileKeySource struct { + path string +} + +func (s fileKeySource) Keys(ctx context.Context) ([]*KeyAndID, error) { + b, err := os.ReadFile(s.path) + if err != nil { + return nil, fmt.Errorf("while reading JWKS file: %w", err) + } + var set jwkSetT + if err := json.Unmarshal(b, &set); err != nil { + return nil, fmt.Errorf("while parsing JWKS file %s: %w", s.path, err) + } + return keysFromJWKSet(ctx, set, "file "+s.path) +} + +// FileKeySource reads a JWK set from a file, for a verifier that is handed the +// issuer's public keys directly rather than fetching them over discovery. The +// file is re-read on every call, so a rotated set is picked up on the +// verifier's next refresh. For that to work the file must be replaced +// atomically, as a projected volume mounted without subPath is: an in-place +// rewrite can be read half-written, and a subPath mount never sees the +// rotation. The file holds public keys only. +func FileKeySource(path string) KeySource { + return fileKeySource{path: path} +} + +// Option configures a Verifier. +type Option func(*Verifier) + +// WithSubstrateClaims makes Verify decode Substrate's claims into +// Claims.SubstrateClaims and refuse a token that carries both or a malformed +// one. It is for the verifier of the actor-id JWT issuer, whose tokens are the +// only ones those claims mean anything on. +func WithSubstrateClaims() Option { return func(v *Verifier) { v.substrate = true } } + // NewVerifier returns a verifier for issuer. A token is accepted when at least // one of its audiences matches audiences. -func NewVerifier(issuer string, audiences []string, httpClient *http.Client) *Verifier { - return &Verifier{issuer: issuer, audiences: slices.Clone(audiences), httpClient: httpClient} +func NewVerifier(issuer string, audiences []string, source KeySource, opts ...Option) *Verifier { + v := &Verifier{issuer: issuer, audiences: slices.Clone(audiences), source: source} + for _, o := range opts { + o(v) + } + return v } // Verify verifies and extracts claims from a JWT. @@ -246,14 +336,22 @@ func (v *Verifier) Verify(ctx context.Context, jwt string, now time.Time) (*Clai return nil, fmt.Errorf("jwt claims to have been issued in the future") } + var substrate SubstrateClaims + if v.substrate { + if substrate, err = decodeSubstrateClaims(rawClaims); err != nil { + return nil, err + } + } + return &Claims{ - Issuer: rawClaims.Issuer, - Audiences: audiences, - Subject: rawClaims.Subject, - Expiration: expiration, - NotBefore: notBefore, - IssuedAt: issuedAt, - JTI: rawClaims.JTI, + Issuer: rawClaims.Issuer, + Audiences: audiences, + Subject: rawClaims.Subject, + Expiration: expiration, + NotBefore: notBefore, + IssuedAt: issuedAt, + JTI: rawClaims.JTI, + SubstrateClaims: substrate, KubernetesClaims: KubernetesClaims{ Namespace: rawClaims.BoundClaims.Namespace, ServiceAccountName: rawClaims.BoundClaims.ServiceAccount.Name, @@ -269,6 +367,41 @@ func (v *Verifier) Verify(ctx context.Context, jwt string, now time.Time) (*Clai }, nil } +// decodeSubstrateClaims parses Substrate's claims from a verified payload and +// applies actoridjwt's rule that a token carries one of them or neither. +func decodeSubstrateClaims(raw parseClaims) (SubstrateClaims, error) { + wire := actoridjwt.WireClaims{} + var err error + if wire.Actor, err = decodeSubstrateClaim("actor.ate.dev", raw.Actor); err != nil { + return SubstrateClaims{}, err + } + if wire.WorkerForActor, err = decodeSubstrateClaim("worker-for-actor.ate.dev", raw.WorkerForActor); err != nil { + return SubstrateClaims{}, err + } + if wire.Actor == nil && wire.WorkerForActor == nil { + return SubstrateClaims{}, nil + } + kind, claims, ok := wire.Substrate() + if !ok { + return SubstrateClaims{}, errors.New("token carries both actor.ate.dev and worker-for-actor.ate.dev") + } + return SubstrateClaims{Kind: kind, Substrate: claims}, nil +} + +// decodeSubstrateClaim parses one of Substrate's claims. An absent or null +// claim is nil. A present claim that is not an object is an error, since a +// trusted issuer never emits one. +func decodeSubstrateClaim(name string, raw json.RawMessage) (*actoridjwt.WireSubstrateClaims, error) { + if len(raw) == 0 || string(raw) == "null" { + return nil, nil + } + var claim actoridjwt.WireSubstrateClaims + if err := json.Unmarshal(raw, &claim); err != nil { + return nil, fmt.Errorf("while parsing the %s claim: %w", name, err) + } + return &claim, nil +} + func (v *Verifier) key(ctx context.Context, keyID string, now time.Time) (crypto.PublicKey, error) { v.mu.RLock() key := findKey(v.keys, keyID) @@ -308,14 +441,14 @@ func (v *Verifier) key(ctx context.Context, keyID string, now time.Time) (crypto if key == nil && len(v.keys) > 0 { v.lastUnknownKeyRefresh = now } - keys, err := discoverKeysForIssuer(ctx, v.httpClient, v.issuer) + keys, err := v.source.Keys(ctx) if err != nil { v.lastFailedRefresh = now if key != nil { - slog.WarnContext(ctx, "Could not refresh OIDC signing keys; using cached key", slog.String("issuer", v.issuer), slog.Any("err", err)) + slog.WarnContext(ctx, "Could not refresh JWT signing keys; using cached key", slog.String("issuer", v.issuer), slog.Any("err", err)) return key.PublicKey, nil } - return nil, fmt.Errorf("while discovering keys from issuer: %w", err) + return nil, fmt.Errorf("while loading signing keys for issuer: %w", err) } v.keys = keys v.lastRefresh = now @@ -480,17 +613,24 @@ func discoverKeysForIssuer(ctx context.Context, httpClient *http.Client, issuer slog.InfoContext(ctx, "Fetched JWK set", slog.Any("jwkSet", jwkSet)) + return keysFromJWKSet(ctx, jwkSet, "issuer "+issuer) +} + +// keysFromJWKSet converts a JWK set into verification keys. origin names where +// the set came from, for errors and logs. +func keysFromJWKSet(ctx context.Context, jwkSet jwkSetT, origin string) ([]*KeyAndID, error) { var ret []*KeyAndID skipped := 0 for _, jwk := range jwkSet.Keys { key, err := parseJWK(jwk) if err != nil { - // Skip an unusable key instead of failing the whole issuer; it's safe because - // keys are selected by kid and the signature is still verified, so a skipped - // key can't be abused. Debug, not Warn: unsupported key types are a normal - // issuer configuration and periodic refreshes would otherwise spam warnings. - slog.DebugContext(ctx, "Skipping unusable JWK from issuer", - slog.String("kid", jwk.KeyID), slog.String("kty", jwk.KeyType), slog.Any("err", err)) + // Skip an unusable key instead of failing the whole set. That is safe + // because keys are selected by kid and the signature is still verified, + // so a skipped key can't be abused. Debug, not Warn: unsupported key + // types are a normal issuer configuration and periodic refreshes would + // otherwise spam warnings. + slog.DebugContext(ctx, "Skipping unusable JWK", + slog.String("origin", origin), slog.String("kid", jwk.KeyID), slog.String("kty", jwk.KeyType), slog.Any("err", err)) skipped++ continue } @@ -501,14 +641,14 @@ func discoverKeysForIssuer(ctx context.Context, httpClient *http.Client, issuer // that fails later as a vaguer "unknown key ID". if len(ret) == 0 { if len(jwkSet.Keys) == 0 { - return nil, fmt.Errorf("issuer %q published an empty JWKS", issuer) + return nil, fmt.Errorf("%s published an empty JWKS", origin) } - return nil, fmt.Errorf("no usable keys in JWKS for issuer %q (%d skipped)", issuer, skipped) + return nil, fmt.Errorf("no usable keys in JWKS from %s (%d skipped)", origin, skipped) } if skipped > 0 { - slog.DebugContext(ctx, "Skipped unusable JWKs from issuer", - slog.String("issuer", issuer), slog.Int("skipped", skipped), slog.Int("usable", len(ret))) + slog.DebugContext(ctx, "Skipped unusable JWKs", + slog.String("origin", origin), slog.Int("skipped", skipped), slog.Int("usable", len(ret))) } return ret, nil diff --git a/internal/oidcjwt/oidcjwt_test.go b/internal/oidcjwt/oidcjwt_test.go index 28e97cf536..a7b9cf0684 100644 --- a/internal/oidcjwt/oidcjwt_test.go +++ b/internal/oidcjwt/oidcjwt_test.go @@ -28,11 +28,15 @@ import ( "math/big" "net/http" "net/http/httptest" + "os" + "path/filepath" "strings" "sync" "sync/atomic" "testing" "time" + + "go.ate.dev/substrate/internal/actoridjwt" ) const testAudience = "ate-api" @@ -76,7 +80,7 @@ func TestVerifierRetriesInitialDiscoveryFailure(t *testing.T) { key := testRSAKey(t) ti.addRSA("key", &key.PublicKey) ti.jwksFailures.Store(1) - verifier := NewVerifier(ti.issuer(), []string{testAudience}, ti.server.Client()) + verifier := NewVerifier(ti.issuer(), []string{testAudience}, HTTPKeySource(ti.issuer(), ti.server.Client())) token := mintJWT(t, "RS256", "key", key, validClaims(ti.issuer())) now := time.Now() @@ -98,7 +102,7 @@ func TestVerifierUsesCachedKeyWhenRefreshFails(t *testing.T) { ti := newTestIssuer(t) key := testRSAKey(t) ti.addRSA("key", &key.PublicKey) - verifier := NewVerifier(ti.issuer(), []string{testAudience}, ti.server.Client()) + verifier := NewVerifier(ti.issuer(), []string{testAudience}, HTTPKeySource(ti.issuer(), ti.server.Client())) token := mintJWT(t, "RS256", "key", key, validClaims(ti.issuer())) now := time.Now() @@ -128,7 +132,7 @@ func TestVerifierReturnsKubernetesClaims(t *testing.T) { } token := mintJWT(t, "RS256", "key", key, claims) - got, err := NewVerifier(ti.issuer(), []string{testAudience}, ti.server.Client()).Verify(t.Context(), token, time.Now()) + got, err := NewVerifier(ti.issuer(), []string{testAudience}, HTTPKeySource(ti.issuer(), ti.server.Client())).Verify(t.Context(), token, time.Now()) if err != nil { t.Fatal(err) } @@ -141,7 +145,7 @@ func TestVerifierCachesAndRefreshesKeys(t *testing.T) { ti := newTestIssuer(t) key1 := testRSAKey(t) ti.addRSA("key-1", &key1.PublicKey) - verifier := NewVerifier(ti.issuer(), []string{"other", testAudience}, ti.server.Client()) + verifier := NewVerifier(ti.issuer(), []string{"other", testAudience}, HTTPKeySource(ti.issuer(), ti.server.Client())) now := time.Now() token1 := mintJWT(t, "RS256", "key-1", key1, validClaims(ti.issuer())) @@ -359,7 +363,7 @@ func TestVerifyECDSA(t *testing.T) { ti.addEC(t, "ec-1", tc.crv, &key.PublicKey) tok := mintJWT(t, tc.alg, "ec-1", key, validClaims(ti.issuer())) - if _, err := NewVerifier(ti.issuer(), []string{testAudience}, ti.server.Client()).Verify(context.Background(), tok, time.Now()); err != nil { + if _, err := NewVerifier(ti.issuer(), []string{testAudience}, HTTPKeySource(ti.issuer(), ti.server.Client())).Verify(context.Background(), tok, time.Now()); err != nil { t.Fatalf("Verify(%s) = %v, want nil", tc.alg, err) } }) @@ -379,7 +383,7 @@ func TestVerifyRejectsECKeyForRSAlg(t *testing.T) { // RS256 header pointing at the EC key; the RSA signing key is irrelevant because // the key-type check fails before signature verification. tok := mintJWT(t, "RS256", "ec-1", testRSAKey(t), validClaims(ti.issuer())) - if _, err := NewVerifier(ti.issuer(), []string{testAudience}, ti.server.Client()).Verify(context.Background(), tok, time.Now()); err == nil { + if _, err := NewVerifier(ti.issuer(), []string{testAudience}, HTTPKeySource(ti.issuer(), ti.server.Client())).Verify(context.Background(), tok, time.Now()); err == nil { t.Fatal("Verify accepted an RS256 token whose kid names an EC key") } } @@ -398,7 +402,7 @@ func TestVerifyMixedJWKS(t *testing.T) { ti.addEC(t, "ec-1", "P-256", &ecKey.PublicKey) tok := mintJWT(t, "RS256", "rsa-1", rsaKey, validClaims(ti.issuer())) - if _, err := NewVerifier(ti.issuer(), []string{testAudience}, ti.server.Client()).Verify(context.Background(), tok, time.Now()); err != nil { + if _, err := NewVerifier(ti.issuer(), []string{testAudience}, HTTPKeySource(ti.issuer(), ti.server.Client())).Verify(context.Background(), tok, time.Now()); err != nil { t.Fatalf("Verify with a mixed RSA+EC JWKS = %v, want nil", err) } } @@ -417,12 +421,12 @@ func TestVerifyUnusableJWKSKeySkipped(t *testing.T) { }) good := mintJWT(t, "RS256", "rsa-1", rsaKey, validClaims(ti.issuer())) - if _, err := NewVerifier(ti.issuer(), []string{testAudience}, ti.server.Client()).Verify(context.Background(), good, time.Now()); err != nil { + if _, err := NewVerifier(ti.issuer(), []string{testAudience}, HTTPKeySource(ti.issuer(), ti.server.Client())).Verify(context.Background(), good, time.Now()); err != nil { t.Fatalf("Verify with an unusable key in the JWKS = %v, want nil (bad key should be skipped)", err) } referencesSkipped := mintJWT(t, "RS256", "ec-bad", rsaKey, validClaims(ti.issuer())) - if _, err := NewVerifier(ti.issuer(), []string{testAudience}, ti.server.Client()).Verify(context.Background(), referencesSkipped, time.Now()); err == nil { + if _, err := NewVerifier(ti.issuer(), []string{testAudience}, HTTPKeySource(ti.issuer(), ti.server.Client())).Verify(context.Background(), referencesSkipped, time.Now()); err == nil { t.Fatal("Verify accepted a token whose kid names a key that was skipped") } } @@ -489,3 +493,173 @@ func TestEllipticCurveForJWK(t *testing.T) { t.Error("ellipticCurveForJWK(P-192) = nil, want error") } } + +// writeJWKSFile writes the issuer's current JWK set to a file and returns the +// path, standing in for the public key file a signer publishes next to its +// pool. +func writeJWKSFile(t *testing.T, ti *testIssuer) string { + t.Helper() + b, err := json.Marshal(ti.jwks) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "jwks.json") + if err := os.WriteFile(path, b, 0o600); err != nil { + t.Fatal(err) + } + return path +} + +func TestFileKeySource(t *testing.T) { + ti := newTestIssuer(t) + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + ti.addEC(t, "ec-1", "P-256", &key.PublicKey) + path := writeJWKSFile(t, ti) + + v := NewVerifier(ti.issuer(), []string{testAudience}, FileKeySource(path)) + if _, err := v.Verify(context.Background(), mintJWT(t, "ES256", "ec-1", key, validClaims(ti.issuer())), time.Now()); err != nil { + t.Fatalf("Verify = %v, want nil", err) + } + if ti.jwksRequests.Load() != 0 { + t.Errorf("the file source fetched the JWKS over HTTP %d times", ti.jwksRequests.Load()) + } + + // Rotation: a key added to the file is found on the unknown-kid refresh. + next, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + ti.addEC(t, "ec-2", "P-256", &next.PublicKey) + b, _ := json.Marshal(ti.jwks) + if err := os.WriteFile(path, b, 0o600); err != nil { + t.Fatal(err) + } + if _, err := v.Verify(context.Background(), mintJWT(t, "ES256", "ec-2", next, validClaims(ti.issuer())), time.Now()); err != nil { + t.Fatalf("Verify after rotation = %v, want nil", err) + } +} + +func TestFileKeySourceErrors(t *testing.T) { + dir := t.TempDir() + empty := filepath.Join(dir, "empty.json") + if err := os.WriteFile(empty, []byte(`{"keys":[]}`), 0o600); err != nil { + t.Fatal(err) + } + garbage := filepath.Join(dir, "garbage.json") + if err := os.WriteFile(garbage, []byte(`not json`), 0o600); err != nil { + t.Fatal(err) + } + for name, path := range map[string]string{ + "missing file": filepath.Join(dir, "absent.json"), + "empty set": empty, + "not JSON": garbage, + } { + t.Run(name, func(t *testing.T) { + if _, err := FileKeySource(path).Keys(context.Background()); err == nil { + t.Fatal("Keys succeeded, want error") + } + }) + } +} + +func TestVerifySubstrateClaims(t *testing.T) { + ti := newTestIssuer(t) + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + ti.addEC(t, "ec-1", "P-256", &key.PublicKey) + v := NewVerifier(ti.issuer(), []string{testAudience}, HTTPKeySource(ti.issuer(), ti.server.Client()), WithSubstrateClaims()) + + withClaim := func(name string, value any) map[string]any { + c := validClaims(ti.issuer()) + c[name] = value + return c + } + for _, tc := range []struct { + name string + claims map[string]any + wantErr string + check func(*Claims) bool + }{ + {"absent", validClaims(ti.issuer()), "", func(c *Claims) bool { return c.Kind == actoridjwt.KindUnspecified && c.Substrate == nil }}, + {"actor", withClaim("actor.ate.dev", map[string]string{"atespace": "a", "actorName": "b", "actorUID": "u"}), "", + func(c *Claims) bool { + return c.Kind == actoridjwt.KindActor && c.Substrate != nil && c.Substrate.ActorUID == "u" + }}, + {"worker-for-actor", withClaim("worker-for-actor.ate.dev", map[string]string{"atespace": "a"}), "", + func(c *Claims) bool { + return c.Kind == actoridjwt.KindWorkerForActor && c.Substrate != nil && c.Substrate.Atespace == "a" + }}, + {"not an object", withClaim("actor.ate.dev", "junk"), "actor.ate.dev", nil}, + {"null", withClaim("actor.ate.dev", nil), "", func(c *Claims) bool { return c.Kind == actoridjwt.KindUnspecified && c.Substrate == nil }}, + {"both", func() map[string]any { + c := withClaim("actor.ate.dev", map[string]string{"atespace": "a"}) + c["worker-for-actor.ate.dev"] = map[string]string{"atespace": "a"} + return c + }(), "both", nil}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := v.Verify(context.Background(), mintJWT(t, "ES256", "ec-1", key, tc.claims), time.Now()) + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("err = %v, want it to mention %q", err, tc.wantErr) + } + return + } + if err != nil { + t.Fatalf("Verify = %v", err) + } + if !tc.check(got) { + t.Errorf("claims = %+v", got.SubstrateClaims) + } + }) + } +} + +func TestVerifyIgnoresSubstrateClaimsOnBadSignature(t *testing.T) { + // A malformed Substrate claim on a token with a bad signature must fail on + // the signature, proving the claim is not parsed before verification. + ti := newTestIssuer(t) + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + ti.addEC(t, "ec-1", "P-256", &key.PublicKey) + stranger, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + claims := validClaims(ti.issuer()) + claims["actor.ate.dev"] = "junk" + v := NewVerifier(ti.issuer(), []string{testAudience}, HTTPKeySource(ti.issuer(), ti.server.Client()), WithSubstrateClaims()) + _, err = v.Verify(context.Background(), mintJWT(t, "ES256", "ec-1", stranger, claims), time.Now()) + if err == nil || !strings.Contains(err.Error(), "signature") || strings.Contains(err.Error(), "actor.ate.dev") { + t.Fatalf("err = %v, want a signature error that does not mention the claim", err) + } +} + +func TestVerifyLeavesSubstrateClaimsAloneByDefault(t *testing.T) { + // A verifier for another issuer must not fail on, or expose, whatever that + // issuer put under Substrate's claim names. + ti := newTestIssuer(t) + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + ti.addEC(t, "ec-1", "P-256", &key.PublicKey) + v := NewVerifier(ti.issuer(), []string{testAudience}, HTTPKeySource(ti.issuer(), ti.server.Client())) + claims := validClaims(ti.issuer()) + claims["actor.ate.dev"] = "junk" + claims["worker-for-actor.ate.dev"] = map[string]string{"atespace": "a"} + got, err := v.Verify(context.Background(), mintJWT(t, "ES256", "ec-1", key, claims), time.Now()) + if err != nil { + t.Fatalf("Verify = %v, want nil", err) + } + if got.Kind != actoridjwt.KindUnspecified || got.Substrate != nil { + t.Errorf("Substrate claims = %+v, want none", got.SubstrateClaims) + } +} diff --git a/internal/otlprelay/e2e_test.go b/internal/otlprelay/e2e_test.go index b355b6c1be..da75f3b3ee 100644 --- a/internal/otlprelay/e2e_test.go +++ b/internal/otlprelay/e2e_test.go @@ -20,7 +20,7 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/serverboot" + "go.ate.dev/substrate/internal/serverboot" ) // TestEndToEndThroughServerboot exercises the whole path a worker span actually diff --git a/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/clientpool.go b/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/clientpool.go index 0c965a13b7..efc383bf1c 100644 --- a/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/clientpool.go +++ b/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/clientpool.go @@ -18,7 +18,7 @@ import ( "errors" "sync" - "github.com/agent-substrate/substrate/internal/plugins/gcp-secret-manager/internal/provider" + "go.ate.dev/substrate/internal/plugins/gcp-secret-manager/internal/provider" ) // secretManagerClient is a closable SecretAccessor. diff --git a/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/imports_test.go b/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/imports_test.go index 1913be81fc..60b57818b7 100644 --- a/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/imports_test.go +++ b/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/imports_test.go @@ -26,7 +26,7 @@ import ( ) // substrateModule is the module this one is hosted in until it moves out. -const substrateModule = "github.com/agent-substrate/substrate" +const substrateModule = "go.ate.dev/substrate" // This module moves to a repository of its own by being copied, so it may // import only substrate's public packages. Go's internal rule goes by import diff --git a/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/main.go b/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/main.go index 5700ac64db..77304c459a 100644 --- a/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/main.go +++ b/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/main.go @@ -37,10 +37,10 @@ import ( "google.golang.org/api/option" "google.golang.org/grpc" - "github.com/agent-substrate/substrate/pkg/proto/credproviderpb" + "go.ate.dev/substrate/pkg/proto/credproviderpb" - "github.com/agent-substrate/substrate/internal/plugins/gcp-secret-manager/internal/mtls" - "github.com/agent-substrate/substrate/internal/plugins/gcp-secret-manager/internal/provider" + "go.ate.dev/substrate/internal/plugins/gcp-secret-manager/internal/mtls" + "go.ate.dev/substrate/internal/plugins/gcp-secret-manager/internal/provider" ) var ( diff --git a/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/main_test.go b/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/main_test.go index cec540dcbb..70102a1aa7 100644 --- a/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/main_test.go +++ b/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/main_test.go @@ -32,7 +32,7 @@ import ( "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/test/bufconn" - "github.com/agent-substrate/substrate/pkg/proto/credproviderpb" + "go.ate.dev/substrate/pkg/proto/credproviderpb" ) // /healthz passes regardless of readiness; /readyz follows the ready flag. diff --git a/internal/plugins/gcp-secret-manager/config/deployment.yaml b/internal/plugins/gcp-secret-manager/config/deployment.yaml index 689521f19d..66838d737a 100644 --- a/internal/plugins/gcp-secret-manager/config/deployment.yaml +++ b/internal/plugins/gcp-secret-manager/config/deployment.yaml @@ -36,7 +36,7 @@ spec: runAsNonRoot: true containers: - name: gsm-credential-provider - image: ko://github.com/agent-substrate/substrate/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider + image: ko://go.ate.dev/substrate/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider args: - "--listen-address=:50051" - "--health-address=:9090" diff --git a/internal/plugins/gcp-secret-manager/go.mod b/internal/plugins/gcp-secret-manager/go.mod index feb77e0d4b..b45e90bcf4 100644 --- a/internal/plugins/gcp-secret-manager/go.mod +++ b/internal/plugins/gcp-secret-manager/go.mod @@ -1,11 +1,11 @@ -module github.com/agent-substrate/substrate/internal/plugins/gcp-secret-manager +module go.ate.dev/substrate/internal/plugins/gcp-secret-manager go 1.27.0 require ( cloud.google.com/go/secretmanager v1.16.0 - github.com/agent-substrate/substrate v0.2.0 github.com/googleapis/gax-go/v2 v2.21.0 + go.ate.dev/substrate v0.2.0 google.golang.org/api v0.274.0 google.golang.org/grpc v1.83.2 sigs.k8s.io/yaml v1.6.0 @@ -42,4 +42,4 @@ require ( google.golang.org/protobuf v1.36.12 // indirect ) -replace github.com/agent-substrate/substrate => ../../.. +replace go.ate.dev/substrate => ../../.. diff --git a/internal/plugins/gcp-secret-manager/internal/mtls/mtls.go b/internal/plugins/gcp-secret-manager/internal/mtls/mtls.go index a8c2265577..e146262381 100644 --- a/internal/plugins/gcp-secret-manager/internal/mtls/mtls.go +++ b/internal/plugins/gcp-secret-manager/internal/mtls/mtls.go @@ -25,7 +25,7 @@ import ( "google.golang.org/grpc/credentials" - "github.com/agent-substrate/substrate/internal/plugins/gcp-secret-manager/internal/credbundle" + "go.ate.dev/substrate/internal/plugins/gcp-secret-manager/internal/credbundle" ) // Config names the files and caller identity ServerCredentials serves with. diff --git a/internal/plugins/gcp-secret-manager/internal/provider/provider.go b/internal/plugins/gcp-secret-manager/internal/provider/provider.go index 86cccfd273..61fd97ac70 100644 --- a/internal/plugins/gcp-secret-manager/internal/provider/provider.go +++ b/internal/plugins/gcp-secret-manager/internal/provider/provider.go @@ -34,7 +34,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/agent-substrate/substrate/pkg/proto/credproviderpb" + "go.ate.dev/substrate/pkg/proto/credproviderpb" ) // providerName is the ate-secret:// URI host this backend serves. diff --git a/internal/plugins/gcp-secret-manager/internal/provider/provider_test.go b/internal/plugins/gcp-secret-manager/internal/provider/provider_test.go index 6f191d7f5c..4488769214 100644 --- a/internal/plugins/gcp-secret-manager/internal/provider/provider_test.go +++ b/internal/plugins/gcp-secret-manager/internal/provider/provider_test.go @@ -28,7 +28,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/agent-substrate/substrate/pkg/proto/credproviderpb" + "go.ate.dev/substrate/pkg/proto/credproviderpb" ) // fakeAccessor is a fake Secret Manager client serving payloads by resource diff --git a/internal/pluginsocket/calls_test.go b/internal/pluginsocket/calls_test.go index d9ae9cc717..f123a79ad4 100644 --- a/internal/pluginsocket/calls_test.go +++ b/internal/pluginsocket/calls_test.go @@ -18,7 +18,7 @@ import ( "errors" "testing" - "github.com/agent-substrate/substrate/internal/apierror" + "go.ate.dev/substrate/internal/apierror" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) diff --git a/internal/pluginsocket/pluginsocket.go b/internal/pluginsocket/pluginsocket.go index e6fedc41da..6d853f1356 100644 --- a/internal/pluginsocket/pluginsocket.go +++ b/internal/pluginsocket/pluginsocket.go @@ -27,7 +27,7 @@ import ( "path/filepath" "time" - "github.com/agent-substrate/substrate/internal/apierror" + "go.ate.dev/substrate/internal/apierror" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "google.golang.org/grpc/backoff" diff --git a/internal/pluginsocket/pluginsocket_test.go b/internal/pluginsocket/pluginsocket_test.go index b26e83a490..6b2d085045 100644 --- a/internal/pluginsocket/pluginsocket_test.go +++ b/internal/pluginsocket/pluginsocket_test.go @@ -22,7 +22,7 @@ import ( "testing" "time" - objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/connectivity" diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index b05826b2a6..4bef1e0548 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -366,12 +366,14 @@ type VersionedSandboxCompat struct { // +k8s:maxLength=64 SchemaVersion string `protobuf:"bytes,1,opt,name=schema_version,json=schemaVersion,proto3" json:"schema_version,omitempty"` // Key-value compatibility attributes defined by this schema version, covering - // both runtime assets (e.g. "gvisor_asset_hash") and effective guest hardware - // features after masking (e.g. "architecture", "cpu_features"). + // both runtime assets (e.g. "sandbox_runtime_hash") and effective guest + // hardware features after masking (e.g. "architecture", "cpu_features"). // Matched against a snapshot under the same schema_version: every attribute - // on the snapshot must be present on the worker with an equal value; keys - // only the worker has are ignored. Bump schema_version when a key's meaning - // changes or older snapshots must no longer match. + // on the snapshot must be present on the worker with an equal value (except + // "cpu_features", where the snapshot's hexadecimal bitmask must be a bitwise + // subset of the worker's); keys only the worker has are ignored. Bump + // schema_version when a key's meaning changes or older snapshots must no + // longer match. // Order as sent does not matter: ate-api-server sorts attributes by key // before recording them. // @@ -431,7 +433,7 @@ func (x *VersionedSandboxCompat) GetAttributes() []*AttributeEntry { // AttributeEntry is a single key-value compatibility attribute. type AttributeEntry struct { state protoimpl.MessageState `protogen:"open.v1"` - // Attribute name (e.g. "architecture", "cpu_features", "gvisor_asset_hash"). + // Attribute name (e.g. "architecture", "cpu_features", "sandbox_runtime_hash"). // // +k8s:required // +k8s:maxLength=128 @@ -3291,7 +3293,7 @@ func (x *SnapshotReport) GetDurableCompleteTime() *timestamppb.Timestamp { } // ActorSnapshotState is what atelet holds for one actor on this node. -// Returned by Checkpoint, PromoteSnapshot. +// Returned by Checkpoint, PromoteSnapshot, DetachActor. type ActorSnapshotState struct { state protoimpl.MessageState `protogen:"open.v1"` // The snapshot atelet restores the actor from locally. Unset when the @@ -3980,6 +3982,120 @@ func (x *PromoteSnapshotResponse) GetState() *ActorSnapshotState { return nil } +type DetachActorRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Golden actors are accepted: an IMMEDIATE checkpoint keeps a local copy. + // + // +k8s:required + // +k8s:format=k8s-short-name + Atespace string `protobuf:"bytes,1,opt,name=atespace,proto3" json:"atespace,omitempty"` + // +k8s:required + // +k8s:format=k8s-short-name + ActorName string `protobuf:"bytes,2,opt,name=actor_name,json=actorName,proto3" json:"actor_name,omitempty"` + // +k8s:required + // +k8s:format=k8s-uuid + ActorUid string `protobuf:"bytes,3,opt,name=actor_uid,json=actorUid,proto3" json:"actor_uid,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DetachActorRequest) Reset() { + *x = DetachActorRequest{} + mi := &file_atelet_proto_msgTypes[56] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DetachActorRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DetachActorRequest) ProtoMessage() {} + +func (x *DetachActorRequest) ProtoReflect() protoreflect.Message { + mi := &file_atelet_proto_msgTypes[56] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DetachActorRequest.ProtoReflect.Descriptor instead. +func (*DetachActorRequest) Descriptor() ([]byte, []int) { + return file_atelet_proto_rawDescGZIP(), []int{56} +} + +func (x *DetachActorRequest) GetAtespace() string { + if x != nil { + return x.Atespace + } + return "" +} + +func (x *DetachActorRequest) GetActorName() string { + if x != nil { + return x.ActorName + } + return "" +} + +func (x *DetachActorRequest) GetActorUid() string { + if x != nil { + return x.ActorUid + } + return "" +} + +type DetachActorResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The actor's state once detached: latest_local is unset, and latest_durable + // is its latest snapshot, if atelet knows one. + State *ActorSnapshotState `protobuf:"bytes,1,opt,name=state,proto3" json:"state,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DetachActorResponse) Reset() { + *x = DetachActorResponse{} + mi := &file_atelet_proto_msgTypes[57] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DetachActorResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DetachActorResponse) ProtoMessage() {} + +func (x *DetachActorResponse) ProtoReflect() protoreflect.Message { + mi := &file_atelet_proto_msgTypes[57] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DetachActorResponse.ProtoReflect.Descriptor instead. +func (*DetachActorResponse) Descriptor() ([]byte, []int) { + return file_atelet_proto_rawDescGZIP(), []int{57} +} + +func (x *DetachActorResponse) GetState() *ActorSnapshotState { + if x != nil { + return x.State + } + return nil +} + var File_atelet_proto protoreflect.FileDescriptor const file_atelet_proto_rawDesc = "" + @@ -4221,6 +4337,13 @@ const file_atelet_proto_rawDesc = "" + "\x13actor_template_name\x18\x05 \x01(\tR\x11actorTemplateName\x12,\n" + "\bsnapshot\x18\x06 \x01(\v2\x10.atelet.SnapshotR\bsnapshot\"K\n" + "\x17PromoteSnapshotResponse\x120\n" + + "\x05state\x18\x01 \x01(\v2\x1a.atelet.ActorSnapshotStateR\x05state\"l\n" + + "\x12DetachActorRequest\x12\x1a\n" + + "\batespace\x18\x01 \x01(\tR\batespace\x12\x1d\n" + + "\n" + + "actor_name\x18\x02 \x01(\tR\tactorName\x12\x1b\n" + + "\tactor_uid\x18\x03 \x01(\tR\bactorUid\"G\n" + + "\x13DetachActorResponse\x120\n" + "\x05state\x18\x01 \x01(\v2\x1a.atelet.ActorSnapshotStateR\x05state*\x9a\x01\n" + "\x12ActorMetadataField\x12$\n" + " ACTOR_METADATA_FIELD_UNSPECIFIED\x10\x00\x12\x1d\n" + @@ -4241,14 +4364,15 @@ const file_atelet_proto_rawDesc = "" + "\x14MintActorCertificate\x12#.atelet.MintActorCertificateRequest\x1a$.atelet.MintActorCertificateResponse\"\x00\x12Q\n" + "\x0eRegisterWorker\x12\x1d.atelet.RegisterWorkerRequest\x1a\x1e.atelet.RegisterWorkerResponse\"\x00\x12`\n" + "\x13RequestActorSuspend\x12\".atelet.RequestActorSuspendRequest\x1a#.atelet.RequestActorSuspendResponse\"\x00\x12c\n" + - "\x14GetActorEgressPolicy\x12#.atelet.GetActorEgressPolicyRequest\x1a$.atelet.GetActorEgressPolicyResponse\"\x002\xd9\x02\n" + + "\x14GetActorEgressPolicy\x12#.atelet.GetActorEgressPolicyRequest\x1a$.atelet.GetActorEgressPolicyResponse\"\x002\xa3\x03\n" + "\x06Atelet\x120\n" + "\x03Run\x12\x12.atelet.RunRequest\x1a\x13.atelet.RunResponse\"\x00\x12E\n" + "\n" + "Checkpoint\x12\x19.atelet.CheckpointRequest\x1a\x1a.atelet.CheckpointResponse\"\x00\x12<\n" + "\aRestore\x12\x16.atelet.RestoreRequest\x1a\x17.atelet.RestoreResponse\"\x00\x12T\n" + - "\x0fPromoteSnapshot\x12\x1e.atelet.PromoteSnapshotRequest\x1a\x1f.atelet.PromoteSnapshotResponse\"\x00\x12B\n" + - "\tTerminate\x12\x18.atelet.TerminateRequest\x1a\x19.atelet.TerminateResponse\"\x00B>Z atelet.WorkerResources @@ -4349,12 +4475,12 @@ var file_atelet_proto_depIdxs = []int32{ 29, // 15: atelet.RunRequest.spec:type_name -> atelet.WorkloadSpec 28, // 16: atelet.RunRequest.sandbox_assets:type_name -> atelet.SandboxAssets 25, // 17: atelet.RunRequest.egress_gateway:type_name -> atelet.EgressGateway - 59, // 18: atelet.ArchAssets.files:type_name -> atelet.ArchAssets.FilesEntry - 60, // 19: atelet.SandboxAssets.assets:type_name -> atelet.SandboxAssets.AssetsEntry + 61, // 18: atelet.ArchAssets.files:type_name -> atelet.ArchAssets.FilesEntry + 62, // 19: atelet.SandboxAssets.assets:type_name -> atelet.SandboxAssets.AssetsEntry 40, // 20: atelet.WorkloadSpec.containers:type_name -> atelet.Container 38, // 21: atelet.WorkloadSpec.volumes:type_name -> atelet.Volume - 61, // 22: atelet.ExternalVolumeSource.volume_context:type_name -> atelet.ExternalVolumeSource.VolumeContextEntry - 62, // 23: atelet.ExternalVolumeSource.publish_context:type_name -> atelet.ExternalVolumeSource.PublishContextEntry + 63, // 22: atelet.ExternalVolumeSource.volume_context:type_name -> atelet.ExternalVolumeSource.VolumeContextEntry + 64, // 23: atelet.ExternalVolumeSource.publish_context:type_name -> atelet.ExternalVolumeSource.PublishContextEntry 0, // 24: atelet.ActorMetadataItem.field:type_name -> atelet.ActorMetadataField 33, // 25: atelet.ActorMetadataDataSource.items:type_name -> atelet.ActorMetadataItem 34, // 26: atelet.SystemInfoDataSource.actor_metadata:type_name -> atelet.ActorMetadataDataSource @@ -4374,12 +4500,12 @@ var file_atelet_proto_depIdxs = []int32{ 1, // 40: atelet.Snapshot.fidelity:type_name -> atelet.SnapshotFidelity 48, // 41: atelet.Snapshot.object:type_name -> atelet.ObjectStorage 49, // 42: atelet.SnapshotReport.snapshot:type_name -> atelet.Snapshot - 63, // 43: atelet.SnapshotReport.local_commit_time:type_name -> google.protobuf.Timestamp - 63, // 44: atelet.SnapshotReport.durable_complete_time:type_name -> google.protobuf.Timestamp + 65, // 43: atelet.SnapshotReport.local_commit_time:type_name -> google.protobuf.Timestamp + 65, // 44: atelet.SnapshotReport.durable_complete_time:type_name -> google.protobuf.Timestamp 50, // 45: atelet.ActorSnapshotState.latest_local:type_name -> atelet.SnapshotReport 50, // 46: atelet.ActorSnapshotState.uploading:type_name -> atelet.SnapshotReport 50, // 47: atelet.ActorSnapshotState.latest_durable:type_name -> atelet.SnapshotReport - 63, // 48: atelet.ActorSnapshotState.upload_due_time:type_name -> google.protobuf.Timestamp + 65, // 48: atelet.ActorSnapshotState.upload_due_time:type_name -> google.protobuf.Timestamp 2, // 49: atelet.UploadConfig.mode:type_name -> atelet.UploadMode 29, // 50: atelet.CheckpointRequest.spec:type_name -> atelet.WorkloadSpec 49, // 51: atelet.CheckpointRequest.snapshot:type_name -> atelet.Snapshot @@ -4391,31 +4517,34 @@ var file_atelet_proto_depIdxs = []int32{ 28, // 57: atelet.RestoreRequest.sandbox_assets:type_name -> atelet.SandboxAssets 49, // 58: atelet.PromoteSnapshotRequest.snapshot:type_name -> atelet.Snapshot 51, // 59: atelet.PromoteSnapshotResponse.state:type_name -> atelet.ActorSnapshotState - 26, // 60: atelet.ArchAssets.FilesEntry.value:type_name -> atelet.AssetFile - 27, // 61: atelet.SandboxAssets.AssetsEntry.value:type_name -> atelet.ArchAssets - 20, // 62: atelet.WorkerSupport.MintActorCertificate:input_type -> atelet.MintActorCertificateRequest - 3, // 63: atelet.WorkerSupport.RegisterWorker:input_type -> atelet.RegisterWorkerRequest - 11, // 64: atelet.WorkerSupport.RequestActorSuspend:input_type -> atelet.RequestActorSuspendRequest - 13, // 65: atelet.WorkerSupport.GetActorEgressPolicy:input_type -> atelet.GetActorEgressPolicyRequest - 24, // 66: atelet.Atelet.Run:input_type -> atelet.RunRequest - 53, // 67: atelet.Atelet.Checkpoint:input_type -> atelet.CheckpointRequest - 55, // 68: atelet.Atelet.Restore:input_type -> atelet.RestoreRequest - 57, // 69: atelet.Atelet.PromoteSnapshot:input_type -> atelet.PromoteSnapshotRequest - 22, // 70: atelet.Atelet.Terminate:input_type -> atelet.TerminateRequest - 21, // 71: atelet.WorkerSupport.MintActorCertificate:output_type -> atelet.MintActorCertificateResponse - 10, // 72: atelet.WorkerSupport.RegisterWorker:output_type -> atelet.RegisterWorkerResponse - 12, // 73: atelet.WorkerSupport.RequestActorSuspend:output_type -> atelet.RequestActorSuspendResponse - 14, // 74: atelet.WorkerSupport.GetActorEgressPolicy:output_type -> atelet.GetActorEgressPolicyResponse - 47, // 75: atelet.Atelet.Run:output_type -> atelet.RunResponse - 54, // 76: atelet.Atelet.Checkpoint:output_type -> atelet.CheckpointResponse - 56, // 77: atelet.Atelet.Restore:output_type -> atelet.RestoreResponse - 58, // 78: atelet.Atelet.PromoteSnapshot:output_type -> atelet.PromoteSnapshotResponse - 23, // 79: atelet.Atelet.Terminate:output_type -> atelet.TerminateResponse - 71, // [71:80] is the sub-list for method output_type - 62, // [62:71] is the sub-list for method input_type - 62, // [62:62] is the sub-list for extension type_name - 62, // [62:62] is the sub-list for extension extendee - 0, // [0:62] is the sub-list for field type_name + 51, // 60: atelet.DetachActorResponse.state:type_name -> atelet.ActorSnapshotState + 26, // 61: atelet.ArchAssets.FilesEntry.value:type_name -> atelet.AssetFile + 27, // 62: atelet.SandboxAssets.AssetsEntry.value:type_name -> atelet.ArchAssets + 20, // 63: atelet.WorkerSupport.MintActorCertificate:input_type -> atelet.MintActorCertificateRequest + 3, // 64: atelet.WorkerSupport.RegisterWorker:input_type -> atelet.RegisterWorkerRequest + 11, // 65: atelet.WorkerSupport.RequestActorSuspend:input_type -> atelet.RequestActorSuspendRequest + 13, // 66: atelet.WorkerSupport.GetActorEgressPolicy:input_type -> atelet.GetActorEgressPolicyRequest + 24, // 67: atelet.Atelet.Run:input_type -> atelet.RunRequest + 53, // 68: atelet.Atelet.Checkpoint:input_type -> atelet.CheckpointRequest + 55, // 69: atelet.Atelet.Restore:input_type -> atelet.RestoreRequest + 57, // 70: atelet.Atelet.PromoteSnapshot:input_type -> atelet.PromoteSnapshotRequest + 59, // 71: atelet.Atelet.DetachActor:input_type -> atelet.DetachActorRequest + 22, // 72: atelet.Atelet.Terminate:input_type -> atelet.TerminateRequest + 21, // 73: atelet.WorkerSupport.MintActorCertificate:output_type -> atelet.MintActorCertificateResponse + 10, // 74: atelet.WorkerSupport.RegisterWorker:output_type -> atelet.RegisterWorkerResponse + 12, // 75: atelet.WorkerSupport.RequestActorSuspend:output_type -> atelet.RequestActorSuspendResponse + 14, // 76: atelet.WorkerSupport.GetActorEgressPolicy:output_type -> atelet.GetActorEgressPolicyResponse + 47, // 77: atelet.Atelet.Run:output_type -> atelet.RunResponse + 54, // 78: atelet.Atelet.Checkpoint:output_type -> atelet.CheckpointResponse + 56, // 79: atelet.Atelet.Restore:output_type -> atelet.RestoreResponse + 58, // 80: atelet.Atelet.PromoteSnapshot:output_type -> atelet.PromoteSnapshotResponse + 60, // 81: atelet.Atelet.DetachActor:output_type -> atelet.DetachActorResponse + 23, // 82: atelet.Atelet.Terminate:output_type -> atelet.TerminateResponse + 73, // [73:83] is the sub-list for method output_type + 63, // [63:73] is the sub-list for method input_type + 63, // [63:63] is the sub-list for extension type_name + 63, // [63:63] is the sub-list for extension extendee + 0, // [0:63] is the sub-list for field type_name } func init() { file_atelet_proto_init() } @@ -4431,7 +4560,7 @@ func file_atelet_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_atelet_proto_rawDesc), len(file_atelet_proto_rawDesc)), NumEnums: 3, - NumMessages: 60, + NumMessages: 62, NumExtensions: 0, NumServices: 2, }, diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index 7608234b8e..94fbbdefbd 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -18,7 +18,7 @@ package atelet; import "google/protobuf/timestamp.proto"; -option go_package = "github.com/agent-substrate/substrate/internal/proto/ateletpb"; +option go_package = "go.ate.dev/substrate/internal/proto/ateletpb"; // WorkerSupport provides callouts for worker implementations. // @@ -110,12 +110,14 @@ message VersionedSandboxCompat { string schema_version = 1; // Key-value compatibility attributes defined by this schema version, covering - // both runtime assets (e.g. "gvisor_asset_hash") and effective guest hardware - // features after masking (e.g. "architecture", "cpu_features"). + // both runtime assets (e.g. "sandbox_runtime_hash") and effective guest + // hardware features after masking (e.g. "architecture", "cpu_features"). // Matched against a snapshot under the same schema_version: every attribute - // on the snapshot must be present on the worker with an equal value; keys - // only the worker has are ignored. Bump schema_version when a key's meaning - // changes or older snapshots must no longer match. + // on the snapshot must be present on the worker with an equal value (except + // "cpu_features", where the snapshot's hexadecimal bitmask must be a bitwise + // subset of the worker's); keys only the worker has are ignored. Bump + // schema_version when a key's meaning changes or older snapshots must no + // longer match. // Order as sent does not matter: ate-api-server sorts attributes by key // before recording them. // @@ -128,7 +130,7 @@ message VersionedSandboxCompat { // AttributeEntry is a single key-value compatibility attribute. message AttributeEntry { - // Attribute name (e.g. "architecture", "cpu_features", "gvisor_asset_hash"). + // Attribute name (e.g. "architecture", "cpu_features", "sandbox_runtime_hash"). // // +k8s:required // +k8s:maxLength=128 @@ -342,6 +344,17 @@ service Atelet { // caller, the suspend of a paused actor, relies on the deletion. rpc PromoteSnapshot(PromoteSnapshotRequest) returns (PromoteSnapshotResponse) {} + // DetachActor makes the actor's latest local snapshot durable, then deletes + // the actor's local snapshots on this node, so it can resume on any node. + // atelet also detaches a suspended actor on its own after its detach delay. + // Waits for an operation already in progress on the actor, such as an + // upload or a detach. Does nothing when atelet holds no local snapshot for + // the actor. Returns ABORTED while a Run, Restore, or Checkpoint of the + // actor is in progress or the actor runs on this node. + // + // TODO: Temporarily, atelet returns UNIMPLEMENTED. + rpc DetachActor(DetachActorRequest) returns (DetachActorResponse) {} + // Terminate tells atelet to terminate/kill any running workload for an actor, // unmount its volumes, and clean up actor state on the node. rpc Terminate(TerminateRequest) returns (TerminateResponse) {} @@ -926,7 +939,7 @@ message SnapshotReport { } // ActorSnapshotState is what atelet holds for one actor on this node. -// Returned by Checkpoint, PromoteSnapshot. +// Returned by Checkpoint, PromoteSnapshot, DetachActor. message ActorSnapshotState { // The snapshot atelet restores the actor from locally. Unset when the // latest checkpoint kept no local copy, and once a detach has made the @@ -1162,3 +1175,25 @@ message PromoteSnapshotResponse { // and latest_local remains set if atelet still holds the local copy. ActorSnapshotState state = 1; } + +message DetachActorRequest { + // Golden actors are accepted: an IMMEDIATE checkpoint keeps a local copy. + // + // +k8s:required + // +k8s:format=k8s-short-name + string atespace = 1; + + // +k8s:required + // +k8s:format=k8s-short-name + string actor_name = 2; + + // +k8s:required + // +k8s:format=k8s-uuid + string actor_uid = 3; +} + +message DetachActorResponse { + // The actor's state once detached: latest_local is unset, and latest_durable + // is its latest snapshot, if atelet knows one. + ActorSnapshotState state = 1; +} diff --git a/internal/proto/ateletpb/atelet_grpc.pb.go b/internal/proto/ateletpb/atelet_grpc.pb.go index 6a9edd52c7..2282a4651b 100644 --- a/internal/proto/ateletpb/atelet_grpc.pb.go +++ b/internal/proto/ateletpb/atelet_grpc.pb.go @@ -301,6 +301,7 @@ const ( Atelet_Checkpoint_FullMethodName = "/atelet.Atelet/Checkpoint" Atelet_Restore_FullMethodName = "/atelet.Atelet/Restore" Atelet_PromoteSnapshot_FullMethodName = "/atelet.Atelet/PromoteSnapshot" + Atelet_DetachActor_FullMethodName = "/atelet.Atelet/DetachActor" Atelet_Terminate_FullMethodName = "/atelet.Atelet/Terminate" ) @@ -339,6 +340,16 @@ type AteletClient interface { // the upload and does not check whether the actor is running. Its only // caller, the suspend of a paused actor, relies on the deletion. PromoteSnapshot(ctx context.Context, in *PromoteSnapshotRequest, opts ...grpc.CallOption) (*PromoteSnapshotResponse, error) + // DetachActor makes the actor's latest local snapshot durable, then deletes + // the actor's local snapshots on this node, so it can resume on any node. + // atelet also detaches a suspended actor on its own after its detach delay. + // Waits for an operation already in progress on the actor, such as an + // upload or a detach. Does nothing when atelet holds no local snapshot for + // the actor. Returns ABORTED while a Run, Restore, or Checkpoint of the + // actor is in progress or the actor runs on this node. + // + // TODO: Temporarily, atelet returns UNIMPLEMENTED. + DetachActor(ctx context.Context, in *DetachActorRequest, opts ...grpc.CallOption) (*DetachActorResponse, error) // Terminate tells atelet to terminate/kill any running workload for an actor, // unmount its volumes, and clean up actor state on the node. Terminate(ctx context.Context, in *TerminateRequest, opts ...grpc.CallOption) (*TerminateResponse, error) @@ -392,6 +403,16 @@ func (c *ateletClient) PromoteSnapshot(ctx context.Context, in *PromoteSnapshotR return out, nil } +func (c *ateletClient) DetachActor(ctx context.Context, in *DetachActorRequest, opts ...grpc.CallOption) (*DetachActorResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(DetachActorResponse) + err := c.cc.Invoke(ctx, Atelet_DetachActor_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *ateletClient) Terminate(ctx context.Context, in *TerminateRequest, opts ...grpc.CallOption) (*TerminateResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(TerminateResponse) @@ -437,6 +458,16 @@ type AteletServer interface { // the upload and does not check whether the actor is running. Its only // caller, the suspend of a paused actor, relies on the deletion. PromoteSnapshot(context.Context, *PromoteSnapshotRequest) (*PromoteSnapshotResponse, error) + // DetachActor makes the actor's latest local snapshot durable, then deletes + // the actor's local snapshots on this node, so it can resume on any node. + // atelet also detaches a suspended actor on its own after its detach delay. + // Waits for an operation already in progress on the actor, such as an + // upload or a detach. Does nothing when atelet holds no local snapshot for + // the actor. Returns ABORTED while a Run, Restore, or Checkpoint of the + // actor is in progress or the actor runs on this node. + // + // TODO: Temporarily, atelet returns UNIMPLEMENTED. + DetachActor(context.Context, *DetachActorRequest) (*DetachActorResponse, error) // Terminate tells atelet to terminate/kill any running workload for an actor, // unmount its volumes, and clean up actor state on the node. Terminate(context.Context, *TerminateRequest) (*TerminateResponse, error) @@ -462,6 +493,9 @@ func (UnimplementedAteletServer) Restore(context.Context, *RestoreRequest) (*Res func (UnimplementedAteletServer) PromoteSnapshot(context.Context, *PromoteSnapshotRequest) (*PromoteSnapshotResponse, error) { return nil, status.Error(codes.Unimplemented, "method PromoteSnapshot not implemented") } +func (UnimplementedAteletServer) DetachActor(context.Context, *DetachActorRequest) (*DetachActorResponse, error) { + return nil, status.Error(codes.Unimplemented, "method DetachActor not implemented") +} func (UnimplementedAteletServer) Terminate(context.Context, *TerminateRequest) (*TerminateResponse, error) { return nil, status.Error(codes.Unimplemented, "method Terminate not implemented") } @@ -558,6 +592,24 @@ func _Atelet_PromoteSnapshot_Handler(srv interface{}, ctx context.Context, dec f return interceptor(ctx, in, info, handler) } +func _Atelet_DetachActor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DetachActorRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AteletServer).DetachActor(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Atelet_DetachActor_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AteletServer).DetachActor(ctx, req.(*DetachActorRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _Atelet_Terminate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(TerminateRequest) if err := dec(in); err != nil { @@ -599,6 +651,10 @@ var Atelet_ServiceDesc = grpc.ServiceDesc{ MethodName: "PromoteSnapshot", Handler: _Atelet_PromoteSnapshot_Handler, }, + { + MethodName: "DetachActor", + Handler: _Atelet_DetachActor_Handler, + }, { MethodName: "Terminate", Handler: _Atelet_Terminate_Handler, diff --git a/internal/proto/ateworkerpb/ateworker.pb.go b/internal/proto/ateworkerpb/ateworker.pb.go index ee23673a67..5d1331f7d4 100644 --- a/internal/proto/ateworkerpb/ateworker.pb.go +++ b/internal/proto/ateworkerpb/ateworker.pb.go @@ -2166,7 +2166,7 @@ const file_ateworker_proto_rawDesc = "" + "\x0fRestoreWorkload\x12!.ateworker.RestoreWorkloadRequest\x1a\".ateworker.RestoreWorkloadResponse\"\x00\x12]\n" + "\x10GetWorkloadStats\x12\".ateworker.GetWorkloadStatsRequest\x1a#.ateworker.GetWorkloadStatsResponse\"\x00\x12o\n" + "\x16GetActiveWorkloadStats\x12(.ateworker.GetActiveWorkloadStatsRequest\x1a).ateworker.GetActiveWorkloadStatsResponse\"\x00\x12`\n" + - "\x11TerminateWorkload\x12#.ateworker.TerminateWorkloadRequest\x1a$.ateworker.TerminateWorkloadResponse\"\x00BAZ?github.com/agent-substrate/substrate/internal/proto/ateworkerpbb\x06proto3" + "\x11TerminateWorkload\x12#.ateworker.TerminateWorkloadRequest\x1a$.ateworker.TerminateWorkloadResponse\"\x00B1Z/go.ate.dev/substrate/internal/proto/ateworkerpbb\x06proto3" var ( file_ateworker_proto_rawDescOnce sync.Once diff --git a/internal/proto/ateworkerpb/ateworker.proto b/internal/proto/ateworkerpb/ateworker.proto index 79b8da40b8..8a379e5c62 100644 --- a/internal/proto/ateworkerpb/ateworker.proto +++ b/internal/proto/ateworkerpb/ateworker.proto @@ -16,7 +16,7 @@ syntax = "proto3"; package ateworker; -option go_package = "github.com/agent-substrate/substrate/internal/proto/ateworkerpb"; +option go_package = "go.ate.dev/substrate/internal/proto/ateworkerpb"; // Worker is the interface to control a single gVisor (or, in the future microVM) // guest inside a worker pod. diff --git a/internal/proto/glutton/glutton.pb.go b/internal/proto/glutton/glutton.pb.go index 426554a768..f7be48fa91 100644 --- a/internal/proto/glutton/glutton.pb.go +++ b/internal/proto/glutton/glutton.pb.go @@ -1308,7 +1308,7 @@ const file_glutton_proto_rawDesc = "" + "\x06Gossip\x12\x16.glutton.GossipRequest\x1a\x17.glutton.GossipResponse\"\x00\x12>\n" + "\aBurnCPU\x12\x17.glutton.BurnCPURequest\x1a\x18.glutton.BurnCPUResponse\"\x00\x12;\n" + "\x06Ingest\x12\x16.glutton.IngestRequest\x1a\x17.glutton.IngestResponse\"\x00\x12;\n" + - "\x06UseCPU\x12\x16.glutton.UseCPURequest\x1a\x17.glutton.UseCPUResponse\"\x00B=Z;github.com/agent-substrate/substrate/internal/proto/gluttonb\x06proto3" + "\x06UseCPU\x12\x16.glutton.UseCPURequest\x1a\x17.glutton.UseCPUResponse\"\x00B-Z+go.ate.dev/substrate/internal/proto/gluttonb\x06proto3" var ( file_glutton_proto_rawDescOnce sync.Once diff --git a/internal/proto/glutton/glutton.proto b/internal/proto/glutton/glutton.proto index 1e278574f3..e717abd4cd 100644 --- a/internal/proto/glutton/glutton.proto +++ b/internal/proto/glutton/glutton.proto @@ -16,7 +16,7 @@ syntax = "proto3"; package glutton; -option go_package = "github.com/agent-substrate/substrate/internal/proto/glutton"; +option go_package = "go.ate.dev/substrate/internal/proto/glutton"; // A glutton is a single small piece of a larger benchmarking suite. It is designed to be instantiated // multiple times, with each piece configured to do different units of work. diff --git a/internal/proto/grpcechopb/grpcecho.pb.go b/internal/proto/grpcechopb/grpcecho.pb.go index d0daaf535f..8ba5112683 100644 --- a/internal/proto/grpcechopb/grpcecho.pb.go +++ b/internal/proto/grpcechopb/grpcecho.pb.go @@ -206,7 +206,7 @@ const file_grpcecho_proto_rawDesc = "" + "\x04Echo\x12\x15.grpcecho.EchoRequest\x1a\x16.grpcecho.EchoResponse\"\x00\x12E\n" + "\n" + "EchoStream\x12\x1b.grpcecho.EchoStreamRequest\x1a\x16.grpcecho.EchoResponse\"\x000\x01\x12?\n" + - "\bEchoBidi\x12\x15.grpcecho.EchoRequest\x1a\x16.grpcecho.EchoResponse\"\x00(\x010\x01B@Z>github.com/agent-substrate/substrate/internal/proto/grpcechopbb\x06proto3" + "\bEchoBidi\x12\x15.grpcecho.EchoRequest\x1a\x16.grpcecho.EchoResponse\"\x00(\x010\x01B0Z.go.ate.dev/substrate/internal/proto/grpcechopbb\x06proto3" var ( file_grpcecho_proto_rawDescOnce sync.Once diff --git a/internal/proto/grpcechopb/grpcecho.proto b/internal/proto/grpcechopb/grpcecho.proto index 8e38ffe3db..437b881f1a 100644 --- a/internal/proto/grpcechopb/grpcecho.proto +++ b/internal/proto/grpcechopb/grpcecho.proto @@ -16,7 +16,7 @@ syntax = "proto3"; package grpcecho; -option go_package = "github.com/agent-substrate/substrate/internal/proto/grpcechopb"; +option go_package = "go.ate.dev/substrate/internal/proto/grpcechopb"; // Echo is the smallest service that can prove a gRPC request and its response // crossed a network path intact. It exists for the egress e2e coverage: the diff --git a/internal/protoredact/protoredact_test.go b/internal/protoredact/protoredact_test.go index d3a6425e95..28c9ef9d92 100644 --- a/internal/protoredact/protoredact_test.go +++ b/internal/protoredact/protoredact_test.go @@ -23,14 +23,15 @@ import ( "sync" "testing" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/proto/glutton" - "github.com/agent-substrate/substrate/internal/proto/grpcechopb" - "github.com/agent-substrate/substrate/internal/protoredact" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - "github.com/agent-substrate/substrate/pkg/proto/credproviderpb" - objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/proto/glutton" + "go.ate.dev/substrate/internal/proto/grpcechopb" + "go.ate.dev/substrate/internal/protoredact" + "go.ate.dev/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateconfigpb" + "go.ate.dev/substrate/pkg/proto/credproviderpb" + objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protodesc" "google.golang.org/protobuf/reflect/protoreflect" @@ -380,6 +381,7 @@ var ourProtoFiles = []protoreflect.FileDescriptor{ ateapipb.File_ateapi_proto, ateletpb.File_atelet_proto, ateworkerpb.File_ateworker_proto, + ateconfigpb.File_apiconfig_proto, credproviderpb.File_credprovider_proto, glutton.File_glutton_proto, grpcechopb.File_grpcecho_proto, diff --git a/internal/resources/actorattribution.go b/internal/resources/actorattribution.go index a1907690d5..4704bb9311 100644 --- a/internal/resources/actorattribution.go +++ b/internal/resources/actorattribution.go @@ -14,7 +14,7 @@ package resources -import "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +import "go.ate.dev/substrate/pkg/proto/ateapipb" // ActorAttribution is what telemetry about an actor is attributed to: an // ActorRef plus the two things a ref does not carry, the server-assigned uid and diff --git a/internal/resources/quantities.go b/internal/resources/quantities.go index 2e4072215f..76c97362f7 100644 --- a/internal/resources/quantities.go +++ b/internal/resources/quantities.go @@ -18,7 +18,7 @@ import ( "fmt" "sort" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "k8s.io/apimachinery/pkg/api/resource" ) diff --git a/internal/resources/quantities_test.go b/internal/resources/quantities_test.go index 34aa1b2f24..c141566259 100644 --- a/internal/resources/quantities_test.go +++ b/internal/resources/quantities_test.go @@ -17,7 +17,7 @@ package resources import ( "testing" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" "k8s.io/apimachinery/pkg/api/resource" ) diff --git a/internal/resources/resourceref.go b/internal/resources/resourceref.go index a14d079d75..3c27bd12f0 100644 --- a/internal/resources/resourceref.go +++ b/internal/resources/resourceref.go @@ -18,7 +18,7 @@ import ( "log/slog" "reflect" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // Resource is any Atespaced resource message carrying the common metadata. diff --git a/internal/resources/resourceref_test.go b/internal/resources/resourceref_test.go index d1c416a85a..47eeaa9b6e 100644 --- a/internal/resources/resourceref_test.go +++ b/internal/resources/resourceref_test.go @@ -19,7 +19,7 @@ import ( "reflect" "testing" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // The ref aliases must stay distinct types: the phantom ResourceRef kind is diff --git a/internal/resources/validate.go b/internal/resources/validate.go index 03002aad97..7376fb6b1b 100644 --- a/internal/resources/validate.go +++ b/internal/resources/validate.go @@ -27,11 +27,11 @@ import ( "strconv" "strings" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/volumepath" - "github.com/agent-substrate/substrate/pkg/sandboxclass" "github.com/distribution/reference" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/volumepath" + "go.ate.dev/substrate/pkg/sandboxclass" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" diff --git a/internal/resources/validate_test.go b/internal/resources/validate_test.go index 0cf25d8261..d702097dd0 100644 --- a/internal/resources/validate_test.go +++ b/internal/resources/validate_test.go @@ -22,9 +22,9 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "k8s.io/apimachinery/pkg/util/validation/field" diff --git a/internal/resources/worker.go b/internal/resources/worker.go index 7ee068f8e4..2ba5dd0d4b 100644 --- a/internal/resources/worker.go +++ b/internal/resources/worker.go @@ -17,7 +17,7 @@ package resources import ( "fmt" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // AddToAllocated adjusts allocation by an assignment; sign is 1 or -1. diff --git a/internal/resources/worker_test.go b/internal/resources/worker_test.go index 68590df846..f56c309695 100644 --- a/internal/resources/worker_test.go +++ b/internal/resources/worker_test.go @@ -17,7 +17,7 @@ package resources import ( "testing" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/testing/protocmp" "github.com/google/go-cmp/cmp" diff --git a/internal/sandboxversions/sandboxversions.go b/internal/sandboxruntimes/sandboxruntimes.go similarity index 86% rename from internal/sandboxversions/sandboxversions.go rename to internal/sandboxruntimes/sandboxruntimes.go index 3ae871c9f0..34c6ca8c6b 100644 --- a/internal/sandboxversions/sandboxversions.go +++ b/internal/sandboxruntimes/sandboxruntimes.go @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package sandboxversions reads and writes a worker's sandbox versions file, +// Package sandboxruntimes reads and writes a worker's sandbox runtimes file, // which atelet keeps in the worker's directory (nodepath.WorkerPath) as -// nodepath.SandboxVersionsFileName. -package sandboxversions +// nodepath.SandboxRuntimesFileName. +package sandboxruntimes import ( "bytes" @@ -25,11 +25,10 @@ import ( "path/filepath" ) -// File is the content of a worker's sandbox versions file: the sandbox +// File is the content of a worker's sandbox runtimes file: the sandbox // runtimes the worker can run, as the compatibility attributes atelet computed -// for them. The worker adds each runtime's attributes to the ones it probes -// itself and registers the result as RegisterWorkerRequest's default_runtime -// and restorable_runtimes. +// for them. The worker registers DefaultRuntime and RestorableRuntimes as +// RegisterWorkerRequest's default_runtime and restorable_runtimes. type File struct { // DefaultRuntime is the runtime new and cold-booted actors start under, // from the default version of the pool's SandboxConfig. @@ -46,7 +45,7 @@ type Attributes map[string]string // tempPrefix starts the name of every temporary file Write renames into // place. It is dot-prefixed so a directory listing does not show one. -const tempPrefix = ".sandbox-versions-" +const tempPrefix = ".sandbox-runtimes-" // Write atomically replaces the file at path with v, unless it already holds // the same content. It reports whether it wrote. diff --git a/internal/sandboxversions/sandboxversions_test.go b/internal/sandboxruntimes/sandboxruntimes_test.go similarity index 91% rename from internal/sandboxversions/sandboxversions_test.go rename to internal/sandboxruntimes/sandboxruntimes_test.go index 5665d6e6c2..0562caa464 100644 --- a/internal/sandboxversions/sandboxversions_test.go +++ b/internal/sandboxruntimes/sandboxruntimes_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sandboxversions +package sandboxruntimes import ( "errors" @@ -26,7 +26,7 @@ import ( "github.com/google/go-cmp/cmp" ) -func testVersions(defaultHash string) *File { +func testRuntimes(defaultHash string) *File { return &File{ DefaultRuntime: Attributes{"sandbox_runtime_hash": defaultHash}, RestorableRuntimes: []Attributes{{"sandbox_runtime_hash": "sha256:41ab"}}, @@ -44,22 +44,22 @@ func TestWrite(t *testing.T) { wantErr bool }{{ name: "new file", - v: testVersions("sha256:7c2e"), + v: testRuntimes("sha256:7c2e"), wantWrote: true, }, { name: "equal content is not rewritten", - existing: testVersions("sha256:7c2e"), - v: testVersions("sha256:7c2e"), + existing: testRuntimes("sha256:7c2e"), + v: testRuntimes("sha256:7c2e"), wantWrote: false, }, { name: "different content replaces the file", - existing: testVersions("sha256:7c2e"), - v: testVersions("sha256:9d01"), + existing: testRuntimes("sha256:7c2e"), + v: testRuntimes("sha256:9d01"), wantWrote: true, }, { name: "missing directory", noDir: true, - v: testVersions("sha256:7c2e"), + v: testRuntimes("sha256:7c2e"), wantErr: true, }} for _, tt := range tests { @@ -68,7 +68,7 @@ func TestWrite(t *testing.T) { if tt.noDir { dir = filepath.Join(dir, "missing") } - path := filepath.Join(dir, "sandbox-versions.json") + path := filepath.Join(dir, "sandbox-runtimes.json") var before time.Time if tt.existing != nil { if _, err := Write(path, tt.existing); err != nil { @@ -156,7 +156,7 @@ func TestRead(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - path := filepath.Join(t.TempDir(), "sandbox-versions.json") + path := filepath.Join(t.TempDir(), "sandbox-runtimes.json") if tt.contents != "" { if err := os.WriteFile(path, []byte(tt.contents), 0o600); err != nil { t.Fatal(err) diff --git a/internal/serverboot/logging.go b/internal/serverboot/logging.go index d50142d607..f9a7f9af57 100644 --- a/internal/serverboot/logging.go +++ b/internal/serverboot/logging.go @@ -24,7 +24,7 @@ import ( sdklog "go.opentelemetry.io/otel/sdk/log" "google.golang.org/grpc" - "github.com/agent-substrate/substrate/internal/actorevent" + "go.ate.dev/substrate/internal/actorevent" ) const logsExporterEnv = "OTEL_LOGS_EXPORTER" diff --git a/internal/serverboot/logging_test.go b/internal/serverboot/logging_test.go index 03c4286e4b..e41df9551a 100644 --- a/internal/serverboot/logging_test.go +++ b/internal/serverboot/logging_test.go @@ -27,10 +27,10 @@ import ( "go.opentelemetry.io/otel/sdk/resource" "google.golang.org/grpc" - "github.com/agent-substrate/substrate/internal/actorevent" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/contextlogging" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/internal/actorevent" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/contextlogging" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // TestResolveLogsExporter pins the logs names on top of the shared rules: otlp diff --git a/internal/serverboot/serverboot.go b/internal/serverboot/serverboot.go index cf52e4ef36..9b9779078c 100644 --- a/internal/serverboot/serverboot.go +++ b/internal/serverboot/serverboot.go @@ -32,11 +32,11 @@ import ( "sync/atomic" "time" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/contextlogging" "github.com/google/uuid" promclient "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/contextlogging" prombridge "go.opentelemetry.io/contrib/bridges/prometheus" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.opentelemetry.io/otel" diff --git a/internal/serverboot/serverboot_test.go b/internal/serverboot/serverboot_test.go index 48763c8700..81c93e6702 100644 --- a/internal/serverboot/serverboot_test.go +++ b/internal/serverboot/serverboot_test.go @@ -43,7 +43,7 @@ import ( "google.golang.org/grpc/health" healthpb "google.golang.org/grpc/health/grpc_health_v1" - "github.com/agent-substrate/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/ateattr" ) func resourceAttrs(res *resource.Resource) map[string]string { diff --git a/internal/snapshotcache/snapshotcache.go b/internal/snapshotcache/snapshotcache.go index a18773a649..a1e2c710db 100644 --- a/internal/snapshotcache/snapshotcache.go +++ b/internal/snapshotcache/snapshotcache.go @@ -39,7 +39,7 @@ type File struct { } var ( - cachestatUnsupportedOnce sync.Once + cachestatWarnOnce sync.Once // cachestatFn and fadviseFn are seams for unit tests so eviction logic can // be verified deterministically regardless of the backing filesystem (ext4, @@ -77,34 +77,44 @@ func FindFiles(dir string, match func(name string) bool) []File { return targets } -// DropFile unlinks t.Path if it still refers to the same inode and size. -func DropFile(t File) { +// DropFile unlinks t.Path if it still refers to the same inode and size, and +// reports whether the file was removed. +func DropFile(t File) bool { st, err := os.Stat(t.Path) if err != nil { if !errors.Is(err, os.ErrNotExist) { slog.Debug("could not stat staged snapshot file", slog.String("path", t.Path), slog.Any("err", err)) } - return + return false } if !os.SameFile(st, t.Info) || st.Size() != t.Info.Size() { - return + return false } - if err := os.Remove(t.Path); err != nil && !errors.Is(err, os.ErrNotExist) { - slog.Warn("could not drop staged snapshot file", slog.String("path", t.Path), slog.Any("err", err)) + if err := os.Remove(t.Path); err != nil { + if !errors.Is(err, os.ErrNotExist) { + slog.Warn("could not drop staged snapshot file", slog.String("path", t.Path), slog.Any("err", err)) + } + return false } + return true } -// DropFiles calls DropFile for each file in files. -func DropFiles(files []File) { +// DropFiles calls DropFile for each file in files and reports whether any file +// was removed. +func DropFiles(files []File) bool { + dropped := false for _, t := range files { - DropFile(t) + if DropFile(t) { + dropped = true + } } + return dropped } // EvictFiles evicts clean cached pages of targets via POSIX_FADV_DONTNEED and -// returns true if any target still exists and has pages (clean or dirty) -// remaining in cache. Files with dirty or writeback pages are skipped so -// FADV_DONTNEED never forces synchronous disk writeback I/O. +// returns true if any target still has dirty or writeback pages remaining in +// cache (or if fadvise failed). Files with dirty or writeback pages are skipped +// so FADV_DONTNEED never forces synchronous disk writeback I/O. func EvictFiles(targets []File) bool { anyRemaining := false for _, t := range targets { @@ -116,10 +126,13 @@ func EvictFiles(targets []File) bool { if err == nil && os.SameFile(info, t.Info) && info.Size() == t.Info.Size() { var cstat unix.Cachestat_t if err := cachestatFn(uint(f.Fd()), &unix.CachestatRange{}, &cstat, 0); err != nil { - if errors.Is(err, unix.ENOSYS) { - cachestatUnsupportedOnce.Do(func() { - slog.Warn("cachestat(2) syscall not supported by kernel; skipping preserved snapshot page-cache eviction", slog.Any("err", err)) - }) + loggedWarn := false + cachestatWarnOnce.Do(func() { + loggedWarn = true + slog.Warn("cachestat(2) syscall failed; skipping preserved snapshot page-cache eviction", slog.String("path", t.Path), slog.Any("err", err)) + }) + if !loggedWarn { + slog.Debug("cachestat(2) syscall failed; skipping preserved snapshot page-cache eviction", slog.String("path", t.Path), slog.Any("err", err)) } _ = f.Close() continue @@ -128,13 +141,15 @@ func EvictFiles(targets []File) bool { _ = f.Close() continue } - anyRemaining = true if cstat.Dirty > 0 || cstat.Writeback > 0 { + anyRemaining = true _ = f.Close() continue } // length zero extends the Fadvise to the whole file. - _ = fadviseFn(int(f.Fd()), 0, 0, unix.FADV_DONTNEED) + if err := fadviseFn(int(f.Fd()), 0, 0, unix.FADV_DONTNEED); err != nil { + anyRemaining = true + } } _ = f.Close() } diff --git a/internal/snapshotcache/snapshotcache_test.go b/internal/snapshotcache/snapshotcache_test.go index d4843aaa2d..ca2770962e 100644 --- a/internal/snapshotcache/snapshotcache_test.go +++ b/internal/snapshotcache/snapshotcache_test.go @@ -43,7 +43,9 @@ func TestFindAndDropFiles(t *testing.T) { t.Fatalf("FindFiles() = %d files, want 3", len(files)) } - DropFiles(files) + if !DropFiles(files) { + t.Fatalf("DropFiles() = false, want true") + } for _, name := range []string{"checkpoint.img", "pages.img", "durable-dir.tar"} { if _, err := os.Stat(filepath.Join(dir, name)); !os.IsNotExist(err) { @@ -74,7 +76,9 @@ func TestDropFileSkipsReplacedInode(t *testing.T) { t.Fatal(err) } - DropFiles(files) + if DropFiles(files) { + t.Fatalf("DropFiles() = true after inode replacement, want false") + } got, err := os.ReadFile(path) if err != nil { t.Fatalf("replaced file was deleted: %v", err) @@ -115,8 +119,8 @@ func TestEvictFiles(t *testing.T) { fadviseCalls++ return nil } - if !EvictFiles(targets) { - t.Errorf("EvictFiles() = false, want true when pages were cached") + if EvictFiles(targets) { + t.Errorf("EvictFiles() = true, want false when clean pages were evicted") } if fadviseCalls != 1 { t.Errorf("fadviseCalls = %d, want 1", fadviseCalls) diff --git a/internal/tarutil/tarutil_test.go b/internal/tarutil/tarutil_test.go index a7ed7f6c0b..47572ae180 100644 --- a/internal/tarutil/tarutil_test.go +++ b/internal/tarutil/tarutil_test.go @@ -28,7 +28,7 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/roottest" + "go.ate.dev/substrate/internal/roottest" "golang.org/x/sys/unix" ) diff --git a/internal/testcert/servertls.go b/internal/testcert/servertls.go index e5783c0371..a3d07e2f98 100644 --- a/internal/testcert/servertls.go +++ b/internal/testcert/servertls.go @@ -25,7 +25,7 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/localca" + "go.ate.dev/substrate/internal/localca" ) // ServerTLS holds an origin key pair and the public CA trusted by its client. diff --git a/internal/testcert/servertls_test.go b/internal/testcert/servertls_test.go index fda0c12772..cde99d2a8f 100644 --- a/internal/testcert/servertls_test.go +++ b/internal/testcert/servertls_test.go @@ -21,7 +21,7 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/localca" + "go.ate.dev/substrate/internal/localca" ) func TestNewServerTLS(t *testing.T) { diff --git a/internal/volume/csi/plugin.go b/internal/volume/csi/plugin.go index 326dc8617f..4b348680cb 100644 --- a/internal/volume/csi/plugin.go +++ b/internal/volume/csi/plugin.go @@ -24,11 +24,11 @@ import ( "path/filepath" "sync" - "github.com/agent-substrate/substrate/internal/credbundle" - "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/volume" - v1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/container-storage-interface/spec/lib/go/csi" + "go.ate.dev/substrate/internal/credbundle" + "go.ate.dev/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/volume" + v1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "k8s.io/apimachinery/pkg/api/resource" diff --git a/internal/volume/csi/plugin_test.go b/internal/volume/csi/plugin_test.go index 5a96ae9067..908e971c1b 100644 --- a/internal/volume/csi/plugin_test.go +++ b/internal/volume/csi/plugin_test.go @@ -22,9 +22,9 @@ import ( "path/filepath" "testing" - "github.com/agent-substrate/substrate/internal/volume" "github.com/container-storage-interface/spec/lib/go/csi" "github.com/google/go-cmp/cmp" + "go.ate.dev/substrate/internal/volume" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" diff --git a/internal/volume/csi/tls_test.go b/internal/volume/csi/tls_test.go index 0bf4d6f2e7..5a4060b648 100644 --- a/internal/volume/csi/tls_test.go +++ b/internal/volume/csi/tls_test.go @@ -32,9 +32,9 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" "github.com/container-storage-interface/spec/lib/go/csi" + "go.ate.dev/substrate/pkg/api/v1alpha1" + listersv1alpha1 "go.ate.dev/substrate/pkg/client/listers/api/v1alpha1" "google.golang.org/grpc" "google.golang.org/grpc/credentials" "k8s.io/apimachinery/pkg/labels" diff --git a/internal/volume/mock.go b/internal/volume/mock.go index b745e819c8..ad8e24758f 100644 --- a/internal/volume/mock.go +++ b/internal/volume/mock.go @@ -21,7 +21,7 @@ import ( "os" "path/filepath" - "github.com/agent-substrate/substrate/internal/nodepath" + "go.ate.dev/substrate/internal/nodepath" ) // Use a directory that is shared between atelet and worker but not cleaned up by atelet diff --git a/internal/wakeupprobe/wakeupprobe.go b/internal/wakeupprobe/wakeupprobe.go index 6995cc284a..86eb79dbca 100644 --- a/internal/wakeupprobe/wakeupprobe.go +++ b/internal/wakeupprobe/wakeupprobe.go @@ -31,7 +31,7 @@ import ( "strings" "time" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/proto/ateworkerpb" "golang.org/x/sync/errgroup" ) diff --git a/internal/wakeupprobe/wakeupprobe_test.go b/internal/wakeupprobe/wakeupprobe_test.go index fd43825824..a19f0e193e 100644 --- a/internal/wakeupprobe/wakeupprobe_test.go +++ b/internal/wakeupprobe/wakeupprobe_test.go @@ -26,7 +26,7 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/proto/ateworkerpb" ) func TestURL(t *testing.T) { diff --git a/internal/workerstats/activation.go b/internal/workerstats/activation.go index f91e4a552f..68b647be30 100644 --- a/internal/workerstats/activation.go +++ b/internal/workerstats/activation.go @@ -21,7 +21,7 @@ import ( "sync" "time" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/proto/ateworkerpb" ) // Activation is the usage state of one activation of an actor: one Run or diff --git a/internal/workerstats/activation_test.go b/internal/workerstats/activation_test.go index 3a9366cb48..d078bf0b6e 100644 --- a/internal/workerstats/activation_test.go +++ b/internal/workerstats/activation_test.go @@ -24,7 +24,7 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/proto/ateworkerpb" ) // reading is one read of counters by name; a single counter reads as "". diff --git a/internal/workerstats/actorattribution.go b/internal/workerstats/actorattribution.go index 332ec026c6..91fedb51ce 100644 --- a/internal/workerstats/actorattribution.go +++ b/internal/workerstats/actorattribution.go @@ -21,8 +21,8 @@ package workerstats import ( - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/resources" ) // attributionSource is the attribution-bearing subset of the worker requests diff --git a/internal/workerstats/actorattribution_test.go b/internal/workerstats/actorattribution_test.go index 51163f9bf4..7233aeee6a 100644 --- a/internal/workerstats/actorattribution_test.go +++ b/internal/workerstats/actorattribution_test.go @@ -19,8 +19,8 @@ import ( "github.com/google/go-cmp/cmp" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/resources" ) // fullAttribution is what every fully-populated request below should produce. The diff --git a/internal/workerstats/usage.go b/internal/workerstats/usage.go index 7f20843aff..0c2e944cb0 100644 --- a/internal/workerstats/usage.go +++ b/internal/workerstats/usage.go @@ -24,11 +24,11 @@ import ( "go.opentelemetry.io/otel/log/noop" sdklog "go.opentelemetry.io/otel/sdk/log" - "github.com/agent-substrate/substrate/internal/actorevent" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/contextlogging" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" - "github.com/agent-substrate/substrate/internal/resources" + "go.ate.dev/substrate/internal/actorevent" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/contextlogging" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/resources" ) // Pool is the WorkerPool of the worker's pod. diff --git a/internal/workerstats/usage_test.go b/internal/workerstats/usage_test.go index 41aad1f8a3..bddcabe5b6 100644 --- a/internal/workerstats/usage_test.go +++ b/internal/workerstats/usage_test.go @@ -23,9 +23,9 @@ import ( "testing/synctest" "time" - "github.com/agent-substrate/substrate/internal/actorevent" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/actorevent" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/proto/ateworkerpb" ) var testPool = Pool{Namespace: "team-a", Name: "default"} diff --git a/internal/workerstats/workerstatstest/workerstatstest.go b/internal/workerstats/workerstatstest/workerstatstest.go index 55d4d3940d..92bb8f6499 100644 --- a/internal/workerstats/workerstatstest/workerstatstest.go +++ b/internal/workerstats/workerstatstest/workerstatstest.go @@ -20,8 +20,8 @@ import ( "log/slog" "sync" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/workerstats" + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/workerstats" ) // Recorder is a stdout handler that keeps the kind and source of each record. diff --git a/manifests/ate-install/ate-api-config.yaml b/manifests/ate-install/ate-api-config.yaml new file mode 100644 index 0000000000..e8132bc340 --- /dev/null +++ b/manifests/ate-install/ate-api-config.yaml @@ -0,0 +1,25 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `hack/render-manifests.sh` to regenerate. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: ate-api-config + namespace: ate-system +data: + apiconfig.textproto: | + logging: { level: "info" } diff --git a/manifests/ate-install/ate-api-server.yaml b/manifests/ate-install/ate-api-server.yaml index bc2beac704..b3fcaab851 100644 --- a/manifests/ate-install/ate-api-server.yaml +++ b/manifests/ate-install/ate-api-server.yaml @@ -89,7 +89,7 @@ spec: # starts only once the startup probe sees the plugin serving. initContainers: - name: objectstore-plugin - image: ko://github.com/agent-substrate/substrate/cmd/objectstore-plugin + image: ko://go.ate.dev/substrate/cmd/objectstore-plugin restartPolicy: Always args: - control @@ -126,9 +126,10 @@ spec: mountPath: /run/objectstore-plugin containers: - name: ate-api-server - image: ko://github.com/agent-substrate/substrate/cmd/ateapi + image: ko://go.ate.dev/substrate/cmd/ateapi args: - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" + - --config-file=/run/apiconfig.ate.dev/apiconfig.textproto - --authentication-config=/etc/ateapi/authentication/authentication.yaml - --postgres-read-write-connection-string=@env - --postgres-owner-connection-string=@env @@ -204,6 +205,9 @@ spec: - name: "actor-id-ca-pool" mountPath: "/run/actor-id-ca-pool" readOnly: true + - name: "api-config" + mountPath: "/run/apiconfig.ate.dev" + readOnly: true - name: "authentication-config" mountPath: "/etc/ateapi/authentication" readOnly: true @@ -315,6 +319,9 @@ spec: items: - key: "pool" path: "pool.json" + - name: "api-config" + configMap: + name: "ate-api-config" - name: "authentication-config" configMap: name: "ate-api-authentication" diff --git a/manifests/ate-install/ate-controller.yaml b/manifests/ate-install/ate-controller.yaml index 7dfd673f6d..c1464e01a8 100644 --- a/manifests/ate-install/ate-controller.yaml +++ b/manifests/ate-install/ate-controller.yaml @@ -101,7 +101,7 @@ spec: serviceAccountName: ate-controller containers: - name: ate-controller - image: ko://github.com/agent-substrate/substrate/cmd/atecontroller + image: ko://go.ate.dev/substrate/cmd/atecontroller args: - --ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem - --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem diff --git a/manifests/ate-install/atelet.yaml b/manifests/ate-install/atelet.yaml index 4db08c5760..7ff7371dd3 100644 --- a/manifests/ate-install/atelet.yaml +++ b/manifests/ate-install/atelet.yaml @@ -34,12 +34,12 @@ rules: # SandboxConfigs are watched to pre-download sandbox assets into the node's # cache before the first actor needs them (see cmd/atelet/sandbox_prewarm.go), # and to write the runtimes of the configs local workers use to each worker's -# sandbox versions file (see cmd/atelet/worker_sandbox_syncer.go). +# sandbox runtimes file (see cmd/atelet/worker_sandbox_syncer.go). - apiGroups: ["ate.dev"] resources: ["sandboxconfigs"] verbs: ["get", "list", "watch"] # WorkerPools are watched to find the SandboxConfig each local worker's pool -# uses, whose versions atelet writes to the worker's sandbox versions file +# uses, whose versions atelet writes to the worker's sandbox runtimes file # (see cmd/atelet/worker_sandbox_syncer.go). - apiGroups: ["ate.dev"] resources: ["workerpools"] @@ -149,7 +149,7 @@ spec: # atelet starts only once the startup probe sees the plugin serving. initContainers: - name: objectstore-plugin - image: ko://github.com/agent-substrate/substrate/cmd/objectstore-plugin + image: ko://go.ate.dev/substrate/cmd/objectstore-plugin restartPolicy: Always args: - node @@ -207,7 +207,7 @@ spec: mountPath: /var/lib/objectstore-plugin/asset-staging containers: - name: atelet - image: ko://github.com/agent-substrate/substrate/cmd/atelet + image: ko://go.ate.dev/substrate/cmd/atelet args: # Pull credentials come from the node's own kubelet credential provider # plugin (mounted read-only below) rather than a cloud SDK compiled into diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index f2672cfeef..7b266df961 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -406,6 +406,30 @@ data: # resolves and what both routes verify the origin against. access_log: - name: envoy.access_loggers.file + # Every actor request lands here, so log the ones that went + # wrong, including gRPC errors sent with HTTP 200, and a sample + # of the rest. The runtime keys let an operator turn full + # logging back on at the admin port: + # /runtime_modify?access_log.min_status=0. + filter: + or_filter: + filters: + - status_code_filter: + comparison: + op: GE + value: {default_value: 400, runtime_key: access_log.min_status} + - response_flag_filter: {} + - and_filter: + filters: + - header_filter: + header: + name: content-type + string_match: {prefix: application/grpc} + - grpc_status_filter: {exclude: true, statuses: [OK]} + - runtime_filter: + runtime_key: access_log.sample + percent_sampled: {numerator: 1, denominator: HUNDRED} + use_independent_randomness: true typed_config: "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog path: /dev/stdout @@ -641,6 +665,30 @@ data: stream_idle_timeout: 0s access_log: - name: envoy.access_loggers.file + # Every actor request lands here, so log the ones that went + # wrong, including gRPC errors sent with HTTP 200, and a sample + # of the rest. The runtime keys let an operator turn full + # logging back on at the admin port: + # /runtime_modify?access_log.min_status=0. + filter: + or_filter: + filters: + - status_code_filter: + comparison: + op: GE + value: {default_value: 400, runtime_key: access_log.min_status} + - response_flag_filter: {} + - and_filter: + filters: + - header_filter: + header: + name: content-type + string_match: {prefix: application/grpc} + - grpc_status_filter: {exclude: true, statuses: [OK]} + - runtime_filter: + runtime_key: access_log.sample + percent_sampled: {numerator: 1, denominator: HUNDRED} + use_independent_randomness: true typed_config: "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog path: /dev/stdout @@ -1228,7 +1276,7 @@ spec: # staying two CONTAINERS is what keeps the MITM signing key off the # data plane and gives the minter the separate memory limit the header # of this file depends on. - image: ko://github.com/agent-substrate/substrate/cmd/atenet + image: ko://go.ate.dev/substrate/cmd/atenet restartPolicy: Always args: - "sdsmint" @@ -1396,7 +1444,7 @@ spec: # Envoy calls it over localhost to # authenticate actor identity against the ate API on every CONNECT. - name: ext-proc - image: ko://github.com/agent-substrate/substrate/cmd/atenet + image: ko://go.ate.dev/substrate/cmd/atenet args: - router - --mode=egress diff --git a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml index 3310864aac..638864efc2 100644 --- a/manifests/ate-install/atenet-router.yaml +++ b/manifests/ate-install/atenet-router.yaml @@ -201,7 +201,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: atenet-router - image: ko://github.com/agent-substrate/substrate/cmd/atenet + image: ko://go.ate.dev/substrate/cmd/atenet args: - "router" # Serve the ingress direction only: the ingress ext_proc handler, plus @@ -321,7 +321,8 @@ spec: - name: "drain-signal" mountPath: "/var/run/atenet" - name: envoy - image: envoyproxy/envoy:v1.39.4@sha256:74bd4eb5629708d3fadca1a5e6b85d208f338d7bb54e06b69565905c5294cc01 + # ENVOY_DATAPLANE_IMAGE is substituted for the image URI by ate-setup + image: ${ENVOY_DATAPLANE_IMAGE} command: - "/usr/local/bin/envoy" - "-c" @@ -361,7 +362,8 @@ spec: # podidentity credential + trust bundles for actor-facing (upstream) # mTLS. Envoy presents this credential bundle as its client cert when # opening CONNECT tunnels to the worker's atunnel on :8443, and - # validates that server against the trust bundle. + # validates that server against the trust bundle and the SPIFFE ID of + # the worker pod's namespace and ServiceAccount. # The issued SPIFFE identity is # spiffe://cluster.local/ns/ate-system/sa/atenet-router, which atunnel's # --atunnel-client-identity allows. diff --git a/manifests/ate-install/components/fake-data-plane/fake-workersync.yaml b/manifests/ate-install/components/fake-data-plane/fake-workersync.yaml index e87f9e614a..db503fd5bd 100644 --- a/manifests/ate-install/components/fake-data-plane/fake-workersync.yaml +++ b/manifests/ate-install/components/fake-data-plane/fake-workersync.yaml @@ -130,7 +130,7 @@ spec: serviceAccountName: fake-workersync containers: - name: fake-workersync - image: ko://github.com/agent-substrate/substrate/cmd/benchmarking/isolate/fake-workersync + image: ko://go.ate.dev/substrate/cmd/benchmarking/isolate/fake-workersync # Ready once it has adopted the registered fake Workers, which needs # ate-api-server serving. readinessProbe: diff --git a/manifests/ate-install/components/fake-data-plane/kustomization.yaml b/manifests/ate-install/components/fake-data-plane/kustomization.yaml index fa0691590b..dcd5f803eb 100644 --- a/manifests/ate-install/components/fake-data-plane/kustomization.yaml +++ b/manifests/ate-install/components/fake-data-plane/kustomization.yaml @@ -56,7 +56,7 @@ patches: path: /spec/template/spec/initContainers - op: replace path: /spec/template/spec/containers/0/image - value: ko://github.com/agent-substrate/substrate/cmd/benchmarking/isolate/fake-atelet + value: ko://go.ate.dev/substrate/cmd/benchmarking/isolate/fake-atelet # fake-atelet runs with its defaults. A benchmark tunes it through the # fake-atelet-tuning ConfigMap, which no install creates or resets. - op: remove diff --git a/manifests/ate-install/generated/ate.dev_workerpools.yaml b/manifests/ate-install/generated/ate.dev_workerpools.yaml index 7d07f9c432..a29c928aab 100644 --- a/manifests/ate-install/generated/ate.dev_workerpools.yaml +++ b/manifests/ate-install/generated/ate.dev_workerpools.yaml @@ -435,8 +435,8 @@ spec: serviceAccountName: description: |- ServiceAccountName for the worker pods. The ServiceAccount must exist - in the WorkerPool's namespace. When omitted, the pods use the - namespace's default ServiceAccount. + in the WorkerPool's namespace. When omitted, the pods run as the + "ateworker" ServiceAccount, which the controller creates. minLength: 1 type: string tolerations: diff --git a/manifests/ate-install/generated/role.yaml b/manifests/ate-install/generated/role.yaml index 7614dac1f4..3298e88aa2 100644 --- a/manifests/ate-install/generated/role.yaml +++ b/manifests/ate-install/generated/role.yaml @@ -35,6 +35,13 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - patch - apiGroups: - apps resources: diff --git a/manifests/ate-install/pod-certificate-controller.yaml b/manifests/ate-install/pod-certificate-controller.yaml index 23886e260e..fab1f38686 100644 --- a/manifests/ate-install/pod-certificate-controller.yaml +++ b/manifests/ate-install/pod-certificate-controller.yaml @@ -151,7 +151,7 @@ spec: spec: containers: - name: controller - image: ko://github.com/agent-substrate/substrate/cmd/podcertcontroller + image: ko://go.ate.dev/substrate/cmd/podcertcontroller resources: limits: cpu: 500m diff --git a/manifests/ate-install/role.yaml b/manifests/ate-install/role.yaml index 61e32b519a..f393cb2d9c 100644 --- a/manifests/ate-install/role.yaml +++ b/manifests/ate-install/role.yaml @@ -37,6 +37,13 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - patch - apiGroups: - apps resources: diff --git a/manifests/egress-credential-injection/k8s-credential-provider.yaml b/manifests/egress-credential-injection/k8s-credential-provider.yaml index 0f2d28d4de..8c75721668 100644 --- a/manifests/egress-credential-injection/k8s-credential-provider.yaml +++ b/manifests/egress-credential-injection/k8s-credential-provider.yaml @@ -69,7 +69,7 @@ spec: runAsNonRoot: true containers: - name: k8s-credential-provider - image: ko://github.com/agent-substrate/substrate/cmd/credential-provider/kubernetes-secrets + image: ko://go.ate.dev/substrate/cmd/credential-provider/kubernetes-secrets args: - "--listen-address=:50051" - "--metrics-address=:9090" diff --git a/pkg/api/v1alpha1/gen.go b/pkg/api/v1alpha1/gen.go index 87dfefe493..2dfe78a064 100644 --- a/pkg/api/v1alpha1/gen.go +++ b/pkg/api/v1alpha1/gen.go @@ -15,6 +15,6 @@ package v1alpha1 //go:generate bash ../../../hack/run-tool.sh controller-gen crd:headerFile=../../../hack/boilerplate/sh.txt object:headerFile=../../../hack/boilerplate/go.txt paths="./" output:crd:dir="../../../manifests/ate-install/generated/" -//go:generate bash ../../../hack/run-tool.sh client-gen --go-header-file=../../../hack/boilerplate/go.txt --clientset-name versioned --input-base github.com/agent-substrate/substrate --input pkg/api/v1alpha1 --output-pkg github.com/agent-substrate/substrate/pkg/client/clientset --output-dir ../../../pkg/client/clientset -//go:generate bash ../../../hack/run-tool.sh lister-gen --go-header-file=../../../hack/boilerplate/go.txt --output-pkg github.com/agent-substrate/substrate/pkg/client/listers --output-dir ../../../pkg/client/listers github.com/agent-substrate/substrate/pkg/api/v1alpha1 -//go:generate bash ../../../hack/run-tool.sh informer-gen --go-header-file=../../../hack/boilerplate/go.txt --versioned-clientset-package github.com/agent-substrate/substrate/pkg/client/clientset/versioned --listers-package github.com/agent-substrate/substrate/pkg/client/listers --output-pkg github.com/agent-substrate/substrate/pkg/client/informers --output-dir ../../../pkg/client/informers github.com/agent-substrate/substrate/pkg/api/v1alpha1 +//go:generate bash ../../../hack/run-tool.sh client-gen --go-header-file=../../../hack/boilerplate/go.txt --clientset-name versioned --input-base go.ate.dev/substrate --input pkg/api/v1alpha1 --output-pkg go.ate.dev/substrate/pkg/client/clientset --output-dir ../../../pkg/client/clientset +//go:generate bash ../../../hack/run-tool.sh lister-gen --go-header-file=../../../hack/boilerplate/go.txt --output-pkg go.ate.dev/substrate/pkg/client/listers --output-dir ../../../pkg/client/listers go.ate.dev/substrate/pkg/api/v1alpha1 +//go:generate bash ../../../hack/run-tool.sh informer-gen --go-header-file=../../../hack/boilerplate/go.txt --versioned-clientset-package go.ate.dev/substrate/pkg/client/clientset/versioned --listers-package go.ate.dev/substrate/pkg/client/listers --output-pkg go.ate.dev/substrate/pkg/client/informers --output-dir ../../../pkg/client/informers go.ate.dev/substrate/pkg/api/v1alpha1 diff --git a/pkg/api/v1alpha1/main_test.go b/pkg/api/v1alpha1/main_test.go index b4e2c6dab9..6a087530d5 100644 --- a/pkg/api/v1alpha1/main_test.go +++ b/pkg/api/v1alpha1/main_test.go @@ -19,7 +19,7 @@ import ( "os" "testing" - "github.com/agent-substrate/substrate/internal/testenv" + "go.ate.dev/substrate/internal/testenv" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" diff --git a/pkg/api/v1alpha1/sandboxconfig_types.go b/pkg/api/v1alpha1/sandboxconfig_types.go index 25c8dd863c..8abbf96c8c 100644 --- a/pkg/api/v1alpha1/sandboxconfig_types.go +++ b/pkg/api/v1alpha1/sandboxconfig_types.go @@ -17,7 +17,7 @@ package v1alpha1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/agent-substrate/substrate/pkg/sandboxclass" + "go.ate.dev/substrate/pkg/sandboxclass" ) // SandboxClass selects the sandbox runtime family. It is shared by WorkerPool diff --git a/pkg/api/v1alpha1/workerpool_types.go b/pkg/api/v1alpha1/workerpool_types.go index cb2e450510..20782b0b72 100644 --- a/pkg/api/v1alpha1/workerpool_types.go +++ b/pkg/api/v1alpha1/workerpool_types.go @@ -77,8 +77,8 @@ type WorkerPoolPodTemplate struct { Resources *corev1.ResourceRequirements `json:"resources,omitempty"` // ServiceAccountName for the worker pods. The ServiceAccount must exist - // in the WorkerPool's namespace. When omitted, the pods use the - // namespace's default ServiceAccount. + // in the WorkerPool's namespace. When omitted, the pods run as the + // "ateworker" ServiceAccount, which the controller creates. // // +optional // +kubebuilder:validation:MinLength=1 diff --git a/pkg/assetfetch/assetfetch.go b/pkg/assetfetch/assetfetch.go index 450a26917e..3480705ff7 100644 --- a/pkg/assetfetch/assetfetch.go +++ b/pkg/assetfetch/assetfetch.go @@ -42,7 +42,7 @@ import ( "regexp" "strings" - objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" "golang.org/x/sync/semaphore" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" diff --git a/pkg/assetfetch/assetfetch_test.go b/pkg/assetfetch/assetfetch_test.go index 9b8ee6006b..c9b3a24a73 100644 --- a/pkg/assetfetch/assetfetch_test.go +++ b/pkg/assetfetch/assetfetch_test.go @@ -29,8 +29,8 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/resources" - objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "go.ate.dev/substrate/internal/resources" + objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) diff --git a/pkg/client/clientset/versioned/clientset.go b/pkg/client/clientset/versioned/clientset.go index fd34793756..1b037f499c 100644 --- a/pkg/client/clientset/versioned/clientset.go +++ b/pkg/client/clientset/versioned/clientset.go @@ -20,7 +20,7 @@ import ( fmt "fmt" http "net/http" - apiv1alpha1 "github.com/agent-substrate/substrate/pkg/client/clientset/versioned/typed/api/v1alpha1" + apiv1alpha1 "go.ate.dev/substrate/pkg/client/clientset/versioned/typed/api/v1alpha1" discovery "k8s.io/client-go/discovery" rest "k8s.io/client-go/rest" flowcontrol "k8s.io/client-go/util/flowcontrol" diff --git a/pkg/client/clientset/versioned/fake/clientset_generated.go b/pkg/client/clientset/versioned/fake/clientset_generated.go index f2650039d4..89549e4d95 100644 --- a/pkg/client/clientset/versioned/fake/clientset_generated.go +++ b/pkg/client/clientset/versioned/fake/clientset_generated.go @@ -17,9 +17,9 @@ package fake import ( - clientset "github.com/agent-substrate/substrate/pkg/client/clientset/versioned" - apiv1alpha1 "github.com/agent-substrate/substrate/pkg/client/clientset/versioned/typed/api/v1alpha1" - fakeapiv1alpha1 "github.com/agent-substrate/substrate/pkg/client/clientset/versioned/typed/api/v1alpha1/fake" + clientset "go.ate.dev/substrate/pkg/client/clientset/versioned" + apiv1alpha1 "go.ate.dev/substrate/pkg/client/clientset/versioned/typed/api/v1alpha1" + fakeapiv1alpha1 "go.ate.dev/substrate/pkg/client/clientset/versioned/typed/api/v1alpha1/fake" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" diff --git a/pkg/client/clientset/versioned/fake/register.go b/pkg/client/clientset/versioned/fake/register.go index e72d8098c9..1677e7f9dc 100644 --- a/pkg/client/clientset/versioned/fake/register.go +++ b/pkg/client/clientset/versioned/fake/register.go @@ -17,7 +17,7 @@ package fake import ( - apiv1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + apiv1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/client/clientset/versioned/scheme/register.go b/pkg/client/clientset/versioned/scheme/register.go index 83ce5d1275..38ea0d1a46 100644 --- a/pkg/client/clientset/versioned/scheme/register.go +++ b/pkg/client/clientset/versioned/scheme/register.go @@ -17,7 +17,7 @@ package scheme import ( - apiv1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + apiv1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/client/clientset/versioned/typed/api/v1alpha1/api_client.go b/pkg/client/clientset/versioned/typed/api/v1alpha1/api_client.go index 500c137f09..16c190a986 100644 --- a/pkg/client/clientset/versioned/typed/api/v1alpha1/api_client.go +++ b/pkg/client/clientset/versioned/typed/api/v1alpha1/api_client.go @@ -19,8 +19,8 @@ package v1alpha1 import ( http "net/http" - apiv1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - scheme "github.com/agent-substrate/substrate/pkg/client/clientset/versioned/scheme" + apiv1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + scheme "go.ate.dev/substrate/pkg/client/clientset/versioned/scheme" rest "k8s.io/client-go/rest" ) diff --git a/pkg/client/clientset/versioned/typed/api/v1alpha1/csidriverconfig.go b/pkg/client/clientset/versioned/typed/api/v1alpha1/csidriverconfig.go index 4abe3bb5b8..f03c92574f 100644 --- a/pkg/client/clientset/versioned/typed/api/v1alpha1/csidriverconfig.go +++ b/pkg/client/clientset/versioned/typed/api/v1alpha1/csidriverconfig.go @@ -19,8 +19,8 @@ package v1alpha1 import ( context "context" - apiv1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - scheme "github.com/agent-substrate/substrate/pkg/client/clientset/versioned/scheme" + apiv1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + scheme "go.ate.dev/substrate/pkg/client/clientset/versioned/scheme" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" diff --git a/pkg/client/clientset/versioned/typed/api/v1alpha1/fake/fake_api_client.go b/pkg/client/clientset/versioned/typed/api/v1alpha1/fake/fake_api_client.go index b70615df1f..dd733289f5 100644 --- a/pkg/client/clientset/versioned/typed/api/v1alpha1/fake/fake_api_client.go +++ b/pkg/client/clientset/versioned/typed/api/v1alpha1/fake/fake_api_client.go @@ -17,7 +17,7 @@ package fake import ( - v1alpha1 "github.com/agent-substrate/substrate/pkg/client/clientset/versioned/typed/api/v1alpha1" + v1alpha1 "go.ate.dev/substrate/pkg/client/clientset/versioned/typed/api/v1alpha1" rest "k8s.io/client-go/rest" testing "k8s.io/client-go/testing" ) diff --git a/pkg/client/clientset/versioned/typed/api/v1alpha1/fake/fake_csidriverconfig.go b/pkg/client/clientset/versioned/typed/api/v1alpha1/fake/fake_csidriverconfig.go index e8bc825422..8a24f5f679 100644 --- a/pkg/client/clientset/versioned/typed/api/v1alpha1/fake/fake_csidriverconfig.go +++ b/pkg/client/clientset/versioned/typed/api/v1alpha1/fake/fake_csidriverconfig.go @@ -17,8 +17,8 @@ package fake import ( - v1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - apiv1alpha1 "github.com/agent-substrate/substrate/pkg/client/clientset/versioned/typed/api/v1alpha1" + v1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + apiv1alpha1 "go.ate.dev/substrate/pkg/client/clientset/versioned/typed/api/v1alpha1" gentype "k8s.io/client-go/gentype" ) diff --git a/pkg/client/clientset/versioned/typed/api/v1alpha1/fake/fake_sandboxconfig.go b/pkg/client/clientset/versioned/typed/api/v1alpha1/fake/fake_sandboxconfig.go index a6a43e374b..dddc8ed559 100644 --- a/pkg/client/clientset/versioned/typed/api/v1alpha1/fake/fake_sandboxconfig.go +++ b/pkg/client/clientset/versioned/typed/api/v1alpha1/fake/fake_sandboxconfig.go @@ -17,8 +17,8 @@ package fake import ( - v1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - apiv1alpha1 "github.com/agent-substrate/substrate/pkg/client/clientset/versioned/typed/api/v1alpha1" + v1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + apiv1alpha1 "go.ate.dev/substrate/pkg/client/clientset/versioned/typed/api/v1alpha1" gentype "k8s.io/client-go/gentype" ) diff --git a/pkg/client/clientset/versioned/typed/api/v1alpha1/fake/fake_workerpool.go b/pkg/client/clientset/versioned/typed/api/v1alpha1/fake/fake_workerpool.go index 187051e242..9e4da458c1 100644 --- a/pkg/client/clientset/versioned/typed/api/v1alpha1/fake/fake_workerpool.go +++ b/pkg/client/clientset/versioned/typed/api/v1alpha1/fake/fake_workerpool.go @@ -17,8 +17,8 @@ package fake import ( - v1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - apiv1alpha1 "github.com/agent-substrate/substrate/pkg/client/clientset/versioned/typed/api/v1alpha1" + v1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + apiv1alpha1 "go.ate.dev/substrate/pkg/client/clientset/versioned/typed/api/v1alpha1" gentype "k8s.io/client-go/gentype" ) diff --git a/pkg/client/clientset/versioned/typed/api/v1alpha1/sandboxconfig.go b/pkg/client/clientset/versioned/typed/api/v1alpha1/sandboxconfig.go index 45c49226d0..6056bea5e8 100644 --- a/pkg/client/clientset/versioned/typed/api/v1alpha1/sandboxconfig.go +++ b/pkg/client/clientset/versioned/typed/api/v1alpha1/sandboxconfig.go @@ -19,8 +19,8 @@ package v1alpha1 import ( context "context" - apiv1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - scheme "github.com/agent-substrate/substrate/pkg/client/clientset/versioned/scheme" + apiv1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + scheme "go.ate.dev/substrate/pkg/client/clientset/versioned/scheme" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" diff --git a/pkg/client/clientset/versioned/typed/api/v1alpha1/workerpool.go b/pkg/client/clientset/versioned/typed/api/v1alpha1/workerpool.go index c1c2256fac..539f2be3a1 100644 --- a/pkg/client/clientset/versioned/typed/api/v1alpha1/workerpool.go +++ b/pkg/client/clientset/versioned/typed/api/v1alpha1/workerpool.go @@ -19,8 +19,8 @@ package v1alpha1 import ( context "context" - apiv1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - scheme "github.com/agent-substrate/substrate/pkg/client/clientset/versioned/scheme" + apiv1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + scheme "go.ate.dev/substrate/pkg/client/clientset/versioned/scheme" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" diff --git a/pkg/client/informers/externalversions/api/interface.go b/pkg/client/informers/externalversions/api/interface.go index d0a418b9ea..c994543342 100644 --- a/pkg/client/informers/externalversions/api/interface.go +++ b/pkg/client/informers/externalversions/api/interface.go @@ -17,8 +17,8 @@ package api import ( - v1alpha1 "github.com/agent-substrate/substrate/pkg/client/informers/externalversions/api/v1alpha1" - internalinterfaces "github.com/agent-substrate/substrate/pkg/client/informers/externalversions/internalinterfaces" + v1alpha1 "go.ate.dev/substrate/pkg/client/informers/externalversions/api/v1alpha1" + internalinterfaces "go.ate.dev/substrate/pkg/client/informers/externalversions/internalinterfaces" ) // Interface provides access to each of this group's versions. diff --git a/pkg/client/informers/externalversions/api/v1alpha1/csidriverconfig.go b/pkg/client/informers/externalversions/api/v1alpha1/csidriverconfig.go index d31546178e..ec26c674bc 100644 --- a/pkg/client/informers/externalversions/api/v1alpha1/csidriverconfig.go +++ b/pkg/client/informers/externalversions/api/v1alpha1/csidriverconfig.go @@ -20,10 +20,10 @@ import ( context "context" time "time" - pkgapiv1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - versioned "github.com/agent-substrate/substrate/pkg/client/clientset/versioned" - internalinterfaces "github.com/agent-substrate/substrate/pkg/client/informers/externalversions/internalinterfaces" - apiv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" + pkgapiv1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + versioned "go.ate.dev/substrate/pkg/client/clientset/versioned" + internalinterfaces "go.ate.dev/substrate/pkg/client/informers/externalversions/internalinterfaces" + apiv1alpha1 "go.ate.dev/substrate/pkg/client/listers/api/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/client/informers/externalversions/api/v1alpha1/interface.go b/pkg/client/informers/externalversions/api/v1alpha1/interface.go index 9bdd46f176..305eff420d 100644 --- a/pkg/client/informers/externalversions/api/v1alpha1/interface.go +++ b/pkg/client/informers/externalversions/api/v1alpha1/interface.go @@ -17,7 +17,7 @@ package v1alpha1 import ( - internalinterfaces "github.com/agent-substrate/substrate/pkg/client/informers/externalversions/internalinterfaces" + internalinterfaces "go.ate.dev/substrate/pkg/client/informers/externalversions/internalinterfaces" ) // Interface provides access to all the informers in this group version. diff --git a/pkg/client/informers/externalversions/api/v1alpha1/sandboxconfig.go b/pkg/client/informers/externalversions/api/v1alpha1/sandboxconfig.go index f41d4cd632..747ae5d1a1 100644 --- a/pkg/client/informers/externalversions/api/v1alpha1/sandboxconfig.go +++ b/pkg/client/informers/externalversions/api/v1alpha1/sandboxconfig.go @@ -20,10 +20,10 @@ import ( context "context" time "time" - pkgapiv1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - versioned "github.com/agent-substrate/substrate/pkg/client/clientset/versioned" - internalinterfaces "github.com/agent-substrate/substrate/pkg/client/informers/externalversions/internalinterfaces" - apiv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" + pkgapiv1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + versioned "go.ate.dev/substrate/pkg/client/clientset/versioned" + internalinterfaces "go.ate.dev/substrate/pkg/client/informers/externalversions/internalinterfaces" + apiv1alpha1 "go.ate.dev/substrate/pkg/client/listers/api/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/client/informers/externalversions/api/v1alpha1/workerpool.go b/pkg/client/informers/externalversions/api/v1alpha1/workerpool.go index 33357e1d18..27a9214c52 100644 --- a/pkg/client/informers/externalversions/api/v1alpha1/workerpool.go +++ b/pkg/client/informers/externalversions/api/v1alpha1/workerpool.go @@ -20,10 +20,10 @@ import ( context "context" time "time" - pkgapiv1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - versioned "github.com/agent-substrate/substrate/pkg/client/clientset/versioned" - internalinterfaces "github.com/agent-substrate/substrate/pkg/client/informers/externalversions/internalinterfaces" - apiv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" + pkgapiv1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + versioned "go.ate.dev/substrate/pkg/client/clientset/versioned" + internalinterfaces "go.ate.dev/substrate/pkg/client/informers/externalversions/internalinterfaces" + apiv1alpha1 "go.ate.dev/substrate/pkg/client/listers/api/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/client/informers/externalversions/factory.go b/pkg/client/informers/externalversions/factory.go index de9c7b410d..36d54285a2 100644 --- a/pkg/client/informers/externalversions/factory.go +++ b/pkg/client/informers/externalversions/factory.go @@ -22,9 +22,9 @@ import ( sync "sync" time "time" - versioned "github.com/agent-substrate/substrate/pkg/client/clientset/versioned" - api "github.com/agent-substrate/substrate/pkg/client/informers/externalversions/api" - internalinterfaces "github.com/agent-substrate/substrate/pkg/client/informers/externalversions/internalinterfaces" + versioned "go.ate.dev/substrate/pkg/client/clientset/versioned" + api "go.ate.dev/substrate/pkg/client/informers/externalversions/api" + internalinterfaces "go.ate.dev/substrate/pkg/client/informers/externalversions/internalinterfaces" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/client/informers/externalversions/generic.go b/pkg/client/informers/externalversions/generic.go index f46d04fd2a..0c14c958a4 100644 --- a/pkg/client/informers/externalversions/generic.go +++ b/pkg/client/informers/externalversions/generic.go @@ -19,7 +19,7 @@ package externalversions import ( fmt "fmt" - v1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + v1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" schema "k8s.io/apimachinery/pkg/runtime/schema" cache "k8s.io/client-go/tools/cache" ) diff --git a/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go b/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go index 4a02977dca..730a96413a 100644 --- a/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go +++ b/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go @@ -19,7 +19,7 @@ package internalinterfaces import ( time "time" - versioned "github.com/agent-substrate/substrate/pkg/client/clientset/versioned" + versioned "go.ate.dev/substrate/pkg/client/clientset/versioned" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" cache "k8s.io/client-go/tools/cache" diff --git a/pkg/client/listers/api/v1alpha1/csidriverconfig.go b/pkg/client/listers/api/v1alpha1/csidriverconfig.go index 5c92f456c8..2ce7f1f788 100644 --- a/pkg/client/listers/api/v1alpha1/csidriverconfig.go +++ b/pkg/client/listers/api/v1alpha1/csidriverconfig.go @@ -17,7 +17,7 @@ package v1alpha1 import ( - apiv1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + apiv1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" labels "k8s.io/apimachinery/pkg/labels" listers "k8s.io/client-go/listers" cache "k8s.io/client-go/tools/cache" diff --git a/pkg/client/listers/api/v1alpha1/sandboxconfig.go b/pkg/client/listers/api/v1alpha1/sandboxconfig.go index a198028b92..4b27c7230f 100644 --- a/pkg/client/listers/api/v1alpha1/sandboxconfig.go +++ b/pkg/client/listers/api/v1alpha1/sandboxconfig.go @@ -17,7 +17,7 @@ package v1alpha1 import ( - apiv1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + apiv1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" labels "k8s.io/apimachinery/pkg/labels" listers "k8s.io/client-go/listers" cache "k8s.io/client-go/tools/cache" diff --git a/pkg/client/listers/api/v1alpha1/workerpool.go b/pkg/client/listers/api/v1alpha1/workerpool.go index c3400f9131..32569334b1 100644 --- a/pkg/client/listers/api/v1alpha1/workerpool.go +++ b/pkg/client/listers/api/v1alpha1/workerpool.go @@ -17,7 +17,7 @@ package v1alpha1 import ( - apiv1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + apiv1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" labels "k8s.io/apimachinery/pkg/labels" listers "k8s.io/client-go/listers" cache "k8s.io/client-go/tools/cache" diff --git a/pkg/objectstorage/gcssparse.go b/pkg/objectstorage/gcssparse.go index cb1ea0602f..b19f2f52a9 100644 --- a/pkg/objectstorage/gcssparse.go +++ b/pkg/objectstorage/gcssparse.go @@ -42,7 +42,7 @@ const partWriterChunk = 16 << 20 // The parts are consecutive byte ranges of the same sparse-extent stream — part 0 // carries the header, the last carries the end sentinel — so the composed object is // exactly what the single-stream writer would have produced. -func (g *gcsClient) PutSparseFile(ctx context.Context, bucket, object string, f *os.File) (res writeContentResult, err error) { +func (g *gcsClient) PutSparseFile(ctx context.Context, bucket, object string, f *os.File, maxParts int) (res writeContentResult, err error) { fi, err := f.Stat() if err != nil { return res, err @@ -54,7 +54,7 @@ func (g *gcsClient) PutSparseFile(ctx context.Context, bucket, object string, f } res = writeContentResult{logicalBytes: size, populatedBytes: populated, sparse: true} - n := sparsePartCount(populated) + n := min(sparsePartCount(populated), maxParts) if n < 2 { // Too small to be worth splitting; the plain streaming path handles it. return res, errSparseTooSmall @@ -88,7 +88,7 @@ func (g *gcsClient) PutSparseFile(ctx context.Context, bucket, object string, f obj := g.uploadClient(gctx, i).Bucket(bucket).Object(parts[i].ObjectName()) w := obj.NewWriter(gctx) w.ChunkSize = partWriterChunk - if err := writeSparsePart(w, f, size, ranges, i == 0, i == len(groups)-1); err != nil { + if err := writeSparsePart(contextWriter{ctx: gctx, dst: w}, f, size, ranges, i == 0, i == len(groups)-1); err != nil { _ = w.Close() return fmt.Errorf("while writing part %d of %q: %w", i, object, err) } diff --git a/pkg/objectstorage/objects.go b/pkg/objectstorage/objects.go index 6e697747cb..c7c696df97 100644 --- a/pkg/objectstorage/objects.go +++ b/pkg/objectstorage/objects.go @@ -24,7 +24,6 @@ import ( "log/slog" "net/url" "os" - "runtime" "strings" "time" @@ -130,22 +129,6 @@ func Open(ctx context.Context, client ObjectStorage, gsURL string) (io.ReadClose return rc, nil } -// SendBytesToGCS uploads the given bytes (uncompressed) to gsURL. Intended for -// small objects such as the snapshot manifest. -func SendBytesToGCS(ctx context.Context, client ObjectStorage, gsURL string, content []byte) error { - ctx, span := tracer.Start(ctx, "sendBytesToGCS") - defer span.End() - - bucket, object, err := parseGCSURL(gsURL) - if err != nil { - return fmt.Errorf("while parsing url: %w", err) - } - if err := client.PutObject(ctx, bucket, object, bytes.NewReader(content)); err != nil { - return fmt.Errorf("while putting object bucket=%q object=%q: %w", bucket, object, err) - } - return nil -} - func SendLocalFileToGCSWithZstd(ctx context.Context, client ObjectStorage, gsURL string, localFilePath string) (err error) { ctx, span := tracer.Start(ctx, "sendLocalFileToGCSWithZstd") defer span.End() @@ -171,24 +154,12 @@ func SendLocalFileToGCSWithZstd(ctx context.Context, client ObjectStorage, gsURL return nil } -// SendFileToGCSWithZstd compresses and uploads an already-open file. The -// caller retains ownership of localFile. -func SendFileToGCSWithZstd(ctx context.Context, client ObjectStorage, gsURL string, localFile *os.File) error { - ctx, span := tracer.Start(ctx, "sendFileToGCSWithZstd") - defer span.End() - - if err := sendZstd(ctx, client, gsURL, localFile); err != nil { - return fmt.Errorf("in sendZstd: %w", err) - } - return nil -} - // sparseFilePutter marks a backend that can upload a sparse FILE directly, splitting // compression and upload by file range instead of piping one compressed stream into a // part splitter. Reports errSparseTooSmall when the file is not worth splitting, in // which case the caller falls back to the streaming path. type sparseFilePutter interface { - PutSparseFile(ctx context.Context, bucket, object string, f *os.File) (writeContentResult, error) + PutSparseFile(ctx context.Context, bucket, object string, f *os.File, maxParts int) (writeContentResult, error) } // streamingPutter marks an ObjectStorage whose PutObject accepts a non-seekable @@ -212,15 +183,15 @@ type writeContentResult struct { // seekable *os.File (compress only the populated extents, skip the holes) or a // plain zstd stream otherwise. It touches only io, so it is unit-testable without // an object store, and is shared by the buffered and streaming upload paths. -func writeContent(out io.Writer, content io.Reader) (writeContentResult, error) { +func writeContent(ctx context.Context, out io.Writer, content io.Reader, workers int) (writeContentResult, error) { if f, ok := content.(*os.File); ok { - logical, populated, err := writeSparseZstd(out, f) + logical, populated, err := writeSparseZstd(ctx, out, f, workers) if err != nil { return writeContentResult{}, err } return writeContentResult{logicalBytes: logical, populatedBytes: populated, sparse: true}, nil } - logical, err := plainZstd(out, content) + logical, err := plainZstd(ctx, out, content, workers) if err != nil { return writeContentResult{}, err } @@ -230,27 +201,40 @@ func writeContent(out io.Writer, content io.Reader) (writeContentResult, error) // sendZstd zstd-compresses content and uploads it to gsURL. // // The snapshot memory-ranges is the large object here (the whole guest RAM image, -// mostly zero) on the SUSPEND critical path, so we compress with SpeedFastest across -// all CPUs — high-ratio levels scan the multi-GiB image far slower for little size -// gain on near-zero data, and the decoder auto-detects the level so restore + older +// mostly zero) on the SUSPEND critical path, so we compress with SpeedFastest within +// the shared upload memory budget. High-ratio levels scan the multi-GiB image +// far slower for little size gain on near-zero data, and the decoder auto-detects the level so restore + older // snapshots are unaffected. // // Upload strategy depends on the backend: -// - Streaming backends (GCS) accept a non-seekable body, so we pipe the compressor +// - Streaming backends (GCS and S3) accept a non-seekable body, so we pipe the compressor // straight into PutObject: the compress overlaps the network PUT and we never // stage the ~100MiB compressed payload to a temp file. -// - S3/rustfs PutObject hands the body to the AWS SDK, which needs a seekable body -// to sign + set Content-Length (a non-seekable pipe hangs there), so we compress -// to a SEEKABLE temp file first. +// - Other backends receive a seekable temporary file containing the compressed +// content. func sendZstd(ctx context.Context, client ObjectStorage, gsURL string, content io.Reader) error { + u := &Uploader{client: client, budget: defaultUploadBudget} + return u.sendZstd(ctx, gsURL, content) +} + +func (u *Uploader) sendZstd(ctx context.Context, gsURL string, content io.Reader) error { + client := u.client bucket, object, err := parseGCSURL(gsURL) if err != nil { return fmt.Errorf("while parsing URL: %w", err) } + plan, err := u.plan(content) + if err != nil { + return err + } + if err := u.budget.reserved.Acquire(ctx, plan.memory); err != nil { + return fmt.Errorf("waiting for upload memory: %w", err) + } + defer u.budget.reserved.Release(plan.memory) tStart := time.Now() if sp, ok := client.(sparseFilePutter); ok { if f, isFile := content.(*os.File); isFile { - res, err := sp.PutSparseFile(ctx, bucket, object, f) + res, err := sp.PutSparseFile(ctx, bucket, object, f, plan.parts) switch { case err == nil: slog.InfoContext(ctx, "Compressed zstd upload", @@ -270,15 +254,15 @@ func sendZstd(ctx context.Context, client ObjectStorage, gsURL string, content i } } if _, ok := client.(streamingPutter); ok { - return sendStreamingZstd(ctx, client, bucket, object, content, tStart) + return sendStreamingZstd(ctx, client, bucket, object, content, tStart, plan.workers) } - return sendBufferedZstd(ctx, client, bucket, object, content, tStart) + return sendBufferedZstd(ctx, client, bucket, object, content, tStart, plan.workers) } // sendBufferedZstd compresses content to a seekable temp file, then uploads it. -// Used for backends (S3/rustfs) whose PutObject needs a seekable body to sign and -// set Content-Length; the streaming counterpart is sendStreamingZstd. -func sendBufferedZstd(ctx context.Context, client ObjectStorage, bucket, object string, content io.Reader, tStart time.Time) error { +// Used for backends that require a seekable body; the streaming counterpart is +// sendStreamingZstd. +func sendBufferedZstd(ctx context.Context, client ObjectStorage, bucket, object string, content io.Reader, tStart time.Time, workers int) error { tmpFile, err := os.CreateTemp("", "substrate-upload-compress-") if err != nil { return fmt.Errorf("while creating temp compress file: %w", err) @@ -287,7 +271,7 @@ func sendBufferedZstd(ctx context.Context, client ObjectStorage, bucket, object defer tmpFile.Close() t0 := time.Now() - res, err := writeContent(tmpFile, content) + res, err := writeContent(ctx, tmpFile, content, workers) if err != nil { return fmt.Errorf("while compressing %q: %w", object, err) } @@ -310,8 +294,10 @@ func sendBufferedZstd(ctx context.Context, client ObjectStorage, bucket, object // goroutine writes the (sparse-extent or plain) zstd stream into an io.Pipe while // PutObject streams the read end to the object store. No seekable temp file, and // the compress runs concurrently with the network PUT. Used only for streaming -// backends (GCS); see sendZstd. -func sendStreamingZstd(ctx context.Context, client ObjectStorage, bucket, object string, content io.Reader, tStart time.Time) error { +// backends (GCS and S3); see sendZstd. +func sendStreamingZstd(ctx context.Context, client ObjectStorage, bucket, object string, content io.Reader, tStart time.Time, workers int) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() type result struct { res writeContentResult err error @@ -319,7 +305,7 @@ func sendStreamingZstd(ctx context.Context, client ObjectStorage, bucket, object pr, pw := io.Pipe() ch := make(chan result, 1) go func() { - res, err := writeContent(pw, content) + res, err := writeContent(ctx, pw, content, workers) // Closing the writer delivers EOF (or the compress error) to PutObject. _ = pw.CloseWithError(err) ch <- result{res: res, err: err} @@ -327,6 +313,7 @@ func sendStreamingZstd(ctx context.Context, client ObjectStorage, bucket, object putErr := client.PutObject(ctx, bucket, object, pr) if putErr != nil { + cancel() // PutObject bailed (e.g. mid-stream); unblock the compressor goroutine so it // can finish and we don't deadlock on the channel receive below. _ = pr.CloseWithError(putErr) @@ -345,16 +332,16 @@ func sendStreamingZstd(ctx context.Context, client ObjectStorage, bucket, object return nil } -// plainZstd writes src to w as a single plain zstd stream (SpeedFastest, all -// cores) and returns the uncompressed byte count. -func plainZstd(w io.Writer, src io.Reader) (int64, error) { +// plainZstd writes src to w as a single plain zstd stream and returns the +// uncompressed byte count, using only the reserved compression workers. +func plainZstd(ctx context.Context, w io.Writer, src io.Reader, workers int) (int64, error) { zw, err := zstd.NewWriter(w, zstd.WithEncoderLevel(zstd.SpeedFastest), - zstd.WithEncoderConcurrency(runtime.GOMAXPROCS(0))) + zstd.WithEncoderConcurrency(workers)) if err != nil { return 0, err } - n, err := io.Copy(zw, src) + n, err := io.Copy(contextWriter{ctx: ctx, dst: zw}, src) if err != nil { zw.Close() return n, err diff --git a/pkg/objectstorage/objects_test.go b/pkg/objectstorage/objects_test.go index 53ae63fdd7..4b74497962 100644 --- a/pkg/objectstorage/objects_test.go +++ b/pkg/objectstorage/objects_test.go @@ -72,7 +72,7 @@ func (s *closeErrorStore) GetObject(ctx context.Context, bucket, object string) func TestFetchFromGCSWithZstdCloseError(t *testing.T) { var compressed bytes.Buffer - if _, err := writeContent(&compressed, bytes.NewBufferString("snapshot payload")); err != nil { + if _, err := writeContent(context.Background(), &compressed, bytes.NewBufferString("snapshot payload"), 2); err != nil { t.Fatal(err) } closeErr := errors.New("object reader close failed") @@ -253,7 +253,7 @@ func TestSparseVersionRejected(t *testing.T) { t.Fatal(err) } var buf bytes.Buffer - if _, _, err := writeSparseZstd(&buf, f); err != nil { + if _, _, err := writeSparseZstd(context.Background(), &buf, f, 2); err != nil { t.Fatalf("writeSparseZstd: %v", err) } f.Close() @@ -307,7 +307,7 @@ func TestWriteSparseZstdSkipsHoles(t *testing.T) { t.Fatal(err) } var buf bytes.Buffer - logical, dataBytes, err := writeSparseZstd(&buf, src) + logical, dataBytes, err := writeSparseZstd(context.Background(), &buf, src, 2) if err != nil { t.Fatalf("writeSparseZstd: %v", err) } @@ -328,8 +328,7 @@ func TestPlainZstdBackwardCompatRoundTrip(t *testing.T) { store := newMemStore() ctx := context.Background() const gsURL = "gs://bucket/snap/config.json.zstd" - // SendBytesToGCS is uncompressed; use sendZstd with a non-file reader to - // hit the plain-zstd branch (no magic). + // Use a non-file reader to hit the plain-zstd branch (no magic). if err := sendZstd(ctx, store, gsURL, bytes.NewReader(want)); err != nil { t.Fatalf("upload: %v", err) } @@ -451,7 +450,7 @@ func TestWriteDecodeContentRoundTrip(t *testing.T) { } var buf bytes.Buffer - wres, err := writeContent(&buf, src) + wres, err := writeContent(context.Background(), &buf, src, 2) if err != nil { t.Fatalf("writeContent: %v", err) } @@ -495,7 +494,7 @@ func TestWriteDecodeContentRoundTrip(t *testing.T) { t.Run("plain reader", func(t *testing.T) { want := bytes.Repeat([]byte("substrate payload\n"), 1000) var buf bytes.Buffer - wres, err := writeContent(&buf, bytes.NewReader(want)) + wres, err := writeContent(context.Background(), &buf, bytes.NewReader(want), 2) if err != nil { t.Fatalf("writeContent: %v", err) } diff --git a/pkg/objectstorage/parzstd.go b/pkg/objectstorage/parzstd.go index 5b8a91f465..894017c3aa 100644 --- a/pkg/objectstorage/parzstd.go +++ b/pkg/objectstorage/parzstd.go @@ -17,6 +17,7 @@ package objectstorage import ( "io" "runtime" + "sync" "github.com/klauspost/compress/zstd" ) @@ -47,12 +48,13 @@ type parZstd struct { dst io.Writer workers int - buf []byte - free chan []byte - jobs chan parZstdJob - ordered chan chan []byte - done chan struct{} - err error + buf []byte + free chan []byte + jobs chan parZstdJob + ordered chan chan []byte + done chan struct{} + err error + workerWG sync.WaitGroup } type parZstdJob struct { @@ -77,7 +79,7 @@ func newParZstd(dst io.Writer, workers int) *parZstd { p.free <- make([]byte, 0, parZstdChunk) } for range workers { - go p.worker() + p.workerWG.Go(p.worker) } go p.writer() p.buf = <-p.free @@ -142,6 +144,7 @@ func (p *parZstd) Close() error { close(p.jobs) close(p.ordered) <-p.done + p.workerWG.Wait() return p.err } diff --git a/pkg/objectstorage/sparseparts_test.go b/pkg/objectstorage/sparseparts_test.go index 02b9871d5b..fdaf959b0a 100644 --- a/pkg/objectstorage/sparseparts_test.go +++ b/pkg/objectstorage/sparseparts_test.go @@ -16,6 +16,7 @@ package objectstorage import ( "bytes" + "context" "io" "math/rand" "os" @@ -57,7 +58,7 @@ func TestSparsePartsRoundTrip(t *testing.T) { // The parts must land the same bytes as the single-stream writer. var want bytes.Buffer - if _, _, err := writeSparseZstd(&want, src); err != nil { + if _, _, err := writeSparseZstd(context.Background(), &want, src, 2); err != nil { t.Fatal(err) } if g, w := decodeSparse(t, &got), decodeSparse(t, &want); !bytes.Equal(g, w) { diff --git a/pkg/objectstorage/sparsezstd.go b/pkg/objectstorage/sparsezstd.go index dc78ddd6d6..05b18643e0 100644 --- a/pkg/objectstorage/sparsezstd.go +++ b/pkg/objectstorage/sparsezstd.go @@ -16,6 +16,7 @@ package objectstorage import ( "bufio" + "context" "encoding/binary" "fmt" "io" @@ -59,7 +60,7 @@ const sparseEndOffset int64 = -1 // compress from "scan the whole logical image" (e.g. 2GiB) to "scan the resident // set" (e.g. ~150MiB). Returns the logical size and the populated (pre-compression) // byte count. All integers are little-endian. -func writeSparseZstd(dst io.Writer, src *os.File) (logical, dataBytes int64, err error) { +func writeSparseZstd(ctx context.Context, dst io.Writer, src *os.File, workers int) (logical, dataBytes int64, err error) { fi, err := src.Stat() if err != nil { return 0, 0, err @@ -78,14 +79,14 @@ func writeSparseZstd(dst io.Writer, src *os.File) (logical, dataBytes int64, err return 0, 0, err } - // One worker per chunk the file can fill, so a small object does not pay for a pool. - zw := newParZstd(dst, int(size/parZstdChunk)+1) + zw := newParZstd(dst, workers) + cw := contextWriter{ctx: ctx, dst: zw} // fail closes the encoder before returning err (Close flushes/frees state). fail := func(e error) (int64, int64, error) { zw.Close() return 0, 0, e } - if err := binary.Write(zw, binary.LittleEndian, size); err != nil { + if err := binary.Write(cw, binary.LittleEndian, size); err != nil { return fail(err) } @@ -104,23 +105,23 @@ func writeSparseZstd(dst io.Writer, src *os.File) (logical, dataBytes int64, err return fail(fmt.Errorf("SEEK_HOLE: %w", serr)) } length := de - ds - if err := binary.Write(zw, binary.LittleEndian, ds); err != nil { + if err := binary.Write(cw, binary.LittleEndian, ds); err != nil { return fail(err) } - if err := binary.Write(zw, binary.LittleEndian, length); err != nil { + if err := binary.Write(cw, binary.LittleEndian, length); err != nil { return fail(err) } if _, err := src.Seek(ds, io.SeekStart); err != nil { return fail(err) } - n, cerr := io.CopyN(zw, src, length) + n, cerr := io.CopyN(cw, src, length) dataBytes += n if cerr != nil { return fail(fmt.Errorf("reading extent @%d+%d: %w", ds, length, cerr)) } off = de } - if err := binary.Write(zw, binary.LittleEndian, sparseEndOffset); err != nil { + if err := binary.Write(cw, binary.LittleEndian, sparseEndOffset); err != nil { return fail(err) } if err := zw.Close(); err != nil { diff --git a/pkg/objectstorage/sparsezstd_test.go b/pkg/objectstorage/sparsezstd_test.go index d26928a0d4..5fbb146c11 100644 --- a/pkg/objectstorage/sparsezstd_test.go +++ b/pkg/objectstorage/sparsezstd_test.go @@ -16,6 +16,7 @@ package objectstorage import ( "bytes" + "context" "encoding/binary" "os" "path/filepath" @@ -91,7 +92,7 @@ func TestSparseZstdRoundTrip(t *testing.T) { defer src.Close() var buf bytes.Buffer - logical, _, err := writeSparseZstd(&buf, src) + logical, _, err := writeSparseZstd(context.Background(), &buf, src, 2) if err != nil { t.Fatalf("writeSparseZstd: %v", err) } diff --git a/pkg/objectstorage/upload.go b/pkg/objectstorage/upload.go new file mode 100644 index 0000000000..ce116acfd3 --- /dev/null +++ b/pkg/objectstorage/upload.go @@ -0,0 +1,169 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstorage + +import ( + "context" + "fmt" + "io" + "os" + "runtime" + + "golang.org/x/sync/semaphore" +) + +// DefaultUploadMemory is the shared reservation budget for snapshot compression +// and upload buffers. It is not a limit on the process's total memory. +const DefaultUploadMemory int64 = 512 << 20 + +const ( + // Each parallel encoder owns input buffers, queued compressed frames, and + // zstd state. Include the writer's frame and the producer's pending frame + // even when there is only one worker. + parallelWorkerMemory int64 = (parZstdQueue*3 + 2) * parZstdChunk + // A serial encoder has a 4 MiB window, history, blocks, and a copy buffer. + serialWorkerMemory int64 = 32 << 20 + sparseUploadMemory = serialWorkerMemory + partWriterChunk +) + +type uploadBudget struct { + capacity int64 + reserved *semaphore.Weighted +} + +var defaultUploadBudget = &uploadBudget{ + capacity: DefaultUploadMemory, + reserved: semaphore.NewWeighted(DefaultUploadMemory), +} + +// Uploader shares a memory reservation budget across concurrent uploads. Reuse +// one Uploader for all snapshot requests served by a node. Reservations cover +// compression and the GCS/S3 clients' buffering, and last until both finish. +// Other ObjectStorage implementations must bound their own upload buffers. +// Runtime overhead, downloads, and memory awaiting garbage collection are not +// covered by the reservation budget. +type Uploader struct { + client ObjectStorage + budget *uploadBudget +} + +// NewUploader creates an uploader with capacity bytes of reservations. Capacity +// must fit one compression worker plus the backend's upload buffers; larger files +// use fewer workers when necessary rather than waiting for an impossible request. +func NewUploader(client ObjectStorage, capacity int64) (*Uploader, error) { + minimum := uploadBufferMemory(client) + parallelWorkerMemory + if capacity < minimum { + return nil, fmt.Errorf("upload memory budget must be at least %d bytes for this backend", minimum) + } + return &Uploader{client: client, budget: &uploadBudget{ + capacity: capacity, + reserved: semaphore.NewWeighted(capacity), + }}, nil +} + +// uploadBufferMemory includes buffers retained while the compressor is running. +func uploadBufferMemory(client ObjectStorage) int64 { + switch client.(type) { + case *s3Client: + // The SDK holds one producer buffer in addition to the active parts. + return (uploadConcurrency + 1) * uploadPartSize + case *gcsClient: + // Composite uploads retain the lookahead, our part pool, and the SDK's + // copy of each part. Single uploads fit within this reservation too. + return uploadCompositeMin + 2*uploadConcurrency*uploadPartSize + default: + return 0 + } +} + +type uploadPlan struct { + workers int + parts int + memory int64 +} + +func (u *Uploader) plan(content io.Reader) (uploadPlan, error) { + buffers := uploadBufferMemory(u.client) + workers := min(int64(runtime.GOMAXPROCS(0)), (u.budget.capacity-buffers)/parallelWorkerMemory) + parts := 1 + if f, ok := content.(*os.File); ok { + info, err := f.Stat() + if err != nil { + return uploadPlan{}, err + } + position, err := f.Seek(0, io.SeekCurrent) + if err != nil { + return uploadPlan{}, err + } + _, populated, err := sparseExtents(f, info.Size()) + _, seekErr := f.Seek(position, io.SeekStart) + if err != nil { + return uploadPlan{}, err + } + if seekErr != nil { + return uploadPlan{}, seekErr + } + // Holes never enter the compressor or range uploads. + workers = min(workers, populated/parZstdChunk+1) + parts = min(sparsePartCount(populated), int(u.budget.capacity/sparseUploadMemory)) + } + memory := buffers + workers*parallelWorkerMemory + if _, ok := u.client.(sparseFilePutter); ok { + // The range-parallel path and streaming fallback are alternatives, + // so reserve the larger requirement rather than adding both. + memory = max(memory, int64(parts)*sparseUploadMemory) + } + return uploadPlan{workers: int(workers), parts: parts, memory: memory}, nil +} + +// SendFile compresses and uploads an already-open snapshot file. The caller keeps +// ownership of the file. Waiting for memory is canceled with ctx. +func (u *Uploader) SendFile(ctx context.Context, url string, file *os.File) error { + ctx, span := tracer.Start(ctx, "sendFileToGCSWithZstd") + defer span.End() + return u.sendZstd(ctx, url, file) +} + +// Send uploads uncompressed content, using the same budget as SendFile. This is +// used for snapshot manifests; the reader is streamed without retaining its body. +func (u *Uploader) Send(ctx context.Context, url string, content io.Reader) error { + ctx, span := tracer.Start(ctx, "sendBytesToGCS") + defer span.End() + + bucket, object, err := parseGCSURL(url) + if err != nil { + return err + } + memory := uploadBufferMemory(u.client) + if err := u.budget.reserved.Acquire(ctx, memory); err != nil { + return err + } + defer u.budget.reserved.Release(memory) + return u.client.PutObject(ctx, bucket, object, content) +} + +// contextWriter stops compression between writes even after an asynchronous +// destination has failed and is draining queued frames. +type contextWriter struct { + ctx context.Context + dst io.Writer +} + +func (w contextWriter) Write(b []byte) (int, error) { + if err := w.ctx.Err(); err != nil { + return 0, err + } + return w.dst.Write(b) +} diff --git a/pkg/objectstorage/upload_bench_test.go b/pkg/objectstorage/upload_bench_test.go new file mode 100644 index 0000000000..40beb8ae93 --- /dev/null +++ b/pkg/objectstorage/upload_bench_test.go @@ -0,0 +1,124 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstorage + +import ( + "context" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "runtime" + "sync" + "sync/atomic" + "testing" + "time" +) + +type delayedSnapshotStore struct{} + +func (delayedSnapshotStore) supportsStreamingPut() {} +func (delayedSnapshotStore) GetObject(context.Context, string, string) (io.ReadCloser, error) { + return nil, fmt.Errorf("unused") +} +func (delayedSnapshotStore) PutObject(ctx context.Context, _, _ string, r io.Reader) error { + var header [len(sparseMagic) + 4]byte + if _, err := io.ReadFull(r, header[:]); err != nil { + return err + } + // Keep compression buffers live while the destination applies backpressure. + select { + case <-time.After(20 * time.Millisecond): + case <-ctx.Done(): + return ctx.Err() + } + _, err := io.Copy(io.Discard, r) + return err +} + +// BenchmarkConcurrentSnapshotUploads compares the shared compression budget with +// admitting all 80 uploads at once. It samples peak heap, not cumulative allocation. +// SDK buffering is covered separately by the slow GCS and S3 tests. +func BenchmarkConcurrentSnapshotUploads(b *testing.B) { + const files = 80 + path := filepath.Join(b.TempDir(), "metadata") + if err := os.WriteFile(path, make([]byte, 36), 0o600); err != nil { + b.Fatal(err) + } + logger := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(io.Discard, nil))) + defer slog.SetDefault(logger) + for _, tc := range []struct { + name string + capacity int64 + }{ + {"shared_512MiB", DefaultUploadMemory}, + {"all_80_admitted", files * parallelWorkerMemory}, + } { + b.Run(tc.name, func(b *testing.B) { + u, err := NewUploader(delayedSnapshotStore{}, tc.capacity) + if err != nil { + b.Fatal(err) + } + runtime.GC() + var baseline runtime.MemStats + runtime.ReadMemStats(&baseline) + var peak atomic.Uint64 + peak.Store(baseline.HeapAlloc) + stop, stopped := make(chan struct{}), make(chan struct{}) + go func() { + defer close(stopped) + tick := time.NewTicker(time.Millisecond) + defer tick.Stop() + for { + select { + case <-stop: + return + case <-tick.C: + var stats runtime.MemStats + runtime.ReadMemStats(&stats) + if stats.HeapAlloc > peak.Load() { + peak.Store(stats.HeapAlloc) + } + } + } + }() + b.ResetTimer() + for range b.N { + var wg sync.WaitGroup + for i := range files { + wg.Go(func() { + f, err := os.Open(path) + if err != nil { + b.Error(err) + return + } + defer f.Close() + if err := u.SendFile(b.Context(), fmt.Sprintf("gs://bucket/file%d", i), f); err != nil { + b.Error(err) + } + }) + } + wg.Wait() + } + b.StopTimer() + close(stop) + <-stopped + b.ReportMetric(float64(peak.Load()-baseline.HeapAlloc)/(1<<20), "peak-heap-MiB") + b.ReportMetric(float64(files*b.N)/b.Elapsed().Seconds(), "files/s") + }) + } +} diff --git a/pkg/objectstorage/upload_gcs_test.go b/pkg/objectstorage/upload_gcs_test.go new file mode 100644 index 0000000000..ea865f978b --- /dev/null +++ b/pkg/objectstorage/upload_gcs_test.go @@ -0,0 +1,178 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstorage + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "math/rand" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "testing/synctest" + + "google.golang.org/api/option" +) + +type slowGCSTransport struct { + parts chan string + release <-chan struct{} +} + +func (s *slowGCSTransport) RoundTrip(r *http.Request) (*http.Response, error) { + reply := func(body string) *http.Response { + return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": {"application/json"}}, Body: io.NopCloser(strings.NewReader(body)), Request: r} + } + switch { + case r.URL.Query().Get("uploadType") == "resumable": + var metadata struct { + Name string `json:"name"` + } + if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil { + return nil, err + } + response := reply("") + response.Header.Set("Location", "https://upload.test/session/"+url.PathEscape(metadata.Name)) + return response, nil + case strings.HasPrefix(r.URL.Path, "/session/"), r.URL.Query().Get("uploadType") == "multipart": + name := strings.TrimPrefix(r.URL.Path, "/session/") + if r.URL.Query().Get("uploadType") == "multipart" { + name = r.URL.Query().Get("name") + } + if _, err := io.Copy(io.Discard, r.Body); err != nil { + return nil, err + } + s.parts <- name + select { + case <-s.release: + case <-r.Context().Done(): + return nil, r.Context().Err() + } + return reply(fmt.Sprintf(`{"name":%q,"bucket":"bucket"}`, name)), nil + case strings.HasSuffix(r.URL.Path, "/compose"): + return reply(`{"name":"composed","bucket":"bucket"}`), nil + case r.Method == http.MethodDelete: + return reply(""), nil + default: + return nil, fmt.Errorf("unexpected GCS request: %s %s", r.Method, r.URL) + } +} + +func TestUploaderSlowGCSComposite(t *testing.T) { + // Use real GCS writers and the real composite part pool, replacing only + // HTTP transport so every part can be held without wall-clock sleeps. + synctest.Test(t, func(t *testing.T) { + gate := make(chan struct{}) + transport := &slowGCSTransport{parts: make(chan string, 16), release: gate} + client, err := NewGCSClient(t.Context(), option.WithoutAuthentication(), option.WithHTTPClient(&http.Client{Transport: transport})) + if err != nil { + t.Fatal(err) + } + g := client.(*gcsClient) + defer func() { + g.client.Close() + for _, c := range g.pool { + c.Close() + } + }() + u, err := NewUploader(client, uploadBufferMemory(client)+parallelWorkerMemory) + if err != nil { + t.Fatal(err) + } + send := func(name string) <-chan error { + done := make(chan error, 1) + go func() { + // A raw body just over the lookahead requires three actual + // composite parts, each with the SDK's retry buffer. + body := io.LimitReader(rand.New(rand.NewSource(2553)), uploadCompositeMin+1) + done <- u.Send(context.Background(), "gs://bucket/"+name, body) + }() + return done + } + first := send("first") + synctest.Wait() + if got := len(transport.parts); got != 3 { + t.Errorf("started %d GCS parts, want 3", got) + } + next := send("next") + synctest.Wait() + if got := len(transport.parts); got != 3 { + t.Errorf("next upload started before the first released its buffers: %d parts", got) + } + close(gate) + synctest.Wait() + if err := <-first; err != nil { + t.Error(err) + } + if err := <-next; err != nil { + t.Error(err) + } + if got := len(transport.parts); got != 6 { + t.Errorf("uploaded %d parts, want 6", got) + } + }) +} + +func TestGCSSparseUploadHonorsReservedParts(t *testing.T) { + file, err := os.Create(filepath.Join(t.TempDir(), "memory")) + if err != nil { + t.Fatal(err) + } + defer file.Close() + block := bytes.Repeat([]byte{0x5a}, 1<<20) + for range 3 * sparsePartTarget / len(block) { + if _, err := file.Write(block); err != nil { + t.Fatal(err) + } + } + if err := file.Truncate(2 << 30); err != nil { + t.Fatal(err) + } + if err := file.Sync(); err != nil { + t.Fatal(err) + } + synctest.Test(t, func(t *testing.T) { + gate := make(chan struct{}) + transport := &slowGCSTransport{parts: make(chan string, 8), release: gate} + client, err := NewGCSClient(t.Context(), option.WithoutAuthentication(), option.WithHTTPClient(&http.Client{Transport: transport})) + if err != nil { + t.Fatal(err) + } + g := client.(*gcsClient) + defer func() { + g.client.Close() + for _, c := range g.pool { + c.Close() + } + }() + done := make(chan error, 1) + go func() { _, err := g.PutSparseFile(t.Context(), "bucket", "memory", file, 2); done <- err }() + synctest.Wait() + if got := len(transport.parts); got != 2 { + t.Errorf("started %d sparse parts, want reserved limit 2", got) + } + close(gate) + synctest.Wait() + if err := <-done; err != nil { + t.Fatal(err) + } + }) +} diff --git a/pkg/objectstorage/upload_s3_test.go b/pkg/objectstorage/upload_s3_test.go new file mode 100644 index 0000000000..fefed3d3db --- /dev/null +++ b/pkg/objectstorage/upload_s3_test.go @@ -0,0 +1,151 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstorage + +import ( + "context" + "errors" + "io" + "math/rand" + "os" + "path/filepath" + "sync/atomic" + "testing" + "testing/synctest" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +// The real SDK uploader owns its normal part buffers; only its network calls +// are replaced, so a slow multipart upload exercises the production pipeline. +type slowMultipartAPI struct { + created chan string + parts chan string + release <-chan struct{} + fail string + aborted atomic.Int64 +} + +func (s *slowMultipartAPI) CreateMultipartUpload(_ context.Context, in *s3.CreateMultipartUploadInput, _ ...func(*s3.Options)) (*s3.CreateMultipartUploadOutput, error) { + s.created <- *in.Key + return &s3.CreateMultipartUploadOutput{UploadId: in.Key}, nil +} +func (s *slowMultipartAPI) UploadPart(ctx context.Context, in *s3.UploadPartInput, _ ...func(*s3.Options)) (*s3.UploadPartOutput, error) { + if _, err := io.Copy(io.Discard, in.Body); err != nil { + return nil, err + } + s.parts <- *in.Key + select { + case <-s.release: + case <-ctx.Done(): + return nil, ctx.Err() + } + if *in.Key == s.fail { + return nil, errors.New("part upload failed") + } + return &s3.UploadPartOutput{ETag: aws.String("etag")}, nil +} +func (s *slowMultipartAPI) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput, ...func(*s3.Options)) (*s3.CompleteMultipartUploadOutput, error) { + return &s3.CompleteMultipartUploadOutput{}, nil +} +func (s *slowMultipartAPI) AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput, ...func(*s3.Options)) (*s3.AbortMultipartUploadOutput, error) { + s.aborted.Add(1) + return &s3.AbortMultipartUploadOutput{}, nil +} +func (*slowMultipartAPI) PutObject(context.Context, *s3.PutObjectInput, ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + return nil, errors.New("expected a multipart upload") +} + +func TestUploaderSlowS3Multipart(t *testing.T) { + path := filepath.Join(t.TempDir(), "sparse-memory") + f, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + // Incompressible data forces multiple SDK parts; the remaining 2 GiB is a + // hole, exercising sparse compression without allocating the logical size. + if _, err := io.CopyN(f, rand.New(rand.NewSource(2553)), 40<<20); err != nil { + t.Fatal(err) + } + if err := f.Truncate(2 << 30); err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + for _, failure := range []bool{false, true} { + t.Run(map[bool]string{false: "success", true: "failure"}[failure], func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + gate := make(chan struct{}) + api := &slowMultipartAPI{created: make(chan string, 4), parts: make(chan string, 8), release: gate} + if failure { + api.fail = "first" + } + //nolint:staticcheck // SA1019: use the same SDK uploader as s3Client. + client := &s3Client{uploader: manager.NewUploader(api, func(u *manager.Uploader) { + u.PartSize = uploadPartSize + u.Concurrency = uploadConcurrency + })} + u, err := NewUploader(client, uploadBufferMemory(client)+parallelWorkerMemory) + if err != nil { + t.Fatal(err) + } + send := func(name string) <-chan error { + done := make(chan error, 1) + go func() { + file, err := os.Open(path) + if err == nil { + defer file.Close() + err = u.SendFile(t.Context(), "gs://bucket/"+name, file) + } + done <- err + }() + return done + } + first := send("first") + synctest.Wait() + if got := len(api.parts); got != 2 { + t.Errorf("active SDK parts = %d, want 2", got) + } + if got := <-api.created; got != "first" { + t.Error(got) + } + next := send("next") + synctest.Wait() + select { + case got := <-api.created: + t.Errorf("%s started while first held SDK buffers", got) + default: + } + close(gate) + synctest.Wait() + if err := <-first; (err != nil) != failure { + t.Errorf("first: %v", err) + } + if err := <-next; err != nil { + t.Errorf("next: %v", err) + } + if got := <-api.created; got != "next" { + t.Error(got) + } + if failure && api.aborted.Load() != 1 { + t.Error("failed multipart upload was not aborted") + } + }) + }) + } +} diff --git a/pkg/objectstorage/upload_test.go b/pkg/objectstorage/upload_test.go new file mode 100644 index 0000000000..c0136d14d6 --- /dev/null +++ b/pkg/objectstorage/upload_test.go @@ -0,0 +1,317 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstorage + +import ( + "context" + "errors" + "io" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "testing/synctest" +) + +func TestUploaderBudgetFitsBothPaths(t *testing.T) { + f, err := os.Create(filepath.Join(t.TempDir(), "sparse")) + if err != nil { + t.Fatal(err) + } + defer f.Close() + if err := f.Truncate(8 << 30); err != nil { + t.Fatal(err) + } + for _, client := range []ObjectStorage{&s3Client{}, &gcsClient{}, newMemStore()} { + minimum := uploadBufferMemory(client) + parallelWorkerMemory + if _, err := NewUploader(client, minimum-1); err == nil { + t.Fatalf("%T accepted a budget smaller than one upload", client) + } + for _, capacity := range []int64{minimum, DefaultUploadMemory} { + u, err := NewUploader(client, capacity) + if err != nil { + t.Fatal(err) + } + plan, err := u.plan(f) + if err != nil { + t.Fatal(err) + } + if plan.workers < 1 || plan.workers > runtime.GOMAXPROCS(0) || plan.memory > capacity { + t.Fatalf("%T: invalid plan %+v for capacity %d", client, plan, capacity) + } + if plan.memory < uploadBufferMemory(client)+int64(plan.workers)*parallelWorkerMemory { + t.Fatalf("%T: streaming pipeline under-reserved: %+v", client, plan) + } + if _, ok := client.(sparseFilePutter); ok && (plan.parts < 1 || int64(plan.parts)*sparseUploadMemory > plan.memory) { + t.Fatalf("%T: sparse pipeline under-reserved: %+v", client, plan) + } + } + } +} + +func TestUploaderPlansFromPopulatedBytes(t *testing.T) { + previous := runtime.GOMAXPROCS(8) + defer runtime.GOMAXPROCS(previous) + for _, tc := range []struct { + name string + populatedMiB int + gcs, s3 uploadPlan + }{ + {"all_holes", 0, uploadPlan{1, 1, 384 << 20}, uploadPlan{1, 1, 224 << 20}}, + {"small", 1, uploadPlan{1, 1, 384 << 20}, uploadPlan{1, 1, 224 << 20}}, + {"two_chunks", 9, uploadPlan{2, 1, 448 << 20}, uploadPlan{2, 1, 288 << 20}}, + {"multiple_ranges", 192, uploadPlan{3, 3, 512 << 20}, uploadPlan{5, 3, 480 << 20}}, + } { + t.Run(tc.name, func(t *testing.T) { + offsets := make([]int64, tc.populatedMiB) + for i := range offsets { + offsets[i] = int64(i) << 20 + } + f := sparseFileWith(t, 2<<30, offsets) + for _, backend := range []struct { + name string + client ObjectStorage + want uploadPlan + }{ + {"gcs", &gcsClient{}, tc.gcs}, + {"s3", &s3Client{}, tc.s3}, + } { + t.Run(backend.name, func(t *testing.T) { + u, err := NewUploader(backend.client, DefaultUploadMemory) + if err != nil { + t.Fatal(err) + } + if _, err := f.Seek(7, io.SeekStart); err != nil { + t.Fatal(err) + } + got, err := u.plan(f) + if err != nil { + t.Fatal(err) + } + if got != backend.want { + t.Errorf("plan = %+v, want %+v", got, backend.want) + } + if position, err := f.Seek(0, io.SeekCurrent); err != nil || position != 7 { + t.Errorf("planning changed file position: position=%d, err=%v", position, err) + } + }) + } + }) + } +} + +func TestUploaderSmallSparseFilesShareBudget(t *testing.T) { + previous := runtime.GOMAXPROCS(2) + defer runtime.GOMAXPROCS(previous) + files := []*os.File{ + sparseFileWith(t, 2<<30, []int64{1 << 30}), + sparseFileWith(t, 2<<30, []int64{1 << 30}), + } + synctest.Test(t, func(t *testing.T) { + gate := make(chan struct{}) + store := &gatedUploadStore{started: make(chan string, len(files)), release: gate} + u, err := NewUploader(store, 2*parallelWorkerMemory) + if err != nil { + t.Fatal(err) + } + done := make(chan error, len(files)) + for _, f := range files { + go func() { done <- u.SendFile(t.Context(), "gs://bucket/"+filepath.Base(f.Name()), f) }() + } + synctest.Wait() + if got := len(store.started); got != len(files) { + t.Errorf("started %d uploads, want both small resident sets to fit the budget", got) + } + close(gate) + for range files { + if err := <-done; err != nil { + t.Error(err) + } + } + }) +} + +type gatedUploadStore struct { + started chan string + release <-chan struct{} + err error +} + +func (*gatedUploadStore) supportsStreamingPut() {} +func (*gatedUploadStore) GetObject(context.Context, string, string) (io.ReadCloser, error) { + return nil, errors.New("unused") +} +func (s *gatedUploadStore) PutObject(ctx context.Context, _, object string, r io.Reader) error { + // Drain compression first: the reservation must remain held while the + // destination is slow even after the producer has finished. + if _, err := io.Copy(io.Discard, r); err != nil { + return err + } + s.started <- object + select { + case <-s.release: + return s.err + case <-ctx.Done(): + return ctx.Err() + } +} + +func TestUploaderWaitCancelAndRelease(t *testing.T) { + for _, failure := range []bool{false, true} { + t.Run(map[bool]string{false: "success", true: "failure"}[failure], func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + gate := make(chan struct{}) + store := &gatedUploadStore{started: make(chan string, 3), release: gate} + if failure { + store.err = errors.New("upload failed") + } + u, err := NewUploader(store, parallelWorkerMemory) + if err != nil { + t.Fatal(err) + } + send := func(ctx context.Context, name string) <-chan error { + done := make(chan error, 1) + go func() { done <- u.sendZstd(ctx, "gs://bucket/"+name, strings.NewReader("payload")) }() + return done + } + first := send(t.Context(), "first") + synctest.Wait() + if got := <-store.started; got != "first" { + t.Fatal(got) + } + ctx, cancel := context.WithCancel(t.Context()) + waiting := send(ctx, "waiting") + synctest.Wait() + select { + case got := <-store.started: + t.Fatalf("%s bypassed budget", got) + default: + } + cancel() + synctest.Wait() + if err := <-waiting; !errors.Is(err, context.Canceled) { + t.Fatalf("waiting upload: %v", err) + } + // Cancellation of a waiter must not release the first upload's reservation. + next := send(t.Context(), "next") + synctest.Wait() + select { + case got := <-store.started: + t.Fatalf("%s bypassed held budget", got) + default: + } + close(gate) + synctest.Wait() + if err := <-first; !errors.Is(err, store.err) { + t.Fatalf("first: %v", err) + } + if err := <-next; !errors.Is(err, store.err) { + t.Fatalf("next: %v", err) + } + if got := <-store.started; got != "next" { + t.Fatal(got) + } + }) + }) + } +} + +func TestUploaderCancellationReleasesActiveReservation(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + gate := make(chan struct{}) + store := &gatedUploadStore{started: make(chan string, 2), release: gate} + u, err := NewUploader(store, parallelWorkerMemory) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + go func() { done <- u.sendZstd(ctx, "gs://bucket/canceled", strings.NewReader("payload")) }() + synctest.Wait() + <-store.started + cancel() + synctest.Wait() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + close(gate) + if err := u.sendZstd(t.Context(), "gs://bucket/next", strings.NewReader("payload")); err != nil { + t.Fatal(err) + } + }) +} + +func TestUploaderReadFailureReleasesReservation(t *testing.T) { + u, err := NewUploader(newMemStore(), parallelWorkerMemory) + if err != nil { + t.Fatal(err) + } + want := errors.New("source failed") + if err := u.sendZstd(t.Context(), "gs://bucket/bad", failingUploadReader{want}); !errors.Is(err, want) { + t.Fatalf("got %v, want source error", err) + } + if err := u.sendZstd(t.Context(), "gs://bucket/next", strings.NewReader("payload")); err != nil { + t.Fatal(err) + } +} + +type failingUploadReader struct{ err error } + +func (r failingUploadReader) Read([]byte) (int, error) { return 0, r.err } + +type earlyUploadFailure struct{ err error } + +func (*earlyUploadFailure) supportsStreamingPut() {} +func (*earlyUploadFailure) GetObject(context.Context, string, string) (io.ReadCloser, error) { + return nil, errors.New("unused") +} +func (s *earlyUploadFailure) PutObject(_ context.Context, _, _ string, r io.Reader) error { + header := make([]byte, len(sparseMagic)+4) + if _, err := io.ReadFull(r, header); err != nil { + return err + } + return s.err +} + +func TestUploaderDestinationFailureStopsCompression(t *testing.T) { + path := filepath.Join(t.TempDir(), "source") + writeSparseSource(t, path, 2<<30, []region{{0, 8 << 20, 0x5a}}) + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + want := errors.New("destination failed before payload") + u, err := NewUploader(&earlyUploadFailure{want}, parallelWorkerMemory) + if err != nil { + t.Fatal(err) + } + if err := u.SendFile(t.Context(), "gs://bucket/object", f); !errors.Is(err, want) { + t.Fatalf("got %v, want destination error", err) + } + // Compression must stop before reading the full populated extent. + at, err := f.Seek(0, io.SeekCurrent) + if err != nil { + t.Fatal(err) + } + if at >= 8<<20 { + t.Fatalf("read the entire extent after destination failure: %d", at) + } + if !u.budget.reserved.TryAcquire(u.budget.capacity) { + t.Fatal("failed upload retained its reservation") + } + u.budget.reserved.Release(u.budget.capacity) +} diff --git a/pkg/proto/ateapipb/ateapi.pb.go b/pkg/proto/ateapipb/ateapi.pb.go index ad88e8685f..f2263209df 100644 --- a/pkg/proto/ateapipb/ateapi.pb.go +++ b/pkg/proto/ateapipb/ateapi.pb.go @@ -2178,6 +2178,13 @@ type WorkerAssignment struct { // // +k8s:optional SandboxRuntime *SandboxRuntime `protobuf:"bytes,9,opt,name=sandbox_runtime,json=sandboxRuntime,proto3" json:"sandbox_runtime,omitempty"` + // worker_spiffe_id is the SPIFFE ID worker_pod's certificate carries, copied + // from Worker.worker_spiffe_id. + // + // +k8s:optional + // +k8s:maxLength=2048 + // +k8s:customValidation # a SPIFFE ID + WorkerSpiffeId string `protobuf:"bytes,10,opt,name=worker_spiffe_id,json=workerSpiffeId,proto3" json:"worker_spiffe_id,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2275,6 +2282,13 @@ func (x *WorkerAssignment) GetSandboxRuntime() *SandboxRuntime { return nil } +func (x *WorkerAssignment) GetWorkerSpiffeId() string { + if x != nil { + return x.WorkerSpiffeId + } + return "" +} + // TagStatus is the server-owned state of a Tag. // The tag is fixed to one external snapshot once its creation completes, so // the rest is immutable provenance. @@ -7191,6 +7205,14 @@ type Worker struct { // +k8s:customValidation # until `format=k8s-ip` is supported // +k8s:immutable Ips []string `protobuf:"bytes,7,rep,name=ips,proto3" json:"ips,omitempty"` + // worker_spiffe_id is the SPIFFE ID the worker pod's certificate carries. + // atenet-router requires it on the certificate of the worker's atunnel. + // + // +k8s:optional + // +k8s:maxLength=2048 + // +k8s:customValidation # a SPIFFE ID + // +k8s:immutable + WorkerSpiffeId string `protobuf:"bytes,13,opt,name=worker_spiffe_id,json=workerSpiffeId,proto3" json:"worker_spiffe_id,omitempty"` // sandbox_class mirrors the WorkerPool's default sandbox class // // +k8s:optional @@ -7311,6 +7333,13 @@ func (x *Worker) GetIps() []string { return nil } +func (x *Worker) GetWorkerSpiffeId() string { + if x != nil { + return x.WorkerSpiffeId + } + return "" +} + func (x *Worker) GetSandboxClass() string { if x != nil { return x.SandboxClass @@ -7534,14 +7563,15 @@ type VersionedSandboxCompat struct { // +k8s:maxLength=64 SchemaVersion string `protobuf:"bytes,1,opt,name=schema_version,json=schemaVersion,proto3" json:"schema_version,omitempty"` // Key-value compatibility attributes defined by this schema version, covering - // both runtime assets (e.g. "gvisor_asset_hash") and effective guest hardware - // features after masking (e.g. "architecture", "cpu_features"). + // both runtime assets (e.g. "sandbox_runtime_hash") and effective guest + // hardware features after masking (e.g. "architecture", "cpu_features"). // Matched against a snapshot under the same schema_version: every attribute - // on the snapshot must be present on the worker with an equal value; keys - // only the worker has are ignored, so a worker can gain keys under one - // schema_version without invalidating older snapshots. Removing or - // recomputing a key is a new schema_version, under which older snapshots - // no longer match. + // on the snapshot must be present on the worker with an equal value (except + // "cpu_features", where the snapshot's hexadecimal bitmask must be a bitwise + // subset of the worker's); keys only the worker has are ignored, so a worker + // can gain keys under one schema_version without invalidating older + // snapshots. Removing or recomputing a key is a new schema_version, under + // which older snapshots no longer match. // Order as sent does not matter: ate-api-server sorts attributes by key // before recording them. // @@ -7601,7 +7631,7 @@ func (x *VersionedSandboxCompat) GetAttributes() []*AttributeEntry { // AttributeEntry is a single key-value compatibility attribute. type AttributeEntry struct { state protoimpl.MessageState `protogen:"open.v1"` - // Attribute name (e.g. "architecture", "cpu_features", "gvisor_asset_hash"). + // Attribute name (e.g. "architecture", "cpu_features", "sandbox_runtime_hash"). // // +k8s:required // +k8s:maxLength=128 @@ -7979,8 +8009,16 @@ type MintWorkerActorCertificateRequest struct { // +k8s:required // +k8s:maxBytes=16384 CertificateSigningRequest []byte `protobuf:"bytes,3,opt,name=certificate_signing_request,json=certificateSigningRequest,proto3" json:"certificate_signing_request,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // The Worker requesting the certificate. atespace is always empty; Workers + // are global-scoped. atelet fills this in from its caller's certificate, so + // it names the Worker that certificate proves the request came from. It + // must be the Worker that the actor is assigned to. + // + // +k8s:required + // +k8s:beta(since: "0.0")=+k8s:subfield(atespace)=+k8s:forbidden # TODO: get rid of beta prefix + Worker *ObjectRef `protobuf:"bytes,4,opt,name=worker,proto3" json:"worker,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *MintWorkerActorCertificateRequest) Reset() { @@ -8034,6 +8072,13 @@ func (x *MintWorkerActorCertificateRequest) GetCertificateSigningRequest() []byt return nil } +func (x *MintWorkerActorCertificateRequest) GetWorker() *ObjectRef { + if x != nil { + return x.Worker + } + return nil +} + // Response message for MintWorkerActorCertificate. type MintWorkerActorCertificateResponse struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -8230,7 +8275,7 @@ type RequestActorSuspendRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // The Worker making the request. atespace is always empty; Workers are // global-scoped. atelet fills this in from its caller's certificate, so it - // names the Worker the request provably came from. + // names the Worker that certificate proves the request came from. // // +k8s:required // +k8s:beta(since: "0.0")=+k8s:subfield(atespace)=+k8s:forbidden # TODO: get rid of beta prefix @@ -9447,7 +9492,7 @@ const file_ateapi_proto_rawDesc = "" + "ActorCrash\x12\x18\n" + "\amessage\x18\x01 \x01(\tR\amessage\x129\n" + "\n" + - "crash_time\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\tcrashTime\"\xf5\x02\n" + + "crash_time\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\tcrashTime\"\x9f\x03\n" + "\x10WorkerAssignment\x12)\n" + "\x06worker\x18\x06 \x01(\v2\x11.ateapi.ObjectRefR\x06worker\x12)\n" + "\x10worker_namespace\x18\x01 \x01(\tR\x0fworkerNamespace\x12\x1f\n" + @@ -9459,7 +9504,9 @@ const file_ateapi_proto_rawDesc = "" + "\x0eworker_pod_ips\x18\x05 \x03(\tR\fworkerPodIps\x12\x1b\n" + "\tnode_name\x18\a \x01(\tR\bnodeName\x12!\n" + "\fworker_epoch\x18\b \x01(\x03R\vworkerEpoch\x12?\n" + - "\x0fsandbox_runtime\x18\t \x01(\v2\x16.ateapi.SandboxRuntimeR\x0esandboxRuntime\"g\n" + + "\x0fsandbox_runtime\x18\t \x01(\v2\x16.ateapi.SandboxRuntimeR\x0esandboxRuntime\x12(\n" + + "\x10worker_spiffe_id\x18\n" + + " \x01(\tR\x0eworkerSpiffeId\"g\n" + "\tTagStatus\x12,\n" + "\bsnapshot\x18\x01 \x01(\v2\x10.ateapi.SnapshotR\bsnapshot\x12,\n" + "\x12actor_template_uid\x18\x02 \x01(\tR\x10actorTemplateUid\"\xd2\x01\n" + @@ -9699,7 +9746,7 @@ const file_ateapi_proto_rawDesc = "" + "\aoptions\x18\x02 \x01(\v2\x13.ateapi.ListOptionsR\aoptions\"c\n" + "\x12ListActorsResponse\x12%\n" + "\x06actors\x18\x01 \x03(\v2\r.ateapi.ActorR\x06actors\x12&\n" + - "\x0fnext_page_token\x18\x02 \x01(\tR\rnextPageToken\"\xe7\x03\n" + + "\x0fnext_page_token\x18\x02 \x01(\tR\rnextPageToken\"\x91\x04\n" + "\x06Worker\x124\n" + "\bmetadata\x18\x01 \x01(\v2\x18.ateapi.ResourceMetadataR\bmetadata\x12)\n" + "\x10worker_namespace\x18\x02 \x01(\tR\x0fworkerNamespace\x12\x1f\n" + @@ -9709,7 +9756,8 @@ const file_ateapi_proto_rawDesc = "" + "worker_pod\x18\x04 \x01(\tR\tworkerPod\x12$\n" + "\x0eworker_pod_uid\x18\x05 \x01(\tR\fworkerPodUid\x12\x1b\n" + "\tnode_name\x18\x06 \x01(\tR\bnodeName\x12\x10\n" + - "\x03ips\x18\a \x03(\tR\x03ips\x12#\n" + + "\x03ips\x18\a \x03(\tR\x03ips\x12(\n" + + "\x10worker_spiffe_id\x18\r \x01(\tR\x0eworkerSpiffeId\x12#\n" + "\rsandbox_class\x18\b \x01(\tR\fsandboxClass\x122\n" + "\x06labels\x18\t \x03(\v2\x1a.ateapi.Worker.LabelsEntryR\x06labels\x12\x14\n" + "\x05epoch\x18\f \x01(\x03R\x05epoch\x12,\n" + @@ -9751,11 +9799,12 @@ const file_ateapi_proto_rawDesc = "" + "\x0fdefault_runtime\x18\x03 \x01(\v2\x16.ateapi.SandboxRuntimeR\x0edefaultRuntime\x12G\n" + "\x13restorable_runtimes\x18\x04 \x03(\v2\x16.ateapi.SandboxRuntimeR\x12restorableRuntimes\"@\n" + "\x16RegisterWorkerResponse\x12&\n" + - "\x06worker\x18\x01 \x01(\v2\x0e.ateapi.WorkerR\x06worker\"\xa9\x01\n" + + "\x06worker\x18\x01 \x01(\v2\x0e.ateapi.WorkerR\x06worker\"\xd4\x01\n" + "!MintWorkerActorCertificateRequest\x12'\n" + "\x05actor\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\x12\x1b\n" + "\tactor_uid\x18\x02 \x01(\tR\bactorUid\x12>\n" + - "\x1bcertificate_signing_request\x18\x03 \x01(\fR\x19certificateSigningRequest\"S\n" + + "\x1bcertificate_signing_request\x18\x03 \x01(\fR\x19certificateSigningRequest\x12)\n" + + "\x06worker\x18\x04 \x01(\v2\x11.ateapi.ObjectRefR\x06worker\"S\n" + "\"MintWorkerActorCertificateResponse\x12-\n" + "\x12actor_certificates\x18\x01 \x03(\fR\x11actorCertificates\"\x7f\n" + "\x19MintWorkerActorJwtRequest\x12'\n" + @@ -9972,7 +10021,7 @@ const file_ateapi_proto_rawDesc = "" + "\x13RequestActorSuspend\x12\".ateapi.RequestActorSuspendRequest\x1a#.ateapi.RequestActorSuspendResponse\x12m\n" + "\x18ReportActorSnapshotState\x12'.ateapi.ReportActorSnapshotStateRequest\x1a(.ateapi.ReportActorSnapshotStateResponse:Y\n" + "\bresource\x12\x1f.google.protobuf.MessageOptions\x18І\x03 \x01(\v2\x1a.ateapi.ResourceAnnotationR\bresource:R\n" + - "\x06method\x12\x1e.google.protobuf.MethodOptions\x18І\x03 \x01(\v2\x18.ateapi.MethodAnnotationR\x06methodB9Z7github.com/agent-substrate/substrate/pkg/proto/ateapipbb\x06proto3" + "\x06method\x12\x1e.google.protobuf.MethodOptions\x18І\x03 \x01(\v2\x18.ateapi.MethodAnnotationR\x06methodB)Z'go.ate.dev/substrate/pkg/proto/ateapipbb\x06proto3" var ( file_ateapi_proto_rawDescOnce sync.Once @@ -10291,140 +10340,141 @@ var file_ateapi_proto_depIdxs = []int32{ 113, // 150: ateapi.RegisterWorkerRequest.restorable_runtimes:type_name -> ateapi.SandboxRuntime 111, // 151: ateapi.RegisterWorkerResponse.worker:type_name -> ateapi.Worker 32, // 152: ateapi.MintWorkerActorCertificateRequest.actor:type_name -> ateapi.ObjectRef - 32, // 153: ateapi.MintWorkerActorJwtRequest.actor:type_name -> ateapi.ObjectRef - 144, // 154: ateapi.MintWorkerActorJwtResponse.expire_time:type_name -> google.protobuf.Timestamp - 32, // 155: ateapi.RequestActorSuspendRequest.worker:type_name -> ateapi.ObjectRef - 32, // 156: ateapi.RequestActorSuspendRequest.actor:type_name -> ateapi.ObjectRef - 15, // 157: ateapi.RequestActorSuspendResponse.actor:type_name -> ateapi.Actor - 32, // 158: ateapi.ReportActorSnapshotStateRequest.actor:type_name -> ateapi.ObjectRef - 128, // 159: ateapi.ReportActorSnapshotStateRequest.state:type_name -> ateapi.ActorSnapshotState - 129, // 160: ateapi.ActorSnapshotState.latest_local:type_name -> ateapi.SnapshotReport - 129, // 161: ateapi.ActorSnapshotState.uploading:type_name -> ateapi.SnapshotReport - 129, // 162: ateapi.ActorSnapshotState.latest_durable:type_name -> ateapi.SnapshotReport - 144, // 163: ateapi.ActorSnapshotState.upload_due_time:type_name -> google.protobuf.Timestamp - 2, // 164: ateapi.SnapshotReport.fidelity:type_name -> ateapi.SnapshotFidelity - 9, // 165: ateapi.SnapshotReport.object:type_name -> ateapi.ObjectSnapshot - 144, // 166: ateapi.SnapshotReport.local_commit_time:type_name -> google.protobuf.Timestamp - 144, // 167: ateapi.SnapshotReport.durable_complete_time:type_name -> google.protobuf.Timestamp - 13, // 168: ateapi.AccessPolicy.metadata:type_name -> ateapi.ResourceMetadata - 131, // 169: ateapi.AccessPolicy.bindings:type_name -> ateapi.Binding - 130, // 170: ateapi.CreateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 130, // 171: ateapi.UpdateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 32, // 172: ateapi.GetAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 32, // 173: ateapi.CreateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 130, // 174: ateapi.CreateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 32, // 175: ateapi.UpdateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 130, // 176: ateapi.UpdateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 32, // 177: ateapi.DeleteAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 93, // 178: ateapi.DeleteAtespaceAccessPolicyRequest.options:type_name -> ateapi.DeleteOptions - 8, // 179: ateapi.ResourceAnnotation.scope:type_name -> ateapi.Scope - 145, // 180: ateapi.resource:extendee -> google.protobuf.MessageOptions - 146, // 181: ateapi.method:extendee -> google.protobuf.MethodOptions - 139, // 182: ateapi.resource:type_name -> ateapi.ResourceAnnotation - 140, // 183: ateapi.method:type_name -> ateapi.MethodAnnotation - 67, // 184: ateapi.Control.GetActor:input_type -> ateapi.GetActorRequest - 68, // 185: ateapi.Control.CreateActor:input_type -> ateapi.CreateActorRequest - 69, // 186: ateapi.Control.UpdateActor:input_type -> ateapi.UpdateActorRequest - 70, // 187: ateapi.Control.SuspendActor:input_type -> ateapi.SuspendActorRequest - 72, // 188: ateapi.Control.PauseActor:input_type -> ateapi.PauseActorRequest - 74, // 189: ateapi.Control.ResumeActor:input_type -> ateapi.ResumeActorRequest - 76, // 190: ateapi.Control.RevertActor:input_type -> ateapi.RevertActorRequest - 78, // 191: ateapi.Control.DeleteActor:input_type -> ateapi.DeleteActorRequest - 79, // 192: ateapi.Control.GetActorEgressPolicy:input_type -> ateapi.GetActorEgressPolicyRequest - 80, // 193: ateapi.Control.CreateActorEgressPolicy:input_type -> ateapi.CreateActorEgressPolicyRequest - 81, // 194: ateapi.Control.UpdateActorEgressPolicy:input_type -> ateapi.UpdateActorEgressPolicyRequest - 82, // 195: ateapi.Control.DeleteActorEgressPolicy:input_type -> ateapi.DeleteActorEgressPolicyRequest - 84, // 196: ateapi.Control.MintActorJwt:input_type -> ateapi.MintActorJwtRequest - 86, // 197: ateapi.Control.MintActorCertificate:input_type -> ateapi.MintActorCertificateRequest - 90, // 198: ateapi.Control.CreateTag:input_type -> ateapi.CreateTagRequest - 83, // 199: ateapi.Control.GetTag:input_type -> ateapi.GetTagRequest - 88, // 200: ateapi.Control.ListTags:input_type -> ateapi.ListTagsRequest - 91, // 201: ateapi.Control.UpdateTag:input_type -> ateapi.UpdateTagRequest - 92, // 202: ateapi.Control.DeleteTag:input_type -> ateapi.DeleteTagRequest - 97, // 203: ateapi.Control.ListWorkers:input_type -> ateapi.ListWorkersRequest - 99, // 204: ateapi.Control.GetWorker:input_type -> ateapi.GetWorkerRequest - 100, // 205: ateapi.Control.CreateWorker:input_type -> ateapi.CreateWorkerRequest - 101, // 206: ateapi.Control.UpdateWorker:input_type -> ateapi.UpdateWorkerRequest - 102, // 207: ateapi.Control.DeleteWorker:input_type -> ateapi.DeleteWorkerRequest - 103, // 208: ateapi.Control.DrainWorker:input_type -> ateapi.DrainWorkerRequest - 105, // 209: ateapi.Control.MarkWorkerAvailable:input_type -> ateapi.MarkWorkerAvailableRequest - 107, // 210: ateapi.Control.MarkWorkerUnavailable:input_type -> ateapi.MarkWorkerUnavailableRequest - 95, // 211: ateapi.Control.ListWorkerActorAssignments:input_type -> ateapi.ListWorkerActorAssignmentsRequest - 109, // 212: ateapi.Control.ListActors:input_type -> ateapi.ListActorsRequest - 57, // 213: ateapi.Control.CreateAtespace:input_type -> ateapi.CreateAtespaceRequest - 58, // 214: ateapi.Control.GetAtespace:input_type -> ateapi.GetAtespaceRequest - 59, // 215: ateapi.Control.ListAtespaces:input_type -> ateapi.ListAtespacesRequest - 61, // 216: ateapi.Control.DeleteAtespace:input_type -> ateapi.DeleteAtespaceRequest - 62, // 217: ateapi.Control.CreateActorTemplate:input_type -> ateapi.CreateActorTemplateRequest - 63, // 218: ateapi.Control.GetActorTemplate:input_type -> ateapi.GetActorTemplateRequest - 64, // 219: ateapi.Control.ListActorTemplates:input_type -> ateapi.ListActorTemplatesRequest - 66, // 220: ateapi.Control.DeleteActorTemplate:input_type -> ateapi.DeleteActorTemplateRequest - 132, // 221: ateapi.Control.GetGlobalAccessPolicy:input_type -> ateapi.GetGlobalAccessPolicyRequest - 133, // 222: ateapi.Control.CreateGlobalAccessPolicy:input_type -> ateapi.CreateGlobalAccessPolicyRequest - 134, // 223: ateapi.Control.UpdateGlobalAccessPolicy:input_type -> ateapi.UpdateGlobalAccessPolicyRequest - 135, // 224: ateapi.Control.GetAtespaceAccessPolicy:input_type -> ateapi.GetAtespaceAccessPolicyRequest - 136, // 225: ateapi.Control.CreateAtespaceAccessPolicy:input_type -> ateapi.CreateAtespaceAccessPolicyRequest - 137, // 226: ateapi.Control.UpdateAtespaceAccessPolicy:input_type -> ateapi.UpdateAtespaceAccessPolicyRequest - 138, // 227: ateapi.Control.DeleteAtespaceAccessPolicy:input_type -> ateapi.DeleteAtespaceAccessPolicyRequest - 118, // 228: ateapi.WorkerService.RegisterWorker:input_type -> ateapi.RegisterWorkerRequest - 120, // 229: ateapi.WorkerService.MintWorkerActorCertificate:input_type -> ateapi.MintWorkerActorCertificateRequest - 122, // 230: ateapi.WorkerService.MintWorkerActorJwt:input_type -> ateapi.MintWorkerActorJwtRequest - 124, // 231: ateapi.WorkerService.RequestActorSuspend:input_type -> ateapi.RequestActorSuspendRequest - 126, // 232: ateapi.WorkerService.ReportActorSnapshotState:input_type -> ateapi.ReportActorSnapshotStateRequest - 15, // 233: ateapi.Control.GetActor:output_type -> ateapi.Actor - 15, // 234: ateapi.Control.CreateActor:output_type -> ateapi.Actor - 15, // 235: ateapi.Control.UpdateActor:output_type -> ateapi.Actor - 71, // 236: ateapi.Control.SuspendActor:output_type -> ateapi.SuspendActorResponse - 73, // 237: ateapi.Control.PauseActor:output_type -> ateapi.PauseActorResponse - 75, // 238: ateapi.Control.ResumeActor:output_type -> ateapi.ResumeActorResponse - 77, // 239: ateapi.Control.RevertActor:output_type -> ateapi.RevertActorResponse - 15, // 240: ateapi.Control.DeleteActor:output_type -> ateapi.Actor - 16, // 241: ateapi.Control.GetActorEgressPolicy:output_type -> ateapi.EgressPolicy - 16, // 242: ateapi.Control.CreateActorEgressPolicy:output_type -> ateapi.EgressPolicy - 16, // 243: ateapi.Control.UpdateActorEgressPolicy:output_type -> ateapi.EgressPolicy - 16, // 244: ateapi.Control.DeleteActorEgressPolicy:output_type -> ateapi.EgressPolicy - 85, // 245: ateapi.Control.MintActorJwt:output_type -> ateapi.MintActorJwtResponse - 87, // 246: ateapi.Control.MintActorCertificate:output_type -> ateapi.MintActorCertificateResponse - 30, // 247: ateapi.Control.CreateTag:output_type -> ateapi.Tag - 30, // 248: ateapi.Control.GetTag:output_type -> ateapi.Tag - 89, // 249: ateapi.Control.ListTags:output_type -> ateapi.ListTagsResponse - 30, // 250: ateapi.Control.UpdateTag:output_type -> ateapi.Tag - 30, // 251: ateapi.Control.DeleteTag:output_type -> ateapi.Tag - 98, // 252: ateapi.Control.ListWorkers:output_type -> ateapi.ListWorkersResponse - 111, // 253: ateapi.Control.GetWorker:output_type -> ateapi.Worker - 111, // 254: ateapi.Control.CreateWorker:output_type -> ateapi.Worker - 111, // 255: ateapi.Control.UpdateWorker:output_type -> ateapi.Worker - 111, // 256: ateapi.Control.DeleteWorker:output_type -> ateapi.Worker - 104, // 257: ateapi.Control.DrainWorker:output_type -> ateapi.DrainWorkerResponse - 106, // 258: ateapi.Control.MarkWorkerAvailable:output_type -> ateapi.MarkWorkerAvailableResponse - 108, // 259: ateapi.Control.MarkWorkerUnavailable:output_type -> ateapi.MarkWorkerUnavailableResponse - 96, // 260: ateapi.Control.ListWorkerActorAssignments:output_type -> ateapi.ListWorkerActorAssignmentsResponse - 110, // 261: ateapi.Control.ListActors:output_type -> ateapi.ListActorsResponse - 31, // 262: ateapi.Control.CreateAtespace:output_type -> ateapi.Atespace - 31, // 263: ateapi.Control.GetAtespace:output_type -> ateapi.Atespace - 60, // 264: ateapi.Control.ListAtespaces:output_type -> ateapi.ListAtespacesResponse - 31, // 265: ateapi.Control.DeleteAtespace:output_type -> ateapi.Atespace - 33, // 266: ateapi.Control.CreateActorTemplate:output_type -> ateapi.ActorTemplate - 33, // 267: ateapi.Control.GetActorTemplate:output_type -> ateapi.ActorTemplate - 65, // 268: ateapi.Control.ListActorTemplates:output_type -> ateapi.ListActorTemplatesResponse - 33, // 269: ateapi.Control.DeleteActorTemplate:output_type -> ateapi.ActorTemplate - 130, // 270: ateapi.Control.GetGlobalAccessPolicy:output_type -> ateapi.AccessPolicy - 130, // 271: ateapi.Control.CreateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy - 130, // 272: ateapi.Control.UpdateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy - 130, // 273: ateapi.Control.GetAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 130, // 274: ateapi.Control.CreateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 130, // 275: ateapi.Control.UpdateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 130, // 276: ateapi.Control.DeleteAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 119, // 277: ateapi.WorkerService.RegisterWorker:output_type -> ateapi.RegisterWorkerResponse - 121, // 278: ateapi.WorkerService.MintWorkerActorCertificate:output_type -> ateapi.MintWorkerActorCertificateResponse - 123, // 279: ateapi.WorkerService.MintWorkerActorJwt:output_type -> ateapi.MintWorkerActorJwtResponse - 125, // 280: ateapi.WorkerService.RequestActorSuspend:output_type -> ateapi.RequestActorSuspendResponse - 127, // 281: ateapi.WorkerService.ReportActorSnapshotState:output_type -> ateapi.ReportActorSnapshotStateResponse - 233, // [233:282] is the sub-list for method output_type - 184, // [184:233] is the sub-list for method input_type - 182, // [182:184] is the sub-list for extension type_name - 180, // [180:182] is the sub-list for extension extendee - 0, // [0:180] is the sub-list for field type_name + 32, // 153: ateapi.MintWorkerActorCertificateRequest.worker:type_name -> ateapi.ObjectRef + 32, // 154: ateapi.MintWorkerActorJwtRequest.actor:type_name -> ateapi.ObjectRef + 144, // 155: ateapi.MintWorkerActorJwtResponse.expire_time:type_name -> google.protobuf.Timestamp + 32, // 156: ateapi.RequestActorSuspendRequest.worker:type_name -> ateapi.ObjectRef + 32, // 157: ateapi.RequestActorSuspendRequest.actor:type_name -> ateapi.ObjectRef + 15, // 158: ateapi.RequestActorSuspendResponse.actor:type_name -> ateapi.Actor + 32, // 159: ateapi.ReportActorSnapshotStateRequest.actor:type_name -> ateapi.ObjectRef + 128, // 160: ateapi.ReportActorSnapshotStateRequest.state:type_name -> ateapi.ActorSnapshotState + 129, // 161: ateapi.ActorSnapshotState.latest_local:type_name -> ateapi.SnapshotReport + 129, // 162: ateapi.ActorSnapshotState.uploading:type_name -> ateapi.SnapshotReport + 129, // 163: ateapi.ActorSnapshotState.latest_durable:type_name -> ateapi.SnapshotReport + 144, // 164: ateapi.ActorSnapshotState.upload_due_time:type_name -> google.protobuf.Timestamp + 2, // 165: ateapi.SnapshotReport.fidelity:type_name -> ateapi.SnapshotFidelity + 9, // 166: ateapi.SnapshotReport.object:type_name -> ateapi.ObjectSnapshot + 144, // 167: ateapi.SnapshotReport.local_commit_time:type_name -> google.protobuf.Timestamp + 144, // 168: ateapi.SnapshotReport.durable_complete_time:type_name -> google.protobuf.Timestamp + 13, // 169: ateapi.AccessPolicy.metadata:type_name -> ateapi.ResourceMetadata + 131, // 170: ateapi.AccessPolicy.bindings:type_name -> ateapi.Binding + 130, // 171: ateapi.CreateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 130, // 172: ateapi.UpdateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 32, // 173: ateapi.GetAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 32, // 174: ateapi.CreateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 130, // 175: ateapi.CreateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 32, // 176: ateapi.UpdateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 130, // 177: ateapi.UpdateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 32, // 178: ateapi.DeleteAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 93, // 179: ateapi.DeleteAtespaceAccessPolicyRequest.options:type_name -> ateapi.DeleteOptions + 8, // 180: ateapi.ResourceAnnotation.scope:type_name -> ateapi.Scope + 145, // 181: ateapi.resource:extendee -> google.protobuf.MessageOptions + 146, // 182: ateapi.method:extendee -> google.protobuf.MethodOptions + 139, // 183: ateapi.resource:type_name -> ateapi.ResourceAnnotation + 140, // 184: ateapi.method:type_name -> ateapi.MethodAnnotation + 67, // 185: ateapi.Control.GetActor:input_type -> ateapi.GetActorRequest + 68, // 186: ateapi.Control.CreateActor:input_type -> ateapi.CreateActorRequest + 69, // 187: ateapi.Control.UpdateActor:input_type -> ateapi.UpdateActorRequest + 70, // 188: ateapi.Control.SuspendActor:input_type -> ateapi.SuspendActorRequest + 72, // 189: ateapi.Control.PauseActor:input_type -> ateapi.PauseActorRequest + 74, // 190: ateapi.Control.ResumeActor:input_type -> ateapi.ResumeActorRequest + 76, // 191: ateapi.Control.RevertActor:input_type -> ateapi.RevertActorRequest + 78, // 192: ateapi.Control.DeleteActor:input_type -> ateapi.DeleteActorRequest + 79, // 193: ateapi.Control.GetActorEgressPolicy:input_type -> ateapi.GetActorEgressPolicyRequest + 80, // 194: ateapi.Control.CreateActorEgressPolicy:input_type -> ateapi.CreateActorEgressPolicyRequest + 81, // 195: ateapi.Control.UpdateActorEgressPolicy:input_type -> ateapi.UpdateActorEgressPolicyRequest + 82, // 196: ateapi.Control.DeleteActorEgressPolicy:input_type -> ateapi.DeleteActorEgressPolicyRequest + 84, // 197: ateapi.Control.MintActorJwt:input_type -> ateapi.MintActorJwtRequest + 86, // 198: ateapi.Control.MintActorCertificate:input_type -> ateapi.MintActorCertificateRequest + 90, // 199: ateapi.Control.CreateTag:input_type -> ateapi.CreateTagRequest + 83, // 200: ateapi.Control.GetTag:input_type -> ateapi.GetTagRequest + 88, // 201: ateapi.Control.ListTags:input_type -> ateapi.ListTagsRequest + 91, // 202: ateapi.Control.UpdateTag:input_type -> ateapi.UpdateTagRequest + 92, // 203: ateapi.Control.DeleteTag:input_type -> ateapi.DeleteTagRequest + 97, // 204: ateapi.Control.ListWorkers:input_type -> ateapi.ListWorkersRequest + 99, // 205: ateapi.Control.GetWorker:input_type -> ateapi.GetWorkerRequest + 100, // 206: ateapi.Control.CreateWorker:input_type -> ateapi.CreateWorkerRequest + 101, // 207: ateapi.Control.UpdateWorker:input_type -> ateapi.UpdateWorkerRequest + 102, // 208: ateapi.Control.DeleteWorker:input_type -> ateapi.DeleteWorkerRequest + 103, // 209: ateapi.Control.DrainWorker:input_type -> ateapi.DrainWorkerRequest + 105, // 210: ateapi.Control.MarkWorkerAvailable:input_type -> ateapi.MarkWorkerAvailableRequest + 107, // 211: ateapi.Control.MarkWorkerUnavailable:input_type -> ateapi.MarkWorkerUnavailableRequest + 95, // 212: ateapi.Control.ListWorkerActorAssignments:input_type -> ateapi.ListWorkerActorAssignmentsRequest + 109, // 213: ateapi.Control.ListActors:input_type -> ateapi.ListActorsRequest + 57, // 214: ateapi.Control.CreateAtespace:input_type -> ateapi.CreateAtespaceRequest + 58, // 215: ateapi.Control.GetAtespace:input_type -> ateapi.GetAtespaceRequest + 59, // 216: ateapi.Control.ListAtespaces:input_type -> ateapi.ListAtespacesRequest + 61, // 217: ateapi.Control.DeleteAtespace:input_type -> ateapi.DeleteAtespaceRequest + 62, // 218: ateapi.Control.CreateActorTemplate:input_type -> ateapi.CreateActorTemplateRequest + 63, // 219: ateapi.Control.GetActorTemplate:input_type -> ateapi.GetActorTemplateRequest + 64, // 220: ateapi.Control.ListActorTemplates:input_type -> ateapi.ListActorTemplatesRequest + 66, // 221: ateapi.Control.DeleteActorTemplate:input_type -> ateapi.DeleteActorTemplateRequest + 132, // 222: ateapi.Control.GetGlobalAccessPolicy:input_type -> ateapi.GetGlobalAccessPolicyRequest + 133, // 223: ateapi.Control.CreateGlobalAccessPolicy:input_type -> ateapi.CreateGlobalAccessPolicyRequest + 134, // 224: ateapi.Control.UpdateGlobalAccessPolicy:input_type -> ateapi.UpdateGlobalAccessPolicyRequest + 135, // 225: ateapi.Control.GetAtespaceAccessPolicy:input_type -> ateapi.GetAtespaceAccessPolicyRequest + 136, // 226: ateapi.Control.CreateAtespaceAccessPolicy:input_type -> ateapi.CreateAtespaceAccessPolicyRequest + 137, // 227: ateapi.Control.UpdateAtespaceAccessPolicy:input_type -> ateapi.UpdateAtespaceAccessPolicyRequest + 138, // 228: ateapi.Control.DeleteAtespaceAccessPolicy:input_type -> ateapi.DeleteAtespaceAccessPolicyRequest + 118, // 229: ateapi.WorkerService.RegisterWorker:input_type -> ateapi.RegisterWorkerRequest + 120, // 230: ateapi.WorkerService.MintWorkerActorCertificate:input_type -> ateapi.MintWorkerActorCertificateRequest + 122, // 231: ateapi.WorkerService.MintWorkerActorJwt:input_type -> ateapi.MintWorkerActorJwtRequest + 124, // 232: ateapi.WorkerService.RequestActorSuspend:input_type -> ateapi.RequestActorSuspendRequest + 126, // 233: ateapi.WorkerService.ReportActorSnapshotState:input_type -> ateapi.ReportActorSnapshotStateRequest + 15, // 234: ateapi.Control.GetActor:output_type -> ateapi.Actor + 15, // 235: ateapi.Control.CreateActor:output_type -> ateapi.Actor + 15, // 236: ateapi.Control.UpdateActor:output_type -> ateapi.Actor + 71, // 237: ateapi.Control.SuspendActor:output_type -> ateapi.SuspendActorResponse + 73, // 238: ateapi.Control.PauseActor:output_type -> ateapi.PauseActorResponse + 75, // 239: ateapi.Control.ResumeActor:output_type -> ateapi.ResumeActorResponse + 77, // 240: ateapi.Control.RevertActor:output_type -> ateapi.RevertActorResponse + 15, // 241: ateapi.Control.DeleteActor:output_type -> ateapi.Actor + 16, // 242: ateapi.Control.GetActorEgressPolicy:output_type -> ateapi.EgressPolicy + 16, // 243: ateapi.Control.CreateActorEgressPolicy:output_type -> ateapi.EgressPolicy + 16, // 244: ateapi.Control.UpdateActorEgressPolicy:output_type -> ateapi.EgressPolicy + 16, // 245: ateapi.Control.DeleteActorEgressPolicy:output_type -> ateapi.EgressPolicy + 85, // 246: ateapi.Control.MintActorJwt:output_type -> ateapi.MintActorJwtResponse + 87, // 247: ateapi.Control.MintActorCertificate:output_type -> ateapi.MintActorCertificateResponse + 30, // 248: ateapi.Control.CreateTag:output_type -> ateapi.Tag + 30, // 249: ateapi.Control.GetTag:output_type -> ateapi.Tag + 89, // 250: ateapi.Control.ListTags:output_type -> ateapi.ListTagsResponse + 30, // 251: ateapi.Control.UpdateTag:output_type -> ateapi.Tag + 30, // 252: ateapi.Control.DeleteTag:output_type -> ateapi.Tag + 98, // 253: ateapi.Control.ListWorkers:output_type -> ateapi.ListWorkersResponse + 111, // 254: ateapi.Control.GetWorker:output_type -> ateapi.Worker + 111, // 255: ateapi.Control.CreateWorker:output_type -> ateapi.Worker + 111, // 256: ateapi.Control.UpdateWorker:output_type -> ateapi.Worker + 111, // 257: ateapi.Control.DeleteWorker:output_type -> ateapi.Worker + 104, // 258: ateapi.Control.DrainWorker:output_type -> ateapi.DrainWorkerResponse + 106, // 259: ateapi.Control.MarkWorkerAvailable:output_type -> ateapi.MarkWorkerAvailableResponse + 108, // 260: ateapi.Control.MarkWorkerUnavailable:output_type -> ateapi.MarkWorkerUnavailableResponse + 96, // 261: ateapi.Control.ListWorkerActorAssignments:output_type -> ateapi.ListWorkerActorAssignmentsResponse + 110, // 262: ateapi.Control.ListActors:output_type -> ateapi.ListActorsResponse + 31, // 263: ateapi.Control.CreateAtespace:output_type -> ateapi.Atespace + 31, // 264: ateapi.Control.GetAtespace:output_type -> ateapi.Atespace + 60, // 265: ateapi.Control.ListAtespaces:output_type -> ateapi.ListAtespacesResponse + 31, // 266: ateapi.Control.DeleteAtespace:output_type -> ateapi.Atespace + 33, // 267: ateapi.Control.CreateActorTemplate:output_type -> ateapi.ActorTemplate + 33, // 268: ateapi.Control.GetActorTemplate:output_type -> ateapi.ActorTemplate + 65, // 269: ateapi.Control.ListActorTemplates:output_type -> ateapi.ListActorTemplatesResponse + 33, // 270: ateapi.Control.DeleteActorTemplate:output_type -> ateapi.ActorTemplate + 130, // 271: ateapi.Control.GetGlobalAccessPolicy:output_type -> ateapi.AccessPolicy + 130, // 272: ateapi.Control.CreateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy + 130, // 273: ateapi.Control.UpdateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy + 130, // 274: ateapi.Control.GetAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 130, // 275: ateapi.Control.CreateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 130, // 276: ateapi.Control.UpdateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 130, // 277: ateapi.Control.DeleteAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 119, // 278: ateapi.WorkerService.RegisterWorker:output_type -> ateapi.RegisterWorkerResponse + 121, // 279: ateapi.WorkerService.MintWorkerActorCertificate:output_type -> ateapi.MintWorkerActorCertificateResponse + 123, // 280: ateapi.WorkerService.MintWorkerActorJwt:output_type -> ateapi.MintWorkerActorJwtResponse + 125, // 281: ateapi.WorkerService.RequestActorSuspend:output_type -> ateapi.RequestActorSuspendResponse + 127, // 282: ateapi.WorkerService.ReportActorSnapshotState:output_type -> ateapi.ReportActorSnapshotStateResponse + 234, // [234:283] is the sub-list for method output_type + 185, // [185:234] is the sub-list for method input_type + 183, // [183:185] is the sub-list for extension type_name + 181, // [181:183] is the sub-list for extension extendee + 0, // [0:181] is the sub-list for field type_name } func init() { file_ateapi_proto_init() } diff --git a/pkg/proto/ateapipb/ateapi.proto b/pkg/proto/ateapipb/ateapi.proto index 233b3fb63e..0ba0d4a3be 100644 --- a/pkg/proto/ateapipb/ateapi.proto +++ b/pkg/proto/ateapipb/ateapi.proto @@ -19,7 +19,7 @@ package ateapi; import "google/protobuf/descriptor.proto"; import "google/protobuf/timestamp.proto"; -option go_package = "github.com/agent-substrate/substrate/pkg/proto/ateapipb"; +option go_package = "go.ate.dev/substrate/pkg/proto/ateapipb"; // Control is the primary RPC interface for Agentic Substrate. service Control { @@ -1007,6 +1007,14 @@ message WorkerAssignment { // // +k8s:optional SandboxRuntime sandbox_runtime = 9; + + // worker_spiffe_id is the SPIFFE ID worker_pod's certificate carries, copied + // from Worker.worker_spiffe_id. + // + // +k8s:optional + // +k8s:maxLength=2048 + // +k8s:customValidation # a SPIFFE ID + string worker_spiffe_id = 10; } // TagStatus is the server-owned state of a Tag. @@ -2281,6 +2289,14 @@ message Worker { // +k8s:customValidation # until `format=k8s-ip` is supported // +k8s:immutable repeated string ips = 7; + // worker_spiffe_id is the SPIFFE ID the worker pod's certificate carries. + // atenet-router requires it on the certificate of the worker's atunnel. + // + // +k8s:optional + // +k8s:maxLength=2048 + // +k8s:customValidation # a SPIFFE ID + // +k8s:immutable + string worker_spiffe_id = 13; // sandbox_class mirrors the WorkerPool's default sandbox class // @@ -2419,14 +2435,15 @@ message VersionedSandboxCompat { string schema_version = 1; // Key-value compatibility attributes defined by this schema version, covering - // both runtime assets (e.g. "gvisor_asset_hash") and effective guest hardware - // features after masking (e.g. "architecture", "cpu_features"). + // both runtime assets (e.g. "sandbox_runtime_hash") and effective guest + // hardware features after masking (e.g. "architecture", "cpu_features"). // Matched against a snapshot under the same schema_version: every attribute - // on the snapshot must be present on the worker with an equal value; keys - // only the worker has are ignored, so a worker can gain keys under one - // schema_version without invalidating older snapshots. Removing or - // recomputing a key is a new schema_version, under which older snapshots - // no longer match. + // on the snapshot must be present on the worker with an equal value (except + // "cpu_features", where the snapshot's hexadecimal bitmask must be a bitwise + // subset of the worker's); keys only the worker has are ignored, so a worker + // can gain keys under one schema_version without invalidating older + // snapshots. Removing or recomputing a key is a new schema_version, under + // which older snapshots no longer match. // Order as sent does not matter: ate-api-server sorts attributes by key // before recording them. // @@ -2439,7 +2456,7 @@ message VersionedSandboxCompat { // AttributeEntry is a single key-value compatibility attribute. message AttributeEntry { - // Attribute name (e.g. "architecture", "cpu_features", "gvisor_asset_hash"). + // Attribute name (e.g. "architecture", "cpu_features", "sandbox_runtime_hash"). // // +k8s:required // +k8s:maxLength=128 @@ -2531,6 +2548,14 @@ service WorkerService { // on behalf of a particular actor. // // SPIFFE URI: spiffe://${trustdomain}/worker-for-actor/${atespace}/${actor} + // + // atelet calls this with its own client certificate, as it does for + // RequestActorSuspend, naming the Worker its caller proved itself to be. + // The control plane mints only for the Worker that the Actor is assigned + // to, and only to an atelet on that Worker's node. A request from any + // other node or Worker gets NotFound; one from the assigned Worker with a + // stale actor_uid gets Aborted. Unlike MintWorkerActorJwt, a paused Actor + // gets NotFound, since it has no Worker to bind the certificate to. rpc MintWorkerActorCertificate(MintWorkerActorCertificateRequest) returns (MintWorkerActorCertificateResponse); // MintWorkerActorJwt creates a Substrate-issued JWT that asserts a worker @@ -2566,18 +2591,16 @@ service WorkerService { // rather than a repeat of its success. rpc RequestActorSuspend(RequestActorSuspendRequest) returns (RequestActorSuspendResponse); - // ReportActorSnapshotState records which snapshots of an Actor the calling - // atelet holds on its node, and which of them are durable. Only atelet can - // observe when an upload finishes or a local copy is deleted, and the - // control plane needs both to decide where the Actor can resume. - // - // Only atelet should call this, with its own client certificate. The - // control plane serves it only for an Actor on the caller's node. - // Idempotent: each report carries the whole state and replaces the previous - // one. Returns ABORTED while another operation holds the Actor; retry. + // ReportActorSnapshotState reports an Actor's snapshot state on the caller's + // node. The control plane records two facts from it: an upload finished, and + // atelet detached the Actor from the node. Only atelet observes either. // - // TODO(#2570): not served yet. Every call returns UNIMPLEMENTED until the - // handler lands. + // atelet calls this with its own client certificate, for an Actor on its + // node. Each report carries the whole state, so repeating one is safe. + // Returns ABORTED while another operation is changing the Actor; retry. + // Returns NOT_FOUND once the Actor is gone or no longer on the caller's + // node. Returns FAILED_PRECONDITION or INVALID_ARGUMENT when the report + // conflicts with the record; don't retry. rpc ReportActorSnapshotState(ReportActorSnapshotStateRequest) returns (ReportActorSnapshotStateResponse); } @@ -2641,6 +2664,15 @@ message MintWorkerActorCertificateRequest { // +k8s:required // +k8s:maxBytes=16384 bytes certificate_signing_request = 3; + + // The Worker requesting the certificate. atespace is always empty; Workers + // are global-scoped. atelet fills this in from its caller's certificate, so + // it names the Worker that certificate proves the request came from. It + // must be the Worker that the actor is assigned to. + // + // +k8s:required + // +k8s:beta(since: "0.0")=+k8s:subfield(atespace)=+k8s:forbidden # TODO: get rid of beta prefix + ObjectRef worker = 4; } // Response message for MintWorkerActorCertificate. @@ -2700,7 +2732,7 @@ message MintWorkerActorJwtResponse { message RequestActorSuspendRequest { // The Worker making the request. atespace is always empty; Workers are // global-scoped. atelet fills this in from its caller's certificate, so it - // names the Worker the request provably came from. + // names the Worker that certificate proves the request came from. // // +k8s:required // +k8s:beta(since: "0.0")=+k8s:subfield(atespace)=+k8s:forbidden # TODO: get rid of beta prefix diff --git a/pkg/proto/ateapipb/ateapi_grpc.pb.go b/pkg/proto/ateapipb/ateapi_grpc.pb.go index 4043f05a5a..ea0042a580 100644 --- a/pkg/proto/ateapipb/ateapi_grpc.pb.go +++ b/pkg/proto/ateapipb/ateapi_grpc.pb.go @@ -1958,6 +1958,14 @@ type WorkerServiceClient interface { // on behalf of a particular actor. // // SPIFFE URI: spiffe://${trustdomain}/worker-for-actor/${atespace}/${actor} + // + // atelet calls this with its own client certificate, as it does for + // RequestActorSuspend, naming the Worker its caller proved itself to be. + // The control plane mints only for the Worker that the Actor is assigned + // to, and only to an atelet on that Worker's node. A request from any + // other node or Worker gets NotFound; one from the assigned Worker with a + // stale actor_uid gets Aborted. Unlike MintWorkerActorJwt, a paused Actor + // gets NotFound, since it has no Worker to bind the certificate to. MintWorkerActorCertificate(ctx context.Context, in *MintWorkerActorCertificateRequest, opts ...grpc.CallOption) (*MintWorkerActorCertificateResponse, error) // MintWorkerActorJwt creates a Substrate-issued JWT that asserts a worker // acting on behalf of a particular actor: the JWT counterpart of @@ -1990,18 +1998,16 @@ type WorkerServiceClient interface { // retries a suspend it had already been granted therefore sees a failure // rather than a repeat of its success. RequestActorSuspend(ctx context.Context, in *RequestActorSuspendRequest, opts ...grpc.CallOption) (*RequestActorSuspendResponse, error) - // ReportActorSnapshotState records which snapshots of an Actor the calling - // atelet holds on its node, and which of them are durable. Only atelet can - // observe when an upload finishes or a local copy is deleted, and the - // control plane needs both to decide where the Actor can resume. - // - // Only atelet should call this, with its own client certificate. The - // control plane serves it only for an Actor on the caller's node. - // Idempotent: each report carries the whole state and replaces the previous - // one. Returns ABORTED while another operation holds the Actor; retry. + // ReportActorSnapshotState reports an Actor's snapshot state on the caller's + // node. The control plane records two facts from it: an upload finished, and + // atelet detached the Actor from the node. Only atelet observes either. // - // TODO(#2570): not served yet. Every call returns UNIMPLEMENTED until the - // handler lands. + // atelet calls this with its own client certificate, for an Actor on its + // node. Each report carries the whole state, so repeating one is safe. + // Returns ABORTED while another operation is changing the Actor; retry. + // Returns NOT_FOUND once the Actor is gone or no longer on the caller's + // node. Returns FAILED_PRECONDITION or INVALID_ARGUMENT when the report + // conflicts with the record; don't retry. ReportActorSnapshotState(ctx context.Context, in *ReportActorSnapshotStateRequest, opts ...grpc.CallOption) (*ReportActorSnapshotStateResponse, error) } @@ -2087,6 +2093,14 @@ type WorkerServiceServer interface { // on behalf of a particular actor. // // SPIFFE URI: spiffe://${trustdomain}/worker-for-actor/${atespace}/${actor} + // + // atelet calls this with its own client certificate, as it does for + // RequestActorSuspend, naming the Worker its caller proved itself to be. + // The control plane mints only for the Worker that the Actor is assigned + // to, and only to an atelet on that Worker's node. A request from any + // other node or Worker gets NotFound; one from the assigned Worker with a + // stale actor_uid gets Aborted. Unlike MintWorkerActorJwt, a paused Actor + // gets NotFound, since it has no Worker to bind the certificate to. MintWorkerActorCertificate(context.Context, *MintWorkerActorCertificateRequest) (*MintWorkerActorCertificateResponse, error) // MintWorkerActorJwt creates a Substrate-issued JWT that asserts a worker // acting on behalf of a particular actor: the JWT counterpart of @@ -2119,18 +2133,16 @@ type WorkerServiceServer interface { // retries a suspend it had already been granted therefore sees a failure // rather than a repeat of its success. RequestActorSuspend(context.Context, *RequestActorSuspendRequest) (*RequestActorSuspendResponse, error) - // ReportActorSnapshotState records which snapshots of an Actor the calling - // atelet holds on its node, and which of them are durable. Only atelet can - // observe when an upload finishes or a local copy is deleted, and the - // control plane needs both to decide where the Actor can resume. - // - // Only atelet should call this, with its own client certificate. The - // control plane serves it only for an Actor on the caller's node. - // Idempotent: each report carries the whole state and replaces the previous - // one. Returns ABORTED while another operation holds the Actor; retry. + // ReportActorSnapshotState reports an Actor's snapshot state on the caller's + // node. The control plane records two facts from it: an upload finished, and + // atelet detached the Actor from the node. Only atelet observes either. // - // TODO(#2570): not served yet. Every call returns UNIMPLEMENTED until the - // handler lands. + // atelet calls this with its own client certificate, for an Actor on its + // node. Each report carries the whole state, so repeating one is safe. + // Returns ABORTED while another operation is changing the Actor; retry. + // Returns NOT_FOUND once the Actor is gone or no longer on the caller's + // node. Returns FAILED_PRECONDITION or INVALID_ARGUMENT when the report + // conflicts with the record; don't retry. ReportActorSnapshotState(context.Context, *ReportActorSnapshotStateRequest) (*ReportActorSnapshotStateResponse, error) mustEmbedUnimplementedWorkerServiceServer() } diff --git a/pkg/proto/ateconfigpb/apiconfig.pb.go b/pkg/proto/ateconfigpb/apiconfig.pb.go new file mode 100644 index 0000000000..0eb9400877 --- /dev/null +++ b/pkg/proto/ateconfigpb/apiconfig.pb.go @@ -0,0 +1,187 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11-devel +// protoc v4.25.3 +// source: apiconfig.proto + +package ateconfigpb + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// The unified configuration object for ate-api-server. +type APIConfig struct { + state protoimpl.MessageState `protogen:"open.v1"` + Logging *APILogging `protobuf:"bytes,1,opt,name=logging,proto3" json:"logging,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *APIConfig) Reset() { + *x = APIConfig{} + mi := &file_apiconfig_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *APIConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*APIConfig) ProtoMessage() {} + +func (x *APIConfig) ProtoReflect() protoreflect.Message { + mi := &file_apiconfig_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use APIConfig.ProtoReflect.Descriptor instead. +func (*APIConfig) Descriptor() ([]byte, []int) { + return file_apiconfig_proto_rawDescGZIP(), []int{0} +} + +func (x *APIConfig) GetLogging() *APILogging { + if x != nil { + return x.Logging + } + return nil +} + +type APILogging struct { + state protoimpl.MessageState `protogen:"open.v1"` + // debug, info, warn, or error + Level string `protobuf:"bytes,1,opt,name=level,proto3" json:"level,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *APILogging) Reset() { + *x = APILogging{} + mi := &file_apiconfig_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *APILogging) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*APILogging) ProtoMessage() {} + +func (x *APILogging) ProtoReflect() protoreflect.Message { + mi := &file_apiconfig_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use APILogging.ProtoReflect.Descriptor instead. +func (*APILogging) Descriptor() ([]byte, []int) { + return file_apiconfig_proto_rawDescGZIP(), []int{1} +} + +func (x *APILogging) GetLevel() string { + if x != nil { + return x.Level + } + return "" +} + +var File_apiconfig_proto protoreflect.FileDescriptor + +const file_apiconfig_proto_rawDesc = "" + + "\n" + + "\x0fapiconfig.proto\x12\vateconfigpb\">\n" + + "\tAPIConfig\x121\n" + + "\alogging\x18\x01 \x01(\v2\x17.ateconfigpb.APILoggingR\alogging\"\"\n" + + "\n" + + "APILogging\x12\x14\n" + + "\x05level\x18\x01 \x01(\tR\x05levelB,Z*go.ate.dev/substrate/pkg/proto/ateconfigpbb\x06proto3" + +var ( + file_apiconfig_proto_rawDescOnce sync.Once + file_apiconfig_proto_rawDescData []byte +) + +func file_apiconfig_proto_rawDescGZIP() []byte { + file_apiconfig_proto_rawDescOnce.Do(func() { + file_apiconfig_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_apiconfig_proto_rawDesc), len(file_apiconfig_proto_rawDesc))) + }) + return file_apiconfig_proto_rawDescData +} + +var file_apiconfig_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_apiconfig_proto_goTypes = []any{ + (*APIConfig)(nil), // 0: ateconfigpb.APIConfig + (*APILogging)(nil), // 1: ateconfigpb.APILogging +} +var file_apiconfig_proto_depIdxs = []int32{ + 1, // 0: ateconfigpb.APIConfig.logging:type_name -> ateconfigpb.APILogging + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_apiconfig_proto_init() } +func file_apiconfig_proto_init() { + if File_apiconfig_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_apiconfig_proto_rawDesc), len(file_apiconfig_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_apiconfig_proto_goTypes, + DependencyIndexes: file_apiconfig_proto_depIdxs, + MessageInfos: file_apiconfig_proto_msgTypes, + }.Build() + File_apiconfig_proto = out.File + file_apiconfig_proto_goTypes = nil + file_apiconfig_proto_depIdxs = nil +} diff --git a/pkg/proto/ateconfigpb/apiconfig.proto b/pkg/proto/ateconfigpb/apiconfig.proto new file mode 100644 index 0000000000..f7bb11ef55 --- /dev/null +++ b/pkg/proto/ateconfigpb/apiconfig.proto @@ -0,0 +1,29 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package ateconfigpb; + +option go_package = "go.ate.dev/substrate/pkg/proto/ateconfigpb"; + +// The unified configuration object for ate-api-server. +message APIConfig { + APILogging logging = 1; +} + +message APILogging { + // debug, info, warn, or error + string level = 1; +} diff --git a/pkg/proto/credproviderpb/credprovider.pb.go b/pkg/proto/credproviderpb/credprovider.pb.go index 22021fd174..e2046707ac 100644 --- a/pkg/proto/credproviderpb/credprovider.pb.go +++ b/pkg/proto/credproviderpb/credprovider.pb.go @@ -150,7 +150,7 @@ const file_credprovider_proto_rawDesc = "" + "\x13FetchSecretResponse\x12&\n" + "\fopaque_bytes\x18\x01 \x01(\fB\x03\x80\x01\x01R\vopaqueBytes2j\n" + "\x12CredentialProvider\x12T\n" + - "\vFetchSecret\x12 .credprovider.FetchSecretRequest\x1a!.credprovider.FetchSecretResponse\"\x00B?Z=github.com/agent-substrate/substrate/pkg/proto/credproviderpbb\x06proto3" + "\vFetchSecret\x12 .credprovider.FetchSecretRequest\x1a!.credprovider.FetchSecretResponse\"\x00B/Z-go.ate.dev/substrate/pkg/proto/credproviderpbb\x06proto3" var ( file_credprovider_proto_rawDescOnce sync.Once diff --git a/pkg/proto/credproviderpb/credprovider.proto b/pkg/proto/credproviderpb/credprovider.proto index 3bcbbe5c37..758604c178 100644 --- a/pkg/proto/credproviderpb/credprovider.proto +++ b/pkg/proto/credproviderpb/credprovider.proto @@ -16,7 +16,7 @@ syntax = "proto3"; package credprovider; -option go_package = "github.com/agent-substrate/substrate/pkg/proto/credproviderpb"; +option go_package = "go.ate.dev/substrate/pkg/proto/credproviderpb"; // CredentialProvider is the plugin API a secret backend implements so Substrate // infrastructure (e.g. the egress gateway's credential injector) can fetch an diff --git a/pkg/proto/objectstorepb/v1/objectstore.pb.go b/pkg/proto/objectstorepb/v1/objectstore.pb.go index 83f9bb8c1a..9a59e63c57 100644 --- a/pkg/proto/objectstorepb/v1/objectstore.pb.go +++ b/pkg/proto/objectstorepb/v1/objectstore.pb.go @@ -591,7 +591,7 @@ const file_objectstore_proto_rawDesc = "" + "\fCopySnapshot\x12#.objectstore.v1.CopySnapshotRequest\x1a$.objectstore.v1.CopySnapshotResponse\"\x002f\n" + "\rAssetProvider\x12U\n" + "\n" + - "FetchAsset\x12!.objectstore.v1.FetchAssetRequest\x1a\".objectstore.v1.FetchAssetResponse\"\x00BOZMgithub.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1;objectstorev1b\x06proto3" + "FetchAsset\x12!.objectstore.v1.FetchAssetRequest\x1a\".objectstore.v1.FetchAssetResponse\"\x00B?Z=go.ate.dev/substrate/pkg/proto/objectstorepb/v1;objectstorev1b\x06proto3" var ( file_objectstore_proto_rawDescOnce sync.Once diff --git a/pkg/proto/objectstorepb/v1/objectstore.proto b/pkg/proto/objectstorepb/v1/objectstore.proto index 0f589229ea..f14141d6b2 100644 --- a/pkg/proto/objectstorepb/v1/objectstore.proto +++ b/pkg/proto/objectstorepb/v1/objectstore.proto @@ -16,7 +16,7 @@ syntax = "proto3"; package objectstore.v1; -option go_package = "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1;objectstorev1"; +option go_package = "go.ate.dev/substrate/pkg/proto/objectstorepb/v1;objectstorev1"; // NodeProvider is the plugin API an object storage platform implements // on every node. atelet calls it to move actor snapshots between the node and diff --git a/tools/apitool/go.mod b/tools/apitool/go.mod index 12aef88bf2..2c138c034d 100644 --- a/tools/apitool/go.mod +++ b/tools/apitool/go.mod @@ -1,11 +1,11 @@ -module github.com/agent-substrate/substrate/tools/apitool +module go.ate.dev/substrate/tools/apitool go 1.27.0 -replace github.com/agent-substrate/substrate => ../.. +replace go.ate.dev/substrate => ../.. require ( - github.com/agent-substrate/substrate v0.0.0-00010101000000-000000000000 + go.ate.dev/substrate v0.0.0-00010101000000-000000000000 github.com/bufbuild/protocompile v0.14.1 github.com/google/go-cmp v0.7.0 github.com/spf13/cobra v1.10.2 diff --git a/tools/apitool/internal/cmd/validate.go b/tools/apitool/internal/cmd/validate.go index 31e81cada2..443a97b393 100644 --- a/tools/apitool/internal/cmd/validate.go +++ b/tools/apitool/internal/cmd/validate.go @@ -20,9 +20,9 @@ import ( "github.com/spf13/cobra" - "github.com/agent-substrate/substrate/tools/apitool/internal/exemption" - "github.com/agent-substrate/substrate/tools/apitool/internal/lint" - "github.com/agent-substrate/substrate/tools/apitool/internal/validate" + "go.ate.dev/substrate/tools/apitool/internal/exemption" + "go.ate.dev/substrate/tools/apitool/internal/lint" + "go.ate.dev/substrate/tools/apitool/internal/validate" ) var validateCmd = &cobra.Command{ diff --git a/tools/apitool/internal/exemption/exemption_test.go b/tools/apitool/internal/exemption/exemption_test.go index 1ef0f65be1..bb59032ed3 100644 --- a/tools/apitool/internal/exemption/exemption_test.go +++ b/tools/apitool/internal/exemption/exemption_test.go @@ -19,7 +19,7 @@ import ( "reflect" "testing" - "github.com/agent-substrate/substrate/tools/apitool/internal/exemption" + "go.ate.dev/substrate/tools/apitool/internal/exemption" ) func TestLoadMissingFileIsEmpty(t *testing.T) { diff --git a/tools/apitool/internal/lint/documented.go b/tools/apitool/internal/lint/documented.go index 3d25d30199..65fae26715 100644 --- a/tools/apitool/internal/lint/documented.go +++ b/tools/apitool/internal/lint/documented.go @@ -17,7 +17,7 @@ package lint import ( "strings" - "github.com/agent-substrate/substrate/tools/apitool/internal/model" + "go.ate.dev/substrate/tools/apitool/internal/model" ) var Documented = Rule{ diff --git a/tools/apitool/internal/lint/documented_test.go b/tools/apitool/internal/lint/documented_test.go index 5896c171c0..75151c20ad 100644 --- a/tools/apitool/internal/lint/documented_test.go +++ b/tools/apitool/internal/lint/documented_test.go @@ -19,8 +19,8 @@ import ( "github.com/google/go-cmp/cmp" - "github.com/agent-substrate/substrate/tools/apitool/internal/lint" - "github.com/agent-substrate/substrate/tools/apitool/internal/model" + "go.ate.dev/substrate/tools/apitool/internal/lint" + "go.ate.dev/substrate/tools/apitool/internal/model" ) func TestDocumented(t *testing.T) { diff --git a/tools/apitool/internal/lint/fields.go b/tools/apitool/internal/lint/fields.go index 13b73b6040..0d6c5f75b6 100644 --- a/tools/apitool/internal/lint/fields.go +++ b/tools/apitool/internal/lint/fields.go @@ -21,7 +21,7 @@ import ( "google.golang.org/protobuf/reflect/protoreflect" - "github.com/agent-substrate/substrate/tools/apitool/internal/model" + "go.ate.dev/substrate/tools/apitool/internal/model" ) // typeMatcher reports whether a field has the wanted type, including diff --git a/tools/apitool/internal/lint/fields_test.go b/tools/apitool/internal/lint/fields_test.go index dc8fda8f54..fbc467750b 100644 --- a/tools/apitool/internal/lint/fields_test.go +++ b/tools/apitool/internal/lint/fields_test.go @@ -20,7 +20,7 @@ import ( "github.com/google/go-cmp/cmp" "google.golang.org/protobuf/reflect/protoreflect" - "github.com/agent-substrate/substrate/tools/apitool/internal/model" + "go.ate.dev/substrate/tools/apitool/internal/model" ) func TestCheckFields(t *testing.T) { diff --git a/tools/apitool/internal/lint/fixtures_test.go b/tools/apitool/internal/lint/fixtures_test.go index 8675de2e5f..08bd26a69c 100644 --- a/tools/apitool/internal/lint/fixtures_test.go +++ b/tools/apitool/internal/lint/fixtures_test.go @@ -19,8 +19,8 @@ package lint_test import ( "google.golang.org/protobuf/reflect/protoreflect" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - "github.com/agent-substrate/substrate/tools/apitool/internal/model" + "go.ate.dev/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/tools/apitool/internal/model" ) // methodAPI builds an API whose test.Control service has the one method diff --git a/tools/apitool/internal/lint/lint.go b/tools/apitool/internal/lint/lint.go index 9cf17d302f..133e41e1d7 100644 --- a/tools/apitool/internal/lint/lint.go +++ b/tools/apitool/internal/lint/lint.go @@ -17,7 +17,7 @@ package lint import ( - "github.com/agent-substrate/substrate/tools/apitool/internal/model" + "go.ate.dev/substrate/tools/apitool/internal/model" ) // Finding is one rule violation: which API entity it's about, and why. diff --git a/tools/apitool/internal/lint/method.go b/tools/apitool/internal/lint/method.go index 8ad7a2522a..5e9e96ecce 100644 --- a/tools/apitool/internal/lint/method.go +++ b/tools/apitool/internal/lint/method.go @@ -21,7 +21,7 @@ import ( "google.golang.org/protobuf/reflect/protoreflect" - "github.com/agent-substrate/substrate/tools/apitool/internal/model" + "go.ate.dev/substrate/tools/apitool/internal/model" ) // method is a standard method with everything a rule needs resolved: its diff --git a/tools/apitool/internal/lint/method_test.go b/tools/apitool/internal/lint/method_test.go index 5daab4ff6e..d3e57db3c3 100644 --- a/tools/apitool/internal/lint/method_test.go +++ b/tools/apitool/internal/lint/method_test.go @@ -20,7 +20,7 @@ import ( "github.com/google/go-cmp/cmp" - "github.com/agent-substrate/substrate/tools/apitool/internal/model" + "go.ate.dev/substrate/tools/apitool/internal/model" ) func TestAllOf(t *testing.T) { diff --git a/tools/apitool/internal/lint/naming.go b/tools/apitool/internal/lint/naming.go index c144f5f64f..d2e09f159f 100644 --- a/tools/apitool/internal/lint/naming.go +++ b/tools/apitool/internal/lint/naming.go @@ -22,7 +22,7 @@ import ( "google.golang.org/protobuf/reflect/protoreflect" - "github.com/agent-substrate/substrate/tools/apitool/internal/model" + "go.ate.dev/substrate/tools/apitool/internal/model" ) var RequestNameMatchesMethod = Rule{ diff --git a/tools/apitool/internal/lint/naming_test.go b/tools/apitool/internal/lint/naming_test.go index aa96ab65ec..b8f1040c38 100644 --- a/tools/apitool/internal/lint/naming_test.go +++ b/tools/apitool/internal/lint/naming_test.go @@ -19,8 +19,8 @@ import ( "github.com/google/go-cmp/cmp" - "github.com/agent-substrate/substrate/tools/apitool/internal/lint" - "github.com/agent-substrate/substrate/tools/apitool/internal/model" + "go.ate.dev/substrate/tools/apitool/internal/lint" + "go.ate.dev/substrate/tools/apitool/internal/model" ) func TestRequestNameMatchesMethod(t *testing.T) { diff --git a/tools/apitool/internal/lint/no_oneofs.go b/tools/apitool/internal/lint/no_oneofs.go index d9cae9c0b2..0bca0bf839 100644 --- a/tools/apitool/internal/lint/no_oneofs.go +++ b/tools/apitool/internal/lint/no_oneofs.go @@ -15,7 +15,7 @@ package lint import ( - "github.com/agent-substrate/substrate/tools/apitool/internal/model" + "go.ate.dev/substrate/tools/apitool/internal/model" ) var NoOneofs = Rule{ diff --git a/tools/apitool/internal/lint/no_oneofs_test.go b/tools/apitool/internal/lint/no_oneofs_test.go index d2bc11f030..a6cfd17abd 100644 --- a/tools/apitool/internal/lint/no_oneofs_test.go +++ b/tools/apitool/internal/lint/no_oneofs_test.go @@ -19,8 +19,8 @@ import ( "github.com/google/go-cmp/cmp" - "github.com/agent-substrate/substrate/tools/apitool/internal/lint" - "github.com/agent-substrate/substrate/tools/apitool/internal/model" + "go.ate.dev/substrate/tools/apitool/internal/lint" + "go.ate.dev/substrate/tools/apitool/internal/model" ) func TestNoOneofs(t *testing.T) { diff --git a/tools/apitool/internal/lint/resources.go b/tools/apitool/internal/lint/resources.go index 017801f7c9..6d16e4de4b 100644 --- a/tools/apitool/internal/lint/resources.go +++ b/tools/apitool/internal/lint/resources.go @@ -17,7 +17,7 @@ package lint import ( "google.golang.org/protobuf/reflect/protoreflect" - "github.com/agent-substrate/substrate/tools/apitool/internal/model" + "go.ate.dev/substrate/tools/apitool/internal/model" ) const resourceMetadataTypeFullName protoreflect.FullName = "ateapi.ResourceMetadata" diff --git a/tools/apitool/internal/lint/resources_test.go b/tools/apitool/internal/lint/resources_test.go index 1dce851caf..89a4f43ad6 100644 --- a/tools/apitool/internal/lint/resources_test.go +++ b/tools/apitool/internal/lint/resources_test.go @@ -20,8 +20,8 @@ import ( "github.com/google/go-cmp/cmp" "google.golang.org/protobuf/reflect/protoreflect" - "github.com/agent-substrate/substrate/tools/apitool/internal/lint" - "github.com/agent-substrate/substrate/tools/apitool/internal/model" + "go.ate.dev/substrate/tools/apitool/internal/lint" + "go.ate.dev/substrate/tools/apitool/internal/model" ) func TestResourceMetadata(t *testing.T) { diff --git a/tools/apitool/internal/lint/standard_methods.go b/tools/apitool/internal/lint/standard_methods.go index b2152d05a1..fd3605765a 100644 --- a/tools/apitool/internal/lint/standard_methods.go +++ b/tools/apitool/internal/lint/standard_methods.go @@ -17,8 +17,8 @@ package lint import ( "google.golang.org/protobuf/reflect/protoreflect" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - "github.com/agent-substrate/substrate/tools/apitool/internal/model" + "go.ate.dev/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/tools/apitool/internal/model" ) var StandardMethodReturnsResource = Rule{ diff --git a/tools/apitool/internal/lint/standard_methods_test.go b/tools/apitool/internal/lint/standard_methods_test.go index d61ef968d2..67e6f3d5e6 100644 --- a/tools/apitool/internal/lint/standard_methods_test.go +++ b/tools/apitool/internal/lint/standard_methods_test.go @@ -20,8 +20,8 @@ import ( "github.com/google/go-cmp/cmp" "google.golang.org/protobuf/reflect/protoreflect" - "github.com/agent-substrate/substrate/tools/apitool/internal/lint" - "github.com/agent-substrate/substrate/tools/apitool/internal/model" + "go.ate.dev/substrate/tools/apitool/internal/lint" + "go.ate.dev/substrate/tools/apitool/internal/model" ) func TestStandardMethodReturnsResource(t *testing.T) { diff --git a/tools/apitool/internal/model/model.go b/tools/apitool/internal/model/model.go index 38760d86d6..a40f551890 100644 --- a/tools/apitool/internal/model/model.go +++ b/tools/apitool/internal/model/model.go @@ -26,7 +26,7 @@ import ( "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/reflect/protoregistry" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) // sourceFileName is an arbitrary internal label: Build only ever compiles diff --git a/tools/apitool/internal/model/model_test.go b/tools/apitool/internal/model/model_test.go index 7071fd4628..8b3e9ab74a 100644 --- a/tools/apitool/internal/model/model_test.go +++ b/tools/apitool/internal/model/model_test.go @@ -21,8 +21,8 @@ import ( "github.com/google/go-cmp/cmp" "google.golang.org/protobuf/reflect/protoreflect" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - "github.com/agent-substrate/substrate/tools/apitool/internal/model" + "go.ate.dev/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/tools/apitool/internal/model" ) func buildAPI(t *testing.T, protoBody string) *model.API { diff --git a/tools/apitool/internal/validate/validate.go b/tools/apitool/internal/validate/validate.go index ff9daf291f..ae7b5f5555 100644 --- a/tools/apitool/internal/validate/validate.go +++ b/tools/apitool/internal/validate/validate.go @@ -25,10 +25,10 @@ import ( "path/filepath" "strings" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - "github.com/agent-substrate/substrate/tools/apitool/internal/exemption" - "github.com/agent-substrate/substrate/tools/apitool/internal/lint" - "github.com/agent-substrate/substrate/tools/apitool/internal/model" + "go.ate.dev/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/tools/apitool/internal/exemption" + "go.ate.dev/substrate/tools/apitool/internal/lint" + "go.ate.dev/substrate/tools/apitool/internal/model" ) // serviceName is the RPC service apitool validates: ateapi.proto's public, @@ -90,7 +90,7 @@ func DefaultExemptionsPath() (string, error) { return root + "/tools/apitool/exemptions.json", nil } -const rootModule = "module github.com/agent-substrate/substrate" +const rootModule = "module go.ate.dev/substrate" // repoRoot walks up from the working directory to find the repo root, i.e. // the directory containing the go.mod declaring rootModule. diff --git a/tools/apitool/internal/validate/validate_test.go b/tools/apitool/internal/validate/validate_test.go index 08911415b0..af4ac9c0a9 100644 --- a/tools/apitool/internal/validate/validate_test.go +++ b/tools/apitool/internal/validate/validate_test.go @@ -17,8 +17,8 @@ package validate_test import ( "testing" - "github.com/agent-substrate/substrate/tools/apitool/internal/exemption" - "github.com/agent-substrate/substrate/tools/apitool/internal/validate" + "go.ate.dev/substrate/tools/apitool/internal/exemption" + "go.ate.dev/substrate/tools/apitool/internal/validate" ) func TestExemptions(t *testing.T) { diff --git a/tools/apitool/main.go b/tools/apitool/main.go index 89ae6e0b6c..45cbedc7af 100644 --- a/tools/apitool/main.go +++ b/tools/apitool/main.go @@ -18,7 +18,7 @@ import ( "fmt" "os" - "github.com/agent-substrate/substrate/tools/apitool/internal/cmd" + "go.ate.dev/substrate/tools/apitool/internal/cmd" ) func main() { diff --git a/tools/junittool/go.mod b/tools/junittool/go.mod index 0ec975ac9a..57e3b7110a 100644 --- a/tools/junittool/go.mod +++ b/tools/junittool/go.mod @@ -1,3 +1,3 @@ -module github.com/agent-substrate/substrate/tools/junittool +module go.ate.dev/substrate/tools/junittool go 1.27.0 diff --git a/tools/setup-gcp/main.go b/tools/setup-gcp/main.go index 485e43c6c3..4aa5b679d0 100644 --- a/tools/setup-gcp/main.go +++ b/tools/setup-gcp/main.go @@ -18,7 +18,7 @@ import ( "fmt" "os" - "github.com/agent-substrate/substrate/tools/setup-gcp/cmd" + "go.ate.dev/substrate/tools/setup-gcp/cmd" ) func main() { diff --git a/tools/validate-image-cache/main.go b/tools/validate-image-cache/main.go index a5ae7f9de0..5ca3b8fe68 100644 --- a/tools/validate-image-cache/main.go +++ b/tools/validate-image-cache/main.go @@ -59,10 +59,10 @@ import ( "sync" "time" - "github.com/agent-substrate/substrate/internal/imagecache" - "github.com/agent-substrate/substrate/internal/nodepath" v1 "github.com/google/go-containerregistry/pkg/v1" googlecontainerauth "github.com/google/go-containerregistry/pkg/v1/google" + "go.ate.dev/substrate/internal/imagecache" + "go.ate.dev/substrate/internal/nodepath" "golang.org/x/sys/unix" ) diff --git a/vendor/golang.org/x/oauth2/google/downscope/downscoping.go b/vendor/golang.org/x/oauth2/google/downscope/downscoping.go new file mode 100644 index 0000000000..f704f7eef6 --- /dev/null +++ b/vendor/golang.org/x/oauth2/google/downscope/downscoping.go @@ -0,0 +1,233 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package downscope implements the ability to downscope, or restrict, the +Identity and Access Management permissions that a short-lived Token +can use. Please note that only Google Cloud Storage supports this feature. +For complete documentation, see https://cloud.google.com/iam/docs/downscoping-short-lived-credentials + +To downscope permissions of a source credential, you need to define +a Credential Access Boundary. Said Boundary specifies which resources +the newly created credential can access, an upper bound on the permissions +it has over those resources, and optionally attribute-based conditional +access to the aforementioned resources. For more information on IAM +Conditions, see https://cloud.google.com/iam/docs/conditions-overview. + +This functionality can be used to provide a third party with +limited access to and permissions on resources held by the owner of the root +credential or internally in conjunction with the principle of least privilege +to ensure that internal services only hold the minimum necessary privileges +for their function. + +For example, a token broker can be set up on a server in a private network. +Various workloads (token consumers) in the same network will send authenticated +requests to that broker for downscoped tokens to access or modify specific google +cloud storage buckets. See the NewTokenSource example for an example of how a +token broker would use this package. + +The broker will use the functionality in this package to generate a downscoped +token with the requested configuration, and then pass it back to the token +consumer. These downscoped access tokens can then be used to access Google +Storage resources. For instance, you can create a NewClient from the +"cloud.google.com/go/storage" package and pass in option.WithTokenSource(yourTokenSource)) +*/ +package downscope + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "golang.org/x/oauth2" +) + +const ( + universeDomainPlaceholder = "UNIVERSE_DOMAIN" + identityBindingEndpointTemplate = "https://sts.UNIVERSE_DOMAIN/v1/token" + defaultUniverseDomain = "googleapis.com" +) + +type accessBoundary struct { + AccessBoundaryRules []AccessBoundaryRule `json:"accessBoundaryRules"` +} + +// An AvailabilityCondition restricts access to a given Resource. +type AvailabilityCondition struct { + // An Expression specifies the Cloud Storage objects where + // permissions are available. For further documentation, see + // https://cloud.google.com/iam/docs/conditions-overview + Expression string `json:"expression"` + // Title is short string that identifies the purpose of the condition. Optional. + Title string `json:"title,omitempty"` + // Description details about the purpose of the condition. Optional. + Description string `json:"description,omitempty"` +} + +// An AccessBoundaryRule Sets the permissions (and optionally conditions) +// that the new token has on given resource. +type AccessBoundaryRule struct { + // AvailableResource is the full resource name of the Cloud Storage bucket that the rule applies to. + // Use the format //storage.googleapis.com/projects/_/buckets/bucket-name. + AvailableResource string `json:"availableResource"` + // AvailablePermissions is a list that defines the upper bound on the available permissions + // for the resource. Each value is the identifier for an IAM predefined role or custom role, + // with the prefix inRole:. For example: inRole:roles/storage.objectViewer. + // Only the permissions in these roles will be available. + AvailablePermissions []string `json:"availablePermissions"` + // An Condition restricts the availability of permissions + // to specific Cloud Storage objects. Optional. + // + // A Condition can be used to make permissions available for specific objects, + // rather than all objects in a Cloud Storage bucket. + Condition *AvailabilityCondition `json:"availabilityCondition,omitempty"` +} + +type downscopedTokenResponse struct { + AccessToken string `json:"access_token"` + IssuedTokenType string `json:"issued_token_type"` + TokenType string `json:"token_type"` + ExpiresIn int `json:"expires_in"` +} + +// DownscopingConfig specifies the information necessary to request a downscoped token. +type DownscopingConfig struct { + // RootSource is the TokenSource used to create the downscoped token. + // The downscoped token therefore has some subset of the accesses of + // the original RootSource. + RootSource oauth2.TokenSource + // Rules defines the accesses held by the new + // downscoped Token. One or more AccessBoundaryRules are required to + // define permissions for the new downscoped token. Each one defines an + // access (or set of accesses) that the new token has to a given resource. + // There can be a maximum of 10 AccessBoundaryRules. + Rules []AccessBoundaryRule + // UniverseDomain is the default service domain for a given Cloud universe. + // The default value is "googleapis.com". Optional. + UniverseDomain string +} + +// identityBindingEndpoint returns the identity binding endpoint with the +// configured universe domain. +func (dc *DownscopingConfig) identityBindingEndpoint() string { + if dc.UniverseDomain == "" { + return strings.Replace(identityBindingEndpointTemplate, universeDomainPlaceholder, defaultUniverseDomain, 1) + } + return strings.Replace(identityBindingEndpointTemplate, universeDomainPlaceholder, dc.UniverseDomain, 1) +} + +// A downscopingTokenSource is used to retrieve a downscoped token with restricted +// permissions compared to the root Token that is used to generate it. +type downscopingTokenSource struct { + // ctx is the context used to query the API to retrieve a downscoped Token. + ctx context.Context + // config holds the information necessary to generate a downscoped Token. + config DownscopingConfig + // identityBindingEndpoint is the identity binding endpoint with the + // configured universe domain. + identityBindingEndpoint string +} + +// NewTokenSource returns a configured downscopingTokenSource. +func NewTokenSource(ctx context.Context, conf DownscopingConfig) (oauth2.TokenSource, error) { + if conf.RootSource == nil { + return nil, fmt.Errorf("downscope: rootSource cannot be nil") + } + if len(conf.Rules) == 0 { + return nil, fmt.Errorf("downscope: length of AccessBoundaryRules must be at least 1") + } + if len(conf.Rules) > 10 { + return nil, fmt.Errorf("downscope: length of AccessBoundaryRules may not be greater than 10") + } + for _, val := range conf.Rules { + if val.AvailableResource == "" { + return nil, fmt.Errorf("downscope: all rules must have a nonempty AvailableResource: %+v", val) + } + if len(val.AvailablePermissions) == 0 { + return nil, fmt.Errorf("downscope: all rules must provide at least one permission: %+v", val) + } + } + return downscopingTokenSource{ + ctx: ctx, + config: conf, + identityBindingEndpoint: conf.identityBindingEndpoint(), + }, nil +} + +// Token() uses a downscopingTokenSource to generate an oauth2 Token. +// Do note that the returned TokenSource is an oauth2.StaticTokenSource. If you wish +// to refresh this token automatically, then initialize a locally defined +// TokenSource struct with the Token held by the StaticTokenSource and wrap +// that TokenSource in an oauth2.ReuseTokenSource. +func (dts downscopingTokenSource) Token() (*oauth2.Token, error) { + + downscopedOptions := struct { + Boundary accessBoundary `json:"accessBoundary"` + }{ + Boundary: accessBoundary{ + AccessBoundaryRules: dts.config.Rules, + }, + } + + tok, err := dts.config.RootSource.Token() + if err != nil { + return nil, fmt.Errorf("downscope: unable to obtain root token: %v", err) + } + + b, err := json.Marshal(downscopedOptions) + if err != nil { + return nil, fmt.Errorf("downscope: unable to marshal AccessBoundary payload %v", err) + } + + form := url.Values{} + form.Add("grant_type", "urn:ietf:params:oauth:grant-type:token-exchange") + form.Add("subject_token_type", "urn:ietf:params:oauth:token-type:access_token") + form.Add("requested_token_type", "urn:ietf:params:oauth:token-type:access_token") + form.Add("subject_token", tok.AccessToken) + form.Add("options", string(b)) + + myClient := oauth2.NewClient(dts.ctx, nil) + resp, err := myClient.PostForm(dts.identityBindingEndpoint, form) + if err != nil { + return nil, fmt.Errorf("unable to generate POST Request %v", err) + } + defer resp.Body.Close() + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("downscope: unable to read response body: %v", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("downscope: unable to exchange token; %v. Server responded: %s", resp.StatusCode, respBody) + } + + var tresp downscopedTokenResponse + + err = json.Unmarshal(respBody, &tresp) + if err != nil { + return nil, fmt.Errorf("downscope: unable to unmarshal response body: %v", err) + } + + // an exchanged token that is derived from a service account (2LO) has an expired_in value + // a token derived from a users token (3LO) does not. + // The following code uses the time remaining on rootToken for a user as the value for the + // derived token's lifetime + var expiryTime time.Time + if tresp.ExpiresIn > 0 { + expiryTime = time.Now().Add(time.Duration(tresp.ExpiresIn) * time.Second) + } else { + expiryTime = tok.Expiry + } + + newToken := &oauth2.Token{ + AccessToken: tresp.AccessToken, + TokenType: tresp.TokenType, + Expiry: expiryTime, + } + return newToken, nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 67eb30ed51..1c6bba7903 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1290,6 +1290,7 @@ golang.org/x/net/websocket golang.org/x/oauth2 golang.org/x/oauth2/authhandler golang.org/x/oauth2/google +golang.org/x/oauth2/google/downscope golang.org/x/oauth2/google/externalaccount golang.org/x/oauth2/google/internal/externalaccountauthorizeduser golang.org/x/oauth2/google/internal/impersonate