From 2f56f1797db9e8df78bbc49cec928743c37abd3d Mon Sep 17 00:00:00 2001 From: Da Huang <127548902+git286@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:09:54 +0000 Subject: [PATCH 1/6] docs: say when to mark proto fields debug_redact and what it covers (#2267) Part of #1743. This is the doc I said I would send in #1942. Three parts: - **API style guide, new section 12.** A field that holds a secret must carry `debug_redact`. When to label it (tokens, keys, secret bytes, user values that may hold secrets), when not (references to a secret, identifiers, page tokens), and to prefer the leaf field over the whole message. Examples are all from the public API. - **New `docs/dev/best-practices/logging.md`**, the logging counterpart of the metrics and tracing guides, with a section "Logging protos that hold secrets". How masking works in the Go servers: the handler `serverboot.InitLogger` installs, what it masks, and what it can't see (protos in slices, maps or structs, protos turned into strings, `google.protobuf.Any`, spans, metrics, errors, actor event OTLP). I checked that protobuf-go's `String()`, `%v`, `prototext` and `protojson` all print labeled fields in clear, so the advice is to log the proto as its own attribute. - **`observability.md`, a short "Secrets in Component Logs" note** for people reading the logs: component logs show `[REDACTED]` for these fields, and actor container output is not touched. It links to the guide. I split it this way because the API guide is about the proto surface, and the logging details are Go implementation that will change. They sit next to the other o11y developer guides. --- docs/api-style-guide.md | 24 ++++++++++++++ docs/dev/best-practices/logging.md | 53 ++++++++++++++++++++++++++++++ docs/observability.md | 6 ++++ 3 files changed, 83 insertions(+) create mode 100644 docs/dev/best-practices/logging.md diff --git a/docs/api-style-guide.md b/docs/api-style-guide.md index c78189b57f..8a733ad102 100644 --- a/docs/api-style-guide.md +++ b/docs/api-style-guide.md @@ -718,3 +718,27 @@ rpc SuspendActor(SuspendActorRequest) returns (SuspendActorResponse) { Rules: - `resource` is the resource's message name, and that message **must** carry a `(resource)` option. - A subresource's method names the subresource, not its parent: `EgressPolicy` for `GetActorEgressPolicy`. + +--- + +## 12. Sensitive Fields (`debug_redact`) + +*Substrate-specific. No AIP covers this.* + +A field that holds a secret **must** carry the standard protobuf `debug_redact` option. Components that log through Substrate's shared log handler mask labeled fields, so request and response bodies can be logged without leaking the secrets in them. + +```proto +message EnvVar { + string name = 1; + string value = 2 [ debug_redact = true ]; +} +``` + +Rules: +- Label a field whose value grants access to something, or is secret material provided by the user: bearer tokens and JWTs (`MintActorJwtResponse.actor_jwt`), keys, passwords, secret bytes, and user-supplied values that may hold secrets (`EnvVar.value`). +- Do not label a reference to a secret, such as an `ate-secret://` URI (`CredentialHeader.credential_uri`). It names the secret and grants nothing by itself. Do not label identifiers, names, or list cursors (`page_token`). +- Label a URL that grants access by itself, such as a signed URL, even when it points to a secret. +- Prefer labeling the leaf field over the message that contains it, so the rest of the message stays readable in logs: a log line still shows which env var was set, but not its value. Label a message-typed field only when the whole message is secret. +- Add the label in the same change that adds the field. A label added later does not help lines already logged. + +Which processes use the shared log handler, and what it does and does not cover, is described in [Logging Best Practices](dev/best-practices/logging.md#logging-protos-that-hold-secrets). diff --git a/docs/dev/best-practices/logging.md b/docs/dev/best-practices/logging.md new file mode 100644 index 0000000000..a1c441a4d7 --- /dev/null +++ b/docs/dev/best-practices/logging.md @@ -0,0 +1,53 @@ +# Logging Best Practices + +This document covers how to write logs in a Substrate component. It is the +logging counterpart of [Metrics Best Practices](metrics.md) and +[Tracing Best Practices](tracing.md). + +For how to read actor and component logs, see +[Actor Observability](../../observability.md#1-logging). + +## Logging protos that hold secrets + +Fields labeled `debug_redact` (see +[API style guide section 12](../../api-style-guide.md#12-sensitive-fields-debug_redact)) +are masked when a process that uses the shared handler below logs a proto. +The label applies to every proto in this repository, not only the public API: +atelet, ateworker, credential provider, and object store protos are logged the +same way. + +Masking is done by the shared slog handler in `internal/contextlogging`, which +`serverboot.InitLogger` and `InitLoggerWithWriter` install as the default +logger. A server must set up logging through one of them; a process that +builds its own handler gets no masking. These processes build their own today: + +- `podcertcontroller` +- `atenet sdsmint` +- `boomer-worker` +- plugins under `internal/plugins/`, such as `gsm-credential-provider`. A + plugin there imports only `pkg/` (see its README), so it cannot call + `serverboot`. If `gsm-credential-provider` logged the + `FetchSecretResponse` it serves, `opaque_bytes` would show in clear. The + plugins under `cmd/` use `serverboot` and are masked. + +The handler masks labeled fields in a proto it is handed as an attribute value +(`slog.Any("resp", resp)`), inside a `slog.Group`, as the result of a +`slog.LogValuer`, or bound with `logger.With`. This covers the request and +response bodies the gRPC interceptors log. A singular string field becomes +`[REDACTED]`. A field of any other kind (bytes, repeated, map, message) is +dropped, because no placeholder fits it. The caller's message is not modified. + +- Log a proto as its own attribute. The handler cannot see a proto that is: + - inside a slice, map, or plain Go struct: `slog.Any("env", []*EnvVar{...})`; + - turned into a string first. protobuf-go's `String()`, `%v`, `prototext`, + and `protojson` all print labeled fields in clear; + - packed in a `google.protobuf.Any`. +- The label only affects logs written by that handler. Span attributes, metric + labels, error messages, and the OTLP copy of actor events (built by + `internal/actorevent`) are not masked; keep secrets out of them. +- Adding or removing a label fails the tests in `internal/protoredact` until + the list of labeled fields there is updated, so the change is reviewed as a + decision about what logs may show. + +The exact list of what the handler covers is on `ContextHandler.Handle` in +`internal/contextlogging`. diff --git a/docs/observability.md b/docs/observability.md index de0d219eb8..e989deeb1d 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -260,6 +260,12 @@ jsonPayload.msg="Actor usage sampled" AND jsonPayload."ate.actor.uid"="8f2a…" **Do not put actor identity on a log-based metric.** Aggregating these events in the log store is what they are for, but a log-based metric built over them must label only by the bounded set (template, sandbox class, source, pool) — promoting `ate.actor.uid` or `ate.actor.name` into a metric label reintroduces exactly the per-actor cardinality this split keeps out of the TSDB. +### Secrets in Component Logs + +Control plane and node component logs mask proto fields marked `debug_redact`, such as actor JWTs and env var values: a request or response body in a log line shows `[REDACTED]` in their place. Not every process masks them; the developer guide below lists the ones that do not. An actor's own container output is not touched. + +> **Developer Guide:** For how to log protos that hold secrets, see [Logging Best Practices](dev/best-practices/logging.md). + --- ## 2. Metrics From 6885d0ce5ddb554bf09bf268ec02a813b1ac90c3 Mon Sep 17 00:00:00 2001 From: Dmitry Berkovich <54346214+dberkov@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:31:44 +0000 Subject: [PATCH 2/6] atelet: stage sandbox assets per actor from the node-local file cache (#2699) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Steps 1 and 2 of #2675: sandbox assets move onto the node-local file cache, with every actor holding hard links to its assets under its own directory. No change to how assets are resolved; Run and Restore still take them from the request. The runtime-hash index and the prewarm rework are later PRs. **Reads well commit by commit.** 1. **`filecache: serve directory trees and keep execute bits`** — library only. `GetDirTo` adds directory entries: a `DirFetcher` fills a directory the store creates, publication admits only directories and regular files, and a hit mirrors the tree at the destination as directories the consumer owns with every file a hard link, built under the hit lock. Published files keep the execute bits the fetcher set (0555 for a binary), and a fetcher that leaves a symlink is rejected. Eviction treats a tree as linked while any file under it is linked; retirement stays whole-entry, so a published tree is never missing a file. Serving a tree entry through a file method, or the reverse, is an error rather than a miss. 2. **`atelet: stage sandbox assets per actor from the file cache`** — `ensureSandboxAssets` takes a destination and links each asset from the cache into `/sandbox-assets/`; the gVisor release is a tree with runsc next to its helpers. One download per node, shared by concurrent callers; a running actor protects its assets by link count. Run fetches into a staging dir before it resets the actor's other directories, so a failed fetch leaves a live activation untouched, and moves the staged set into place together with the sandbox record after the reset, so the two never disagree. An actor keeps its staged links, and so its binaries in the cache, while it stays resumable on this node through its local snapshot; Restore empties the dir before staging, since it may bring another runtime. Two calls staging the same assets for one actor at once both succeed: a hard link is complete the moment it exists and the cache places a tree by rename. Both workers derive the actor's sandbox assets dir from `actor_uid` and reject a request naming any other, so a request cannot choose the root its binary paths are checked against. `ActorDirs` gains `sandbox_assets_dir`; the kata process cleanup identifies a sandbox's VMM by the same derived directory. Startup probes a hard link from the cache dir into the actors dir and refuses to start if it fails, since every hit would fail the same way. The static-files layout and `nodepath.StaticFilesDir` are removed. Prewarm stages into a scratch directory and removes it. Flags: `--sandbox-asset-cache-dir`, `--sandbox-asset-fetch-timeout` (5m), and `--sandbox-asset-cache-min-age` (1h, recorded for the GC pass to come). **Roll together.** atelet and both ateoms change the asset path contract in the same commit. Old nodes keep a stale `static-files` directory until recreated; nothing cleans it up, per the pre-1.0 compatibility policy. **Tests.** Eleven new filecache tests (tree miss/hit with inode and mode checks, singleflight, kind mismatch, exec bits, tree eviction). In atelet: two concurrent Runs share one download and end up with one inode; the gVisor release path is covered end to end for the first time, including that the tarball is not published with the tree; a canceled caller no longer stops a download others can use, which replaces the old canceled-request test whose premise does not hold with detached fetches. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- cmd/atelet/internal/ateletpath/ateletpath.go | 47 ++- .../internal/ateletpath/ateletpath_test.go | 1 + cmd/atelet/internal/filecache/doc.go | 11 +- cmd/atelet/internal/filecache/evict.go | 10 +- cmd/atelet/internal/filecache/evict_test.go | 102 ++++++ cmd/atelet/internal/filecache/getdirto.go | 178 +++++++++++ .../internal/filecache/getdirto_test.go | 301 ++++++++++++++++++ .../internal/filecache/getfilecopyto.go | 12 +- cmd/atelet/internal/filecache/getfileto.go | 105 ++++-- .../internal/filecache/getfileto_test.go | 31 ++ .../internal/snapshottracker/tracker_test.go | 5 +- cmd/atelet/lifecycle_test.go | 192 ++++++++++- cmd/atelet/main.go | 100 +++++- cmd/atelet/main_test.go | 148 +++++---- cmd/atelet/sandbox_asset_cache.go | 113 +++++++ cmd/atelet/sandbox_asset_cache_linux_test.go | 75 +++++ cmd/atelet/sandbox_asset_cache_test.go | 102 ++++++ cmd/atelet/sandbox_assets.go | 263 ++++++++------- cmd/atelet/sandbox_assets_plugin_test.go | 144 +++++---- cmd/atelet/sandbox_assets_test.go | 291 +++++++++++++++++ cmd/atelet/sandbox_prewarm.go | 35 +- cmd/atelet/sandbox_prewarm_test.go | 67 ++-- cmd/atelet/snapshot_transfer_test.go | 1 + cmd/atelet/worker_sandbox_syncer.go | 11 +- cmd/atelet/worker_sandbox_syncer_test.go | 9 +- cmd/atelet/worker_sandbox_versions.go | 9 +- cmd/atelet/worker_sandbox_versions_test.go | 2 +- cmd/ateworker-gvisor/fidelity_test.go | 1 + cmd/ateworker-gvisor/main.go | 23 +- cmd/ateworker-gvisor/main_test.go | 55 +++- cmd/ateworker-gvisor/runsc_test.go | 1 + .../internal/kata/cleanup_linux.go | 10 +- .../internal/kata/cleanup_linux_test.go | 8 +- cmd/ateworker-microvm/main.go | 14 +- cmd/ateworker-microvm/restore.go | 70 +++- cmd/ateworker-microvm/restore_test.go | 183 ++++++++++- cmd/ateworker-microvm/run.go | 8 +- internal/nodepath/nodepath.go | 9 +- internal/proto/ateworkerpb/ateworker.pb.go | 24 +- internal/proto/ateworkerpb/ateworker.proto | 6 + internal/resources/validate.go | 24 +- internal/resources/validate_test.go | 28 ++ 42 files changed, 2419 insertions(+), 410 deletions(-) create mode 100644 cmd/atelet/internal/filecache/getdirto.go create mode 100644 cmd/atelet/internal/filecache/getdirto_test.go create mode 100644 cmd/atelet/sandbox_asset_cache.go create mode 100644 cmd/atelet/sandbox_asset_cache_linux_test.go create mode 100644 cmd/atelet/sandbox_asset_cache_test.go diff --git a/cmd/atelet/internal/ateletpath/ateletpath.go b/cmd/atelet/internal/ateletpath/ateletpath.go index 283c94307f..1711abca83 100644 --- a/cmd/atelet/internal/ateletpath/ateletpath.go +++ b/cmd/atelet/internal/ateletpath/ateletpath.go @@ -34,21 +34,20 @@ var ( // moved through the object-store plugin. It lives under BasePath, which the // node plugin shares with atelet. SnapshotScratchDir = filepath.Join(nodepath.BasePath, "snapshot-scratch") -) - -// AssetPath returns the cache path for a single-file sandbox asset -func AssetPath(sha256 string) string { - return filepath.Join(nodepath.StaticFilesDir, "asset-"+sha256) -} -// GVisorReleaseDir is the directory a gVisor release tarball (gvisor.tar.bz2, -// containing runsc plus its gvisor-bin/ helper binaries) is extracted into, -// content-addressed by the tarball's sha256. runsc requires the gvisor-bin/ -// subdirectory to sit next to it, so the whole release is kept together under -// one directory rather than as loose files in StaticFilesDir. -func GVisorReleaseDir(sha256 string) string { - return filepath.Join(nodepath.StaticFilesDir, "gvisor-"+sha256) -} + // SandboxAssetCacheDir is the node-local sandbox asset cache (see + // cmd/atelet/internal/filecache): runsc releases, micro-VM kernels, VMMs + // and guest images, content-addressed. It lives under BasePath, on the + // same mount as ActorsDir, because an actor's assets are hard links from + // here into its SandboxAssetsDir. + SandboxAssetCacheDir = filepath.Join(nodepath.BasePath, "sandbox-asset-cache") + + // SandboxAssetScratchDir holds short-lived destination directories for + // asset fetches that have no actor, such as prewarming a SandboxConfig's + // default version. Wiped at startup. Under BasePath so the links the + // cache serves into it are on the cache's mount. + SandboxAssetScratchDir = filepath.Join(nodepath.BasePath, "sandbox-asset-scratch") +) func ActorPath(actorUID string) string { return filepath.Join( @@ -81,6 +80,25 @@ func ActorSnapshotStateFile(actorUID string) string { ) } +// SandboxAssetsDir holds the actor's sandbox runtime assets, one entry per +// asset name, each a hard link (or a directory of hard links, for a gVisor +// release) from SandboxAssetCacheDir. The worker runs binaries from here +// only, and derives the same path itself to check a request against. +func SandboxAssetsDir(actorUID string) string { + return nodepath.ActorSandboxAssetsDir(actorUID) +} + +// SandboxAssetsStagingDir is where Run stages an activation's assets before +// moving them into SandboxAssetsDir, so a failed fetch leaves whatever a +// live activation has there untouched. Not among the directories +// resetActorDirs empties. +func SandboxAssetsStagingDir(actorUID string) string { + return filepath.Join( + ActorPath(actorUID), + "sandbox-assets.staging", + ) +} + func OCIBundleDir(actorUID string) string { return filepath.Join( ActorPath(actorUID), @@ -206,5 +224,6 @@ func ActorDirs(actorUID string) *ateworkerpb.ActorDirs { DurableDirVolumeMountsDir: DurableDirVolumeMountsDir(actorUID), SystemInfoVolumeRootsDir: SystemInfoVolumeRootsDir(actorUID), VolumesDir: VolumesDir(actorUID), + SandboxAssetsDir: SandboxAssetsDir(actorUID), } } diff --git a/cmd/atelet/internal/ateletpath/ateletpath_test.go b/cmd/atelet/internal/ateletpath/ateletpath_test.go index d493b0760c..fdb0beacd8 100644 --- a/cmd/atelet/internal/ateletpath/ateletpath_test.go +++ b/cmd/atelet/internal/ateletpath/ateletpath_test.go @@ -33,6 +33,7 @@ func TestActorDirs(t *testing.T) { "durable_dir_volume_mounts_dir": actorDirs.GetDurableDirVolumeMountsDir(), "system_info_volume_roots_dir": actorDirs.GetSystemInfoVolumeRootsDir(), "volumes_dir": actorDirs.GetVolumesDir(), + "sandbox_assets_dir": actorDirs.GetSandboxAssetsDir(), } seen := map[string]string{} for field, dir := range under { diff --git a/cmd/atelet/internal/filecache/doc.go b/cmd/atelet/internal/filecache/doc.go index 4ecb3e73e8..cbea8ef40a 100644 --- a/cmd/atelet/internal/filecache/doc.go +++ b/cmd/atelet/internal/filecache/doc.go @@ -39,6 +39,10 @@ // - GetFileTo serves a hit as a hard link, created in the same locked // step that resolves the hit. Evicting the entry later removes only the // cache's own link; the consumer's file stays valid. +// - GetDirTo serves a hit as a directory the consumer owns, in which every +// file is a hard link, built in that same locked step. The consumer's +// tree stays valid after eviction, and the entry counts as linked while +// any of its files is. // - GetFileCopyTo serves a private copy. The consumer owns the inode and // may mutate it, and a copy in progress reads a held-open handle, so a // concurrent eviction cannot corrupt it. @@ -46,9 +50,10 @@ // than the window between publication and a consumer's use becoming // visible. Size it for the slowest consumer. // -// Entries are published read-only (0444): a consumer writing through its -// hard link fails with EACCES instead of corrupting the shared bytes. -// Consumers that must mutate a file use GetFileCopyTo. +// Files are published read-only (0444, keeping any execute bits so a cached +// binary stays runnable): a consumer writing through its hard link fails +// with EACCES instead of corrupting the shared bytes. Consumers that must +// mutate a file use GetFileCopyTo. // // Keys are identities, not addresses. The store never invalidates: a key // must name content that is immutable at its source (see URIKey); changed diff --git a/cmd/atelet/internal/filecache/evict.go b/cmd/atelet/internal/filecache/evict.go index 59b629d09d..caa15b675b 100644 --- a/cmd/atelet/internal/filecache/evict.go +++ b/cmd/atelet/internal/filecache/evict.go @@ -287,10 +287,12 @@ func (s *Store) listCandidates(ctx context.Context) ([]evictCandidate, error) { } // sizeEntry sums an entry's regular files' allocated bytes and reports -// whether its data file carries consumer hard links (a directory-tree entry -// cannot, so it reports unlinked and relies on the min age and, later, a -// root set). Files that vanish mid-walk (a concurrent retire) are skipped. +// whether its data carries consumer hard links: the data file itself, or +// for a tree entry any file under it, since a consumer that keeps part of a +// mirrored tree still depends on the entry. Files that vanish mid-walk (a +// concurrent retire) are skipped. func (s *Store) sizeEntry(entryDir string) (size int64, linked bool, err error) { + dataPath := filepath.Join(entryDir, dataName) err = filepath.WalkDir(entryDir, func(path string, d fs.DirEntry, err error) error { if err != nil { if errors.Is(err, fs.ErrNotExist) { @@ -309,7 +311,7 @@ func (s *Store) sizeEntry(entryDir string) (size int64, linked bool, err error) return err } size += allocatedSize(info) - if path == filepath.Join(entryDir, dataName) { + if path == dataPath || strings.HasPrefix(path, dataPath+string(filepath.Separator)) { if st, ok := info.Sys().(*syscall.Stat_t); ok && st.Nlink > 1 { linked = true } diff --git a/cmd/atelet/internal/filecache/evict_test.go b/cmd/atelet/internal/filecache/evict_test.go index 4858c342a9..d5ecc7f11e 100644 --- a/cmd/atelet/internal/filecache/evict_test.go +++ b/cmd/atelet/internal/filecache/evict_test.go @@ -18,6 +18,7 @@ import ( "context" "os" "path/filepath" + "slices" "strings" "testing" "time" @@ -387,3 +388,104 @@ func TestRetireEntryRetiresUnchangedEntry(t *testing.T) { t.Errorf("sweep stats = %+v, want RetiredRemoved=1", stats) } } + +// cacheTree materializes a tree entry through GetDirTo, backdates its +// last-use clock by age, and returns its key and the consumer's directory. +// keepFiles is the subset of the tree the consumer keeps linked (nil: the +// whole consumer dir is removed, as when an actor dir is wiped). +func cacheTree(t *testing.T, s *Store, name string, age time.Duration, keepFiles []string) (Key, string) { + t.Helper() + key := URIKey("test://tree/" + name) + fetch, _ := treeFetcher(releaseTree) + dst := dstPath(t, s, "evict-tree-"+name) + if err := s.GetDirTo(context.Background(), key, dst, fetch); err != nil { + t.Fatalf("GetDirTo(%s): %v", name, err) + } + if keepFiles == nil { + if err := os.RemoveAll(dst); err != nil { + t.Fatal(err) + } + } else { + for rel := range releaseTree { + if slices.Contains(keepFiles, rel) { + continue + } + if err := os.Remove(filepath.Join(dst, rel)); err != nil { + t.Fatal(err) + } + } + } + stale := time.Now().Add(-age) + if err := os.Chtimes(s.entryDir(key), stale, stale); err != nil { + t.Fatal(err) + } + return key, dst +} + +func TestEvictUnusedUnlinkedTreeFreesItsBytes(t *testing.T) { + s := newTestStore(t, WithMinAge(0)) + key, _ := cacheTree(t, s, "gone", time.Hour, nil) + size, linked, err := s.sizeEntry(s.entryDir(key)) + if err != nil { + t.Fatal(err) + } + if linked { + t.Fatal("tree with no consumer links reported linked") + } + + stats, err := s.EvictUnused(context.Background(), evictAll, false) + if err != nil { + t.Fatalf("EvictUnused: %v", err) + } + if stats.Retired != 1 || stats.FreedBytes != size || stats.PendingBytes != 0 { + t.Errorf("stats = %+v, want one eviction freeing %d bytes", stats, size) + } + if entryExists(t, s, key) { + t.Error("entry still published after eviction") + } + noRetiredLeftovers(t, s) +} + +// TestEvictUnusedPartlyLinkedTreeIsSafeForConsumer pins that a tree counts +// as linked while a consumer keeps any one of its files, and that evicting +// it leaves the consumer's files intact. +func TestEvictUnusedPartlyLinkedTreeIsSafeForConsumer(t *testing.T) { + s := newTestStore(t, WithMinAge(0)) + key, dst := cacheTree(t, s, "held", time.Hour, []string{"gvisor-bin/LICENSE"}) + if _, linked, err := s.sizeEntry(s.entryDir(key)); err != nil || !linked { + t.Fatalf("tree with one consumer link reported linked=%v, %v; want true", linked, err) + } + + stats, err := s.EvictUnused(context.Background(), evictAll, false) + if err != nil { + t.Fatalf("EvictUnused: %v", err) + } + if stats.Retired != 1 || stats.FreedBytes != 0 || stats.PendingBytes == 0 { + t.Errorf("stats = %+v, want one eviction counted as pending bytes", stats) + } + if entryExists(t, s, key) { + t.Error("entry still published after eviction") + } + got, err := os.ReadFile(filepath.Join(dst, "gvisor-bin", "LICENSE")) + if err != nil || string(got) != releaseTree["gvisor-bin/LICENSE"].content { + t.Errorf("consumer's kept file after eviction: %q, %v", got, err) + } + noRetiredLeftovers(t, s) +} + +func TestEvictUnusedPrefersUnlinkedTreeOverOlderLinkedTree(t *testing.T) { + s := newTestStore(t, WithMinAge(0)) + linkedKey, _ := cacheTree(t, s, "old-linked", 3*time.Hour, []string{"runsc"}) + unlinkedKey, _ := cacheTree(t, s, "new-unlinked", time.Hour, nil) + + stats, err := s.EvictUnused(context.Background(), 1, false) + if err != nil { + t.Fatalf("EvictUnused: %v", err) + } + if stats.Retired != 1 { + t.Fatalf("stats = %+v, want exactly one eviction", stats) + } + if entryExists(t, s, unlinkedKey) || !entryExists(t, s, linkedKey) { + t.Error("evicted the older linked tree instead of the newer unlinked one") + } +} diff --git a/cmd/atelet/internal/filecache/getdirto.go b/cmd/atelet/internal/filecache/getdirto.go new file mode 100644 index 0000000000..c4b7045b98 --- /dev/null +++ b/cmd/atelet/internal/filecache/getdirto.go @@ -0,0 +1,178 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filecache + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "syscall" + "time" +) + +// DirFetcher downloads a directory-tree artifact (an extracted release +// archive, say) into dstDir, which the store has created empty. The tree +// may hold only directories and regular files. As with FileFetcher, the +// store runs it at most once per key across concurrent callers, detached +// from their contexts and bounded by the fetch timeout, and its error +// reaches every waiting caller wrapped with %w. +type DirFetcher func(ctx context.Context, dstDir string) error + +// GetDirTo materializes the tree identified by key at dst, fetching it with +// fetch if it is not cached. dst must be an absolute path that does not +// exist yet and lives on the cache's filesystem. On success it is a +// directory the caller owns, mirroring the cached tree, in which every +// regular file is a hard link to the read-only cache copy. The caller keeps +// a valid tree regardless of later eviction, and may add files of its own +// without touching the cache; writing through a linked file fails loudly. +// +// dst is built beside its final name and renamed into place, so it never +// exists half-linked: an existing dst, reported as fs.ErrExist, is a +// complete tree. Concurrent calls for one key share a single fetch, as in +// GetFileTo. +func (s *Store) GetDirTo(ctx context.Context, key Key, dst string, fetch DirFetcher) error { + if fetch == nil { + return errors.New("filecache: nil DirFetcher") + } + return s.getTo(ctx, key, dst, fetchFunc(fetch), kindTree, s.linkTreeOut) +} + +// linkTreeOut mirrors key's published tree at dst, hard-linking every +// regular file, and touches the entry's last-use clock. It reports false +// (and no error) on a cache miss. The tree is built beside dst and renamed +// into place, so dst appears complete or not at all: a caller that finds it +// already there may use it. hitMu is held shared for the whole walk, so +// eviction cannot retire the entry while its files are being linked. +func (s *Store) linkTreeOut(key Key, dst string) (bool, error) { + s.hitMu.RLock() + defer s.hitMu.RUnlock() + + src := s.dataPath(key) + fi, err := os.Stat(src) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return false, nil + } + return false, fmt.Errorf("while checking cache for %v: %w", key, err) + } + if !fi.IsDir() { + return false, fmt.Errorf("entry for %v is a file, not a directory tree (use GetFileTo)", key) + } + if _, err := os.Lstat(dst); err == nil { + return false, fmt.Errorf("destination %s already exists: %w", dst, fs.ErrExist) + } else if !errors.Is(err, fs.ErrNotExist) { + return false, fmt.Errorf("while checking destination %s: %w", dst, err) + } + tmp, err := os.MkdirTemp(filepath.Dir(dst), "."+filepath.Base(dst)+".linking-") + if err != nil { + return false, fmt.Errorf("while creating %s: %w", dst, err) + } + if err := os.Chmod(tmp, fi.Mode().Perm()); err != nil { + _ = os.RemoveAll(tmp) + return false, fmt.Errorf("while creating %s: %w", dst, err) + } + if err := linkTree(src, tmp); err != nil { + _ = os.RemoveAll(tmp) + return false, fmt.Errorf("while linking %v to %s: %w", key, dst, err) + } + if err := os.Rename(tmp, dst); err != nil { + _ = os.RemoveAll(tmp) + // A concurrent caller for the same destination won the rename; its + // tree is complete, which is what ErrExist tells the caller. + if errors.Is(err, fs.ErrExist) || errors.Is(err, syscall.ENOTEMPTY) { + return false, fmt.Errorf("destination %s already exists: %w", dst, fs.ErrExist) + } + return false, fmt.Errorf("while placing %s: %w", dst, err) + } + // Last-use touch for eviction's LRU ordering; best-effort, a miss only + // ages the entry. + now := time.Now() + _ = os.Chtimes(s.entryDir(key), now, now) + return true, nil +} + +// linkTree recreates src's directories under dst (which exists) and hard +// links its regular files. protectTree admitted nothing else into a +// published tree, so any other entry type is corruption, not a case. +func linkTree(src, dst string) error { + return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(src, path) + if err != nil { + return err + } + if rel == "." { + return nil + } + target := filepath.Join(dst, rel) + switch { + case d.IsDir(): + // protectTree published every directory as 0700; the mirror + // gets the same without a stat per directory. + return os.Mkdir(target, 0o700) + case d.Type().IsRegular(): + if err := os.Link(path, target); err != nil { + if errors.Is(err, syscall.EXDEV) { + return fmt.Errorf("destination is not on the cache's filesystem (link-out requires one filesystem): %w", err) + } + return err + } + return nil + default: + return fmt.Errorf("cached tree holds %q of type %v, want a directory or regular file", rel, d.Type()) + } + }) +} + +// protectTree checks that a tree fetcher left dataPath a directory of +// directories and regular files, with at least one file, and makes every +// file read-only before publication (see protectFile). Directories become +// the store owner's alone: only the store walks a cached tree, to link it +// out or retire it, and a consumer reaches the files through its own +// mirrored directories, which linkTreeOut gives the same mode. +func protectTree(dataPath string) error { + files := 0 + err := filepath.WalkDir(dataPath, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + switch { + case d.IsDir(): + return os.Chmod(path, 0o700) + case d.Type().IsRegular(): + files++ + info, err := d.Info() + if err != nil { + return err + } + return os.Chmod(path, readOnlyMode(info.Mode().Perm())) + default: + rel, _ := filepath.Rel(dataPath, path) + return fmt.Errorf("fetcher produced %q of type %v, want only directories and regular files", rel, d.Type()) + } + }) + if err != nil { + return fmt.Errorf("while protecting fetched tree: %w", err) + } + if files == 0 { + return errors.New("fetcher succeeded but produced an empty tree") + } + return nil +} diff --git a/cmd/atelet/internal/filecache/getdirto_test.go b/cmd/atelet/internal/filecache/getdirto_test.go new file mode 100644 index 0000000000..922b3985d2 --- /dev/null +++ b/cmd/atelet/internal/filecache/getdirto_test.go @@ -0,0 +1,301 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filecache + +import ( + "context" + "errors" + "io/fs" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" +) + +// treeFile is one regular file a treeFetcher writes. +type treeFile struct { + content string + mode fs.FileMode +} + +// releaseTree is the shape of an extracted gVisor release: an executable +// next to a directory of helpers. +var releaseTree = map[string]treeFile{ + "runsc": {"runsc bytes", 0o755}, + "gvisor-bin/containerd": {"shim bytes", 0o755}, + "gvisor-bin/LICENSE": {"license", 0o644}, + "gvisor-bin/docs/README": {"readme", 0o600}, +} + +// treeFetcher returns a DirFetcher writing files (relative name -> file) +// under the directory the store made, and a counter of how many times it +// ran. +func treeFetcher(files map[string]treeFile) (DirFetcher, *atomic.Int32) { + var calls atomic.Int32 + return func(ctx context.Context, dstDir string) error { + calls.Add(1) + for rel, f := range files { + path := filepath.Join(dstDir, rel) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + if err := os.WriteFile(path, []byte(f.content), f.mode); err != nil { + return err + } + } + return nil + }, &calls +} + +// checkTree asserts dst mirrors files: same content, every file a hard +// link to the cache copy, read-only with its execute bits kept. +func checkTree(t *testing.T, s *Store, key Key, dst string, files map[string]treeFile) { + t.Helper() + for rel, f := range files { + got, err := os.ReadFile(filepath.Join(dst, rel)) + if err != nil || string(got) != f.content { + t.Errorf("%s: content = %q, %v; want %q", rel, got, err, f.content) + continue + } + fi, err := os.Stat(filepath.Join(dst, rel)) + if err != nil { + t.Fatal(err) + } + if want := 0o444 | f.mode&0o111; fi.Mode().Perm() != want { + t.Errorf("%s: mode = %v, want %v", rel, fi.Mode().Perm(), want) + } + cfi, err := os.Stat(filepath.Join(s.dataPath(key), rel)) + if err != nil { + t.Fatal(err) + } + if !os.SameFile(fi, cfi) { + t.Errorf("%s: dst and cache copy are not one inode", rel) + } + } +} + +func TestGetDirToMissThenHit(t *testing.T) { + s := newTestStore(t) + key, _ := SHA256Key("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + fetch, calls := treeFetcher(releaseTree) + + dst1 := dstPath(t, s, "t1") + if err := s.GetDirTo(context.Background(), key, dst1, fetch); err != nil { + t.Fatalf("GetDirTo (miss): %v", err) + } + checkTree(t, s, key, dst1, releaseTree) + + dst2 := dstPath(t, s, "t2") + if err := s.GetDirTo(context.Background(), key, dst2, fetch); err != nil { + t.Fatalf("GetDirTo (hit): %v", err) + } + checkTree(t, s, key, dst2, releaseTree) + if calls.Load() != 1 { + t.Errorf("fetch ran %d times, want 1", calls.Load()) + } + + // The consumer owns its directories: it can add a file beside the + // links without the cache seeing it. + if err := os.WriteFile(filepath.Join(dst1, "gvisor-bin", "mine"), []byte("x"), 0o600); err != nil { + t.Errorf("consumer cannot write into its own mirrored dir: %v", err) + } + if _, err := os.Stat(filepath.Join(s.dataPath(key), "gvisor-bin", "mine")); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("consumer's file leaked into the cache tree: %v", err) + } + // But not write through a link: the shared bytes are protected. + if err := os.WriteFile(filepath.Join(dst1, "runsc"), []byte("mutated"), 0o755); err == nil { + t.Error("write through consumer link succeeded, want permission error") + } + + tmpChildren, err := os.ReadDir(s.tmpDir()) + if err != nil { + t.Fatal(err) + } + if len(tmpChildren) != 0 { + t.Errorf("tmp dir has %d leftover children", len(tmpChildren)) + } + // Trees are built beside their destination; nothing of that remains. + siblings, err := os.ReadDir(filepath.Dir(dst1)) + if err != nil { + t.Fatal(err) + } + for _, e := range siblings { + if e.Name() != filepath.Base(dst1) && e.Name() != filepath.Base(dst2) { + t.Errorf("link-out left %q beside the destination", e.Name()) + } + } +} + +// TestGetDirToSameDestinationLoserSeesCompleteTree pins what a caller that +// loses the race for one destination gets: ErrExist, and a tree it can use, +// because the winner's tree was renamed into place whole. +func TestGetDirToSameDestinationLoserSeesCompleteTree(t *testing.T) { + s := newTestStore(t) + key := URIKey("test://release") + fetch, _ := treeFetcher(releaseTree) + dst := dstPath(t, s, "shared") + + const n = 6 + var wg sync.WaitGroup + errs := make([]error, n) + for i := range n { + wg.Go(func() { errs[i] = s.GetDirTo(context.Background(), key, dst, fetch) }) + } + wg.Wait() + wins := 0 + for i, err := range errs { + switch { + case err == nil: + wins++ + case errors.Is(err, fs.ErrExist): + default: + t.Errorf("caller %d: %v, want nil or ErrExist", i, err) + } + } + if wins != 1 { + t.Errorf("%d callers won the destination, want exactly 1", wins) + } + checkTree(t, s, key, dst, releaseTree) +} + +func TestGetDirToConcurrentCallersShareOneFetch(t *testing.T) { + s := newTestStore(t) + key := URIKey("test://release") + fetch, calls := treeFetcher(releaseTree) + + const n = 8 + var wg sync.WaitGroup + errs := make([]error, n) + for i := range n { + wg.Go(func() { + errs[i] = s.GetDirTo(context.Background(), key, dstPath(t, s, "c"+string(rune('a'+i))), fetch) + }) + } + wg.Wait() + for i, err := range errs { + if err != nil { + t.Errorf("caller %d: %v", i, err) + } + } + if calls.Load() != 1 { + t.Errorf("fetch ran %d times, want 1", calls.Load()) + } +} + +func TestGetDirToRejectsExistingDst(t *testing.T) { + s := newTestStore(t) + key := URIKey("test://release") + fetch, _ := treeFetcher(releaseTree) + dst := dstPath(t, s, "taken") + if err := os.Mkdir(dst, 0o755); err != nil { + t.Fatal(err) + } + err := s.GetDirTo(context.Background(), key, dst, fetch) + if !errors.Is(err, fs.ErrExist) { + t.Fatalf("GetDirTo onto existing dst: %v, want ErrExist", err) + } + // The entry was still fetched and published; only the serve failed. + if _, err := os.Stat(filepath.Join(s.dataPath(key), "runsc")); err != nil { + t.Errorf("entry not published after serve failure: %v", err) + } +} + +func TestGetDirToRejectsBadArguments(t *testing.T) { + s := newTestStore(t) + fetch, calls := treeFetcher(releaseTree) + key := URIKey("test://release") + + if err := s.GetDirTo(context.Background(), Key{}, dstPath(t, s, "z"), fetch); err == nil { + t.Error("zero key accepted") + } + if err := s.GetDirTo(context.Background(), key, "relative/dst", fetch); err == nil { + t.Error("relative dst accepted") + } + if err := s.GetDirTo(context.Background(), key, dstPath(t, s, "n"), nil); err == nil { + t.Error("nil fetcher accepted") + } + if calls.Load() != 0 { + t.Errorf("fetch ran %d times on rejected calls", calls.Load()) + } +} + +func TestGetDirToEmptyTreeFails(t *testing.T) { + s := newTestStore(t) + key := URIKey("test://empty") + fetch, _ := treeFetcher(nil) + err := s.GetDirTo(context.Background(), key, dstPath(t, s, "e"), fetch) + if err == nil { + t.Fatal("empty tree published") + } + if entryExists(t, s, key) { + t.Error("entry published despite the failure") + } +} + +func TestGetDirToRejectsSymlinkInTree(t *testing.T) { + s := newTestStore(t) + key := URIKey("test://symlinked") + fetch := func(ctx context.Context, dstDir string) error { + if err := os.WriteFile(filepath.Join(dstDir, "runsc"), []byte("x"), 0o755); err != nil { + return err + } + return os.Symlink("runsc", filepath.Join(dstDir, "alias")) + } + err := s.GetDirTo(context.Background(), key, dstPath(t, s, "l"), fetch) + if err == nil { + t.Fatal("tree with a symlink published") + } + if entryExists(t, s, key) { + t.Error("entry published despite the failure") + } + tmpChildren, err := os.ReadDir(s.tmpDir()) + if err != nil { + t.Fatal(err) + } + if len(tmpChildren) != 0 { + t.Errorf("tmp dir has %d leftover children after a failed publish", len(tmpChildren)) + } +} + +// TestGetKindMismatchIsAnError pins that a key fetched as one kind is never +// silently served as the other: the entry exists, so it is not a miss that +// would trigger a refetch under the other kind. +func TestGetKindMismatchIsAnError(t *testing.T) { + s := newTestStore(t) + treeKey := URIKey("test://tree") + treeFetch, treeCalls := treeFetcher(releaseTree) + if err := s.GetDirTo(context.Background(), treeKey, dstPath(t, s, "t"), treeFetch); err != nil { + t.Fatal(err) + } + fileKey := URIKey("test://file") + fileFetch, fileCalls := countingFetcher("one file") + if err := s.GetFileTo(context.Background(), fileKey, dstPath(t, s, "f"), fileFetch); err != nil { + t.Fatal(err) + } + + if err := s.GetFileTo(context.Background(), treeKey, dstPath(t, s, "t-as-file"), fileFetch); err == nil { + t.Error("GetFileTo served a tree entry") + } + if err := s.GetFileCopyTo(context.Background(), treeKey, dstPath(t, s, "t-as-copy"), fileFetch); err == nil { + t.Error("GetFileCopyTo served a tree entry") + } + if err := s.GetDirTo(context.Background(), fileKey, dstPath(t, s, "f-as-tree"), treeFetch); err == nil { + t.Error("GetDirTo served a file entry") + } + if treeCalls.Load() != 1 || fileCalls.Load() != 1 { + t.Errorf("fetches ran tree=%d file=%d times, want 1 each (a mismatch must not refetch)", treeCalls.Load(), fileCalls.Load()) + } +} diff --git a/cmd/atelet/internal/filecache/getfilecopyto.go b/cmd/atelet/internal/filecache/getfilecopyto.go index 1f82de67c5..3cce5fd25c 100644 --- a/cmd/atelet/internal/filecache/getfilecopyto.go +++ b/cmd/atelet/internal/filecache/getfilecopyto.go @@ -35,7 +35,10 @@ import ( // copy in progress reads a held-open handle, so a concurrent eviction // cannot corrupt it. func (s *Store) GetFileCopyTo(ctx context.Context, key Key, dst string, fetch FileFetcher) error { - return s.getTo(ctx, key, dst, fetch, s.copyOut) + if fetch == nil { + return errors.New("filecache: nil FileFetcher") + } + return s.getTo(ctx, key, dst, fetchFunc(fetch), kindFile, s.copyOut) } // copyOut copies key's published data file to dst, reporting false (and no @@ -84,6 +87,13 @@ func (s *Store) openData(key Key) (*os.File, error) { if err != nil { return nil, fmt.Errorf("while opening cached %v: %w", key, err) } + if fi, err := f.Stat(); err != nil || fi.IsDir() { + _ = f.Close() + if err != nil { + return nil, fmt.Errorf("while checking cached %v: %w", key, err) + } + return nil, fmt.Errorf("entry for %v is a directory tree, not a file (use GetDirTo)", key) + } // Last-use touch for eviction's LRU ordering; best-effort, a miss only // ages the entry. now := time.Now() diff --git a/cmd/atelet/internal/filecache/getfileto.go b/cmd/atelet/internal/filecache/getfileto.go index 077b4c224b..f41bb2777b 100644 --- a/cmd/atelet/internal/filecache/getfileto.go +++ b/cmd/atelet/internal/filecache/getfileto.go @@ -33,9 +33,24 @@ import ( // wrapped with %w, so errors.Is classification sees through the store. type FileFetcher func(ctx context.Context, dstPath string) error +// fetchFunc is the shape shared by FileFetcher and DirFetcher once the +// store has decided what it is fetching: fill in dataPath. +type fetchFunc func(ctx context.Context, dataPath string) error + +// entryKind is what an entry's data is: one regular file, or a directory +// tree. The kind is fixed by the method that fetched the entry; serving it +// through a method of the other kind is an error, not a miss. +type entryKind int + +const ( + kindFile entryKind = iota + kindTree +) + // serveHit materializes a published cache entry at the caller's destination: -// (*Store).linkOut serves a hard link, (*Store).copyOut a private copy. It -// reports false, with no error, on a cache miss. +// (*Store).linkOut serves a hard link, (*Store).copyOut a private copy, +// (*Store).linkTreeOut a directory of hard links. It reports false, with no +// error, on a cache miss. type serveHit func(key Key, dst string) (bool, error) // linkRetries bounds the fetch-then-serve loop. An entry can be evicted @@ -49,30 +64,31 @@ const linkRetries = 3 // exist yet and lives on the cache's filesystem (the same mount, not just // the same disk): on success it is a hard link to the read-only cache // copy, so the caller keeps a valid file regardless of later eviction, and -// the entry is published mode 0444 so an in-place write fails loudly rather -// than corrupting the shared bytes. +// the entry is published read-only (mode 0444, keeping any execute bits the +// fetcher set) so an in-place write fails loudly rather than corrupting the +// shared bytes. // // Concurrent calls for one key share a single fetch. A caller whose ctx is // canceled returns early with ctx.Err() while the fetch keeps running for // the others; there is no negative caching, so after a failed fetch the // next call starts fresh. func (s *Store) GetFileTo(ctx context.Context, key Key, dst string, fetch FileFetcher) error { - return s.getTo(ctx, key, dst, fetch, s.linkOut) + if fetch == nil { + return errors.New("filecache: nil FileFetcher") + } + return s.getTo(ctx, key, dst, fetchFunc(fetch), kindFile, s.linkOut) } -// getTo is the read-through loop shared by GetFileTo and GetFileCopyTo: -// serve a hit, else run the singleflight fetch and retry. serve is the only -// difference between the two public methods. -func (s *Store) getTo(ctx context.Context, key Key, dst string, fetch FileFetcher, serve serveHit) error { +// getTo is the read-through loop shared by the public Get methods: serve a +// hit, else run the singleflight fetch and retry. kind decides how a fetch +// is published and serve how a hit is materialized; nothing else differs. +func (s *Store) getTo(ctx context.Context, key Key, dst string, fetch fetchFunc, kind entryKind, serve serveHit) error { if key.isZero() { return errors.New("filecache: zero Key (use a Key constructor)") } if !filepath.IsAbs(dst) { return fmt.Errorf("filecache: destination %q is not an absolute path", dst) } - if fetch == nil { - return errors.New("filecache: nil FileFetcher") - } for attempt := 0; ; attempt++ { served, err := serve(key, dst) if err != nil { @@ -88,7 +104,7 @@ func (s *Store) getTo(ctx context.Context, key Key, dst string, fetch FileFetche if attempt >= linkRetries { return fmt.Errorf("entry for %v vanished after %d fetches; is something else deleting under the store root?", key, attempt) } - if err := s.fetchFlight(ctx, key, fetch); err != nil { + if err := s.fetchFlight(ctx, key, fetch, kind); err != nil { return err } } @@ -102,12 +118,16 @@ func (s *Store) linkOut(key Key, dst string) (bool, error) { s.hitMu.RLock() defer s.hitMu.RUnlock() - if _, err := os.Stat(s.dataPath(key)); err != nil { + fi, err := os.Stat(s.dataPath(key)) + if err != nil { if errors.Is(err, fs.ErrNotExist) { return false, nil } return false, fmt.Errorf("while checking cache for %v: %w", key, err) } + if fi.IsDir() { + return false, fmt.Errorf("entry for %v is a directory tree, not a file (use GetDirTo)", key) + } if err := os.Link(s.dataPath(key), dst); err != nil { if errors.Is(err, fs.ErrExist) { return false, fmt.Errorf("destination %s already exists: %w", dst, err) @@ -128,9 +148,9 @@ func (s *Store) linkOut(key Key, dst string) (bool, error) { // it or for ctx. The flight itself runs on a context detached from the // callers' — bounded only by the store's fetch timeout — so one canceled // caller never aborts a download other callers are waiting on. -func (s *Store) fetchFlight(ctx context.Context, key Key, fetch FileFetcher) error { +func (s *Store) fetchFlight(ctx context.Context, key Key, fetch fetchFunc, kind entryKind) error { ch := s.sf.DoChan(key.dir, func() (any, error) { - return nil, s.fetchAndPublish(context.WithoutCancel(ctx), key, fetch) + return nil, s.fetchAndPublish(context.WithoutCancel(ctx), key, fetch, kind) }) select { case res := <-ch: @@ -147,7 +167,7 @@ func (s *Store) fetchFlight(ctx context.Context, key Key, fetch FileFetcher) err // result under entries/. On any failure the temp dir is removed, so a bad // fetch is never visible in the cache; a crash instead leaves it for // SweepDebris. -func (s *Store) fetchAndPublish(ctx context.Context, key Key, fetch FileFetcher) error { +func (s *Store) fetchAndPublish(ctx context.Context, key Key, fetch fetchFunc, kind entryKind) error { ctx, cancel := context.WithTimeout(ctx, s.fetchTimeout) defer cancel() @@ -169,22 +189,24 @@ func (s *Store) fetchAndPublish(ctx context.Context, key Key, fetch FileFetcher) }() dataPath := filepath.Join(tmpDir, dataName) + if kind == kindTree { + // A tree fetcher fills a directory the store made; a file fetcher + // creates its file itself (see FileFetcher). + if err := os.Mkdir(dataPath, 0o700); err != nil { + return fmt.Errorf("while creating fetch tree dir: %w", err) + } + } if err := fetch(ctx, dataPath); err != nil { return err } - fi, err := os.Stat(dataPath) - if err != nil { - return fmt.Errorf("fetcher succeeded but produced no file: %w", err) - } - if !fi.Mode().IsRegular() { - return fmt.Errorf("fetcher produced %v, want a regular file", fi.Mode()) + switch kind { + case kindFile: + err = protectFile(dataPath) + case kindTree: + err = protectTree(dataPath) } - - // Read-only before publication: cached bytes are shared through hard - // links, so an in-place write by any consumer must fail rather than - // poison the copy every later caller links. - if err := os.Chmod(dataPath, 0o444); err != nil { - return fmt.Errorf("while making fetched file read-only: %w", err) + if err != nil { + return err } if err := writeEntryMeta(tmpDir, key, time.Now()); err != nil { return err @@ -204,3 +226,28 @@ func (s *Store) fetchAndPublish(ctx context.Context, key Key, fetch FileFetcher) published = true return nil } + +// readOnlyMode is the mode a fetched regular file is published with: +// read-only for everyone, keeping whatever execute bits the fetcher set so +// a cached binary stays runnable. +func readOnlyMode(perm fs.FileMode) fs.FileMode { + return 0o444 | perm&0o111 +} + +// protectFile checks that a file fetcher produced one regular file at +// dataPath and makes it read-only before publication: cached bytes are +// shared through hard links, so an in-place write by any consumer must fail +// rather than poison the copy every later caller links. +func protectFile(dataPath string) error { + fi, err := os.Lstat(dataPath) + if err != nil { + return fmt.Errorf("fetcher succeeded but produced no file: %w", err) + } + if !fi.Mode().IsRegular() { + return fmt.Errorf("fetcher produced %v, want a regular file", fi.Mode()) + } + if err := os.Chmod(dataPath, readOnlyMode(fi.Mode().Perm())); err != nil { + return fmt.Errorf("while making fetched file read-only: %w", err) + } + return nil +} diff --git a/cmd/atelet/internal/filecache/getfileto_test.go b/cmd/atelet/internal/filecache/getfileto_test.go index 168641ed84..8e144ea1ee 100644 --- a/cmd/atelet/internal/filecache/getfileto_test.go +++ b/cmd/atelet/internal/filecache/getfileto_test.go @@ -382,3 +382,34 @@ func TestGetFileToPublishedCopyIsWriteProtected(t *testing.T) { t.Error("write through consumer link succeeded, want permission error") } } + +func TestGetFileToKeepsExecuteBits(t *testing.T) { + s := newTestStore(t) + for _, tc := range []struct { + name string + fetched os.FileMode + published os.FileMode + }{ + {"binary", 0o755, 0o555}, + {"data", 0o644, 0o444}, + {"private binary", 0o700, 0o544}, + } { + t.Run(tc.name, func(t *testing.T) { + key := URIKey("test://mode/" + tc.name) + fetch := func(ctx context.Context, dstPath string) error { + return os.WriteFile(dstPath, []byte("bits"), tc.fetched) + } + dst := dstPath(t, s, "mode-"+tc.name) + if err := s.GetFileTo(context.Background(), key, dst, fetch); err != nil { + t.Fatal(err) + } + fi, err := os.Stat(dst) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != tc.published { + t.Errorf("published mode = %v, want %v", fi.Mode().Perm(), tc.published) + } + }) + } +} diff --git a/cmd/atelet/internal/snapshottracker/tracker_test.go b/cmd/atelet/internal/snapshottracker/tracker_test.go index 997bc0e04e..637617b5ec 100644 --- a/cmd/atelet/internal/snapshottracker/tracker_test.go +++ b/cmd/atelet/internal/snapshottracker/tracker_test.go @@ -38,11 +38,10 @@ const trackedActorUID = "actor-uid-1" func useTempNodeDirs(t *testing.T) { t.Helper() root := t.TempDir() - origActors, origStatic := nodepath.ActorsDir, nodepath.StaticFilesDir + origActors := nodepath.ActorsDir nodepath.ActorsDir = filepath.Join(root, "actors") - nodepath.StaticFilesDir = filepath.Join(root, "static-files") t.Cleanup(func() { - nodepath.ActorsDir, nodepath.StaticFilesDir = origActors, origStatic + nodepath.ActorsDir = origActors }) } diff --git a/cmd/atelet/lifecycle_test.go b/cmd/atelet/lifecycle_test.go index cdf83b1087..48c5e5d332 100644 --- a/cmd/atelet/lifecycle_test.go +++ b/cmd/atelet/lifecycle_test.go @@ -25,7 +25,10 @@ import ( "runtime" "slices" "strings" + "sync" + "sync/atomic" "testing" + "time" "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" "go.ate.dev/substrate/cmd/atelet/internal/snapshottracker" @@ -47,11 +50,10 @@ import ( func useTempNodeDirs(t *testing.T) { t.Helper() root := t.TempDir() - origActors, origStatic := nodepath.ActorsDir, nodepath.StaticFilesDir + origActors := nodepath.ActorsDir nodepath.ActorsDir = filepath.Join(root, "actors") - nodepath.StaticFilesDir = filepath.Join(root, "static-files") t.Cleanup(func() { - nodepath.ActorsDir, nodepath.StaticFilesDir = origActors, origStatic + nodepath.ActorsDir = origActors }) } @@ -61,6 +63,8 @@ func useTempNodeDirs(t *testing.T) { // the request, never derived from the actor UID. type fakeWorker struct { ateworkerpb.UnimplementedWorkerServer + // mu guards the record maps: concurrent RPCs record at the same time. + mu sync.Mutex // snapshotFiles are written at checkpoint and reported back to atelet as // the exact set the snapshot consists of. snapshotFiles map[string]string @@ -78,6 +82,8 @@ type fakeWorker struct { } func (f *fakeWorker) recordFidelity(rpc string, fidelity ateworkerpb.SnapshotFidelity) { + f.mu.Lock() + defer f.mu.Unlock() if f.fidelities == nil { f.fidelities = map[string]ateworkerpb.SnapshotFidelity{} } @@ -85,6 +91,8 @@ func (f *fakeWorker) recordFidelity(rpc string, fidelity ateworkerpb.SnapshotFid } func (f *fakeWorker) recordActorDirs(rpc string, actorDirs *ateworkerpb.ActorDirs) { + f.mu.Lock() + defer f.mu.Unlock() if f.actorDirs == nil { f.actorDirs = map[string]*ateworkerpb.ActorDirs{} } @@ -209,6 +217,7 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity) { s := &Atelet{ workerDialer: newWorkerDialer(1), imageCache: newImageVolumeStore(t), + sandboxAssetCache: newTestAssetCache(t), anonGCSClient: fakeObjectStorage{data: runsc}, systemInfoVolumes: newSystemInfoVolumeRefresher(nil, nil), snapshots: snapshottracker.New(t.Context(), clock.RealClock{}), @@ -261,6 +270,11 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity) { if _, err := os.Stat(snapshotFile); err != nil { t.Fatalf("pause did not write the local snapshot: %v", err) } + // A paused actor stays resumable on this node, so it keeps its staged + // assets, and with them its binaries in the cache. + if _, err := os.Stat(filepath.Join(ateletpath.SandboxAssetsDir(actorUID), runscAssetName)); err != nil { + t.Errorf("paused actor lost its staged runsc: %v", err) + } st := resp.GetState() if got := st.GetLatestLocal(); got.GetSnapshot().GetSnapshotUuid() != snapshotName || got.GetLocalCommitTime() == nil { t.Errorf("latest_local = %v, want the pause snapshot with its commit time", got) @@ -290,6 +304,9 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity) { if got := worker.restored["checkpoint.img"]; got != "guest-memory" { t.Fatalf("restore staged %q for worker, want the pause snapshot's %q", got, "guest-memory") } + if _, err := os.Stat(filepath.Join(ateletpath.SandboxAssetsDir(actorUID), runscAssetName)); err != nil { + t.Errorf("resumed actor has no staged runsc: %v", err) + } for _, rpc := range []string{"CheckpointWorkload", "RestoreWorkload"} { if got := worker.fidelities[rpc]; got != toWorkerFidelity(fidelity) { t.Errorf("%s carried fidelity %v, want %v", rpc, got, toWorkerFidelity(fidelity)) @@ -401,6 +418,7 @@ func TestRestoreUsesRequestSandboxAssets(t *testing.T) { s := &Atelet{ workerDialer: newWorkerDialer(1), imageCache: newImageVolumeStore(t), + sandboxAssetCache: newTestAssetCache(t), anonGCSClient: fakeObjectStorage{data: runsc}, systemInfoVolumes: newSystemInfoVolumeRefresher(nil, nil), snapshots: snapshottracker.New(t.Context(), clock.RealClock{}), @@ -556,11 +574,21 @@ func TestActivationFailureBeforeRegistration(t *testing.T) { t.Fatal(err) } } + // The live activation's staged binary: a Run that fails before + // it swaps its own set in must leave it, and a Run that fails + // after must still leave nothing half-staged behind. + liveAsset := filepath.Join(ateletpath.SandboxAssetsDir(actorUID), "runsc") + if err := os.MkdirAll(filepath.Dir(liveAsset), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(liveAsset, []byte("live runsc"), 0o555); err != nil { + t.Fatal(err) + } storage := fakeObjectStorage{data: content} if tc.failure == "asset" { storage.err = fmt.Errorf("asset unavailable") } - s := &Atelet{anonGCSClient: storage, systemInfoVolumes: refresher, snapshots: snapshottracker.New(t.Context(), clock.RealClock{})} + s := &Atelet{anonGCSClient: storage, systemInfoVolumes: refresher, snapshots: snapshottracker.New(t.Context(), clock.RealClock{}), sandboxAssetCache: newTestAssetCache(t)} assets := &ateletpb.SandboxAssets{ SandboxClass: "gvisor", PauseImage: testPauseImage, Assets: map[string]*ateletpb.ArchAssets{runtime.GOARCH: {Files: map[string]*ateletpb.AssetFile{ @@ -586,6 +614,9 @@ func TestActivationFailureBeforeRegistration(t *testing.T) { if err == nil || !strings.Contains(err.Error(), tc.wantErr) { t.Fatalf("activation error = %v, want %q", err, tc.wantErr) } + if _, err := os.Lstat(ateletpath.SandboxAssetsStagingDir(actorUID)); !os.IsNotExist(err) { + t.Errorf("failed activation left the staging dir behind (lstat err = %v)", err) + } // Inspect disk and registry before changing the bundle hash: a later // rotation can hide a stale registration by recreating deleted files. @@ -596,7 +627,23 @@ func TestActivationFailureBeforeRegistration(t *testing.T) { if refresher.actors[actorUID] != registered || registered.stale { t.Error("pre-reset failure invalidated the existing registration") } + if got, err := os.ReadFile(liveAsset); err != nil || string(got) != "live runsc" { + t.Errorf("pre-reset failure lost the live activation's staged asset: %q, %v", got, err) + } } else { + // Past the reset, the record still describes the previous + // activation (or nothing), so no links of this one may + // remain for a later Terminate to take for that record's: + // at most the previous activation's own, untouched. + entries, err := os.ReadDir(ateletpath.SandboxAssetsDir(actorUID)) + if err != nil && !os.IsNotExist(err) { + t.Fatal(err) + } + for _, e := range entries { + if got, err := os.ReadFile(liveAsset); e.Name() != "runsc" || err != nil || string(got) != "live runsc" { + t.Errorf("failed activation left %q behind in the assets dir", e.Name()) + } + } if _, err := os.Stat(projectedPath); !os.IsNotExist(err) { t.Fatalf("reset did not delete prior projection: stat error = %v", err) } @@ -632,6 +679,7 @@ func TestRunFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { assetHash := fmt.Sprintf("%x", sha256.Sum256(content)) const actorUID = "0f6c2b0e-1b7e-4b3f-9a6a-1d2c3e4f5a62" s := &Atelet{ + sandboxAssetCache: newTestAssetCache(t), anonGCSClient: fakeObjectStorage{data: content}, imageCache: newImageVolumeStore(t), systemInfoVolumes: refresher, @@ -700,6 +748,7 @@ func TestRestoreFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { SystemInfo: trustVolumeSpec("ca.pem"), }}} s := &Atelet{ + sandboxAssetCache: newTestAssetCache(t), anonGCSClient: fakeObjectStorage{data: content}, imageCache: newImageVolumeStore(t), systemInfoVolumes: refresher, @@ -997,6 +1046,7 @@ func TestTerminateWithoutWorkerPodUID(t *testing.T) { s := &Atelet{ workerDialer: newWorkerDialer(1), + sandboxAssetCache: newTestAssetCache(t), systemInfoVolumes: newSystemInfoVolumeRefresher(nil, nil), snapshots: snapshottracker.New(t.Context(), clock.RealClock{}), } @@ -1061,6 +1111,7 @@ func runPluginActor(t *testing.T) *pluginActor { a.s = newPluginHerder(t, a.store) a.s.workerDialer = newWorkerDialer(1) a.s.imageCache = newImageVolumeStore(t) + a.s.sandboxAssetCache = newTestAssetCache(t) a.s.anonGCSClient = fakeObjectStorage{data: runsc} a.s.systemInfoVolumes = newSystemInfoVolumeRefresher(nil, nil) a.assets = &ateletpb.SandboxAssets{ @@ -1233,3 +1284,136 @@ func TestPromoteSnapshotRecordsUpload(t *testing.T) { t.Errorf("state after a retried promote = %v, want %v", got, st) } } + +// TestConcurrentRunsShareOneAssetDownload runs two actors that need the same +// asset at the same time: the asset cache downloads it once, and each actor +// gets its own hard link to the one cached copy. +func TestConcurrentRunsShareOneAssetDownload(t *testing.T) { + useTempNodeDirs(t) + ctx := t.Context() + + worker := &fakeWorker{} + serveFakeWorker(t, worker) + + host := imageVolumeTestRegistry(t) + pinnedImage := pushTestImage(t, host+"/actor:v1", singleFileLayer(t, "bin/app", "app")) + + runsc := []byte("runsc binary") + var gets atomic.Int32 + s := &Atelet{ + workerDialer: newWorkerDialer(2), + imageCache: newImageVolumeStore(t), + sandboxAssetCache: newTestAssetCache(t), + anonGCSClient: fakeObjectStorage{data: runsc, gets: &gets, delay: 200 * time.Millisecond}, + systemInfoVolumes: newSystemInfoVolumeRefresher(nil, nil), + } + sandboxAssets := &ateletpb.SandboxAssets{ + SandboxClass: "gvisor", + PauseImage: pinnedImage, + Assets: map[string]*ateletpb.ArchAssets{ + runtime.GOARCH: {Files: map[string]*ateletpb.AssetFile{ + runscAssetName: {Url: "gs://test-bucket/runsc", Sha256: fmt.Sprintf("%x", sha256.Sum256(runsc))}, + }}, + }, + } + spec := &ateletpb.WorkloadSpec{ + Containers: []*ateletpb.Container{{Name: "app", Image: pinnedImage, Command: []string{"/bin/app"}}}, + } + + actorUIDs := []string{"01234567-89ab-cdef-0123-456789abcde1", "01234567-89ab-cdef-0123-456789abcde2"} + var wg sync.WaitGroup + errs := make([]error, len(actorUIDs)) + for i, uid := range actorUIDs { + wg.Go(func() { + _, errs[i] = s.Run(ctx, &ateletpb.RunRequest{ + Atespace: "ate-demo", + ActorName: "counter-" + uid[len(uid)-1:], + ActorUid: uid, + ActorTemplateAtespace: "default", + ActorTemplateName: "counter", + WorkerPodUid: "5f1c0d2e-3a4b-4c5d-8e9f-0a1b2c3d4e5f", + SandboxAssets: sandboxAssets, + Spec: spec, + }) + }) + } + wg.Wait() + for i, err := range errs { + if err != nil { + t.Fatalf("Run %d: %v", i, err) + } + } + if gets.Load() != 1 { + t.Errorf("asset downloaded %d times for two concurrent Runs, want 1", gets.Load()) + } + + var infos []os.FileInfo + for _, uid := range actorUIDs { + staged := filepath.Join(ateletpath.SandboxAssetsDir(uid), runscAssetName) + got, err := os.ReadFile(staged) + if err != nil || string(got) != string(runsc) { + t.Fatalf("actor %s staged runsc = %q, %v; want the asset bytes", uid, got, err) + } + fi, err := os.Stat(staged) + if err != nil { + t.Fatal(err) + } + infos = append(infos, fi) + } + if !os.SameFile(infos[0], infos[1]) { + t.Error("the two actors' runsc links are not one inode") + } + for _, uid := range actorUIDs { + if _, err := os.Lstat(ateletpath.SandboxAssetsStagingDir(uid)); !os.IsNotExist(err) { + t.Errorf("actor %s: staging dir left behind after Run (lstat err = %v)", uid, err) + } + } +} + +// TestCommitSandboxAssetsChecksPathsBeforeMoving pins the order in +// commitSandboxAssets: a path outside the staging dir fails the commit +// before anything moves, so the staged set stays for Run's cleanup and the +// assets dir, which the sandbox record still describes, is untouched. +func TestCommitSandboxAssetsChecksPathsBeforeMoving(t *testing.T) { + useTempNodeDirs(t) + const actorUID = "01234567-89ab-cdef-0123-456789abcdef" + staging, final := ateletpath.SandboxAssetsStagingDir(actorUID), ateletpath.SandboxAssetsDir(actorUID) + for _, dir := range []string{staging, final} { + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(staging, "runsc"), []byte("new"), 0o555); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(final, "runsc"), []byte("previous"), 0o555); err != nil { + t.Fatal(err) + } + paths := map[string]string{runscAssetName: filepath.Join(staging, "runsc"), "stray": "/elsewhere/kernel"} + if err := commitSandboxAssets(actorUID, staging, paths); err == nil { + t.Fatal("commitSandboxAssets accepted a path outside the staging dir") + } + if got, err := os.ReadFile(filepath.Join(final, "runsc")); err != nil || string(got) != "previous" { + t.Errorf("assets dir changed by a refused commit: %q, %v", got, err) + } + if _, err := os.Stat(filepath.Join(staging, "runsc")); err != nil { + t.Errorf("staged set gone after a refused commit: %v", err) + } + if paths[runscAssetName] != filepath.Join(staging, "runsc") { + t.Errorf("paths rewritten by a refused commit: %v", paths) + } + + delete(paths, "stray") + if err := commitSandboxAssets(actorUID, staging, paths); err != nil { + t.Fatalf("commitSandboxAssets: %v", err) + } + if got, err := os.ReadFile(filepath.Join(final, "runsc")); err != nil || string(got) != "new" { + t.Errorf("assets dir after commit = %q, %v; want the staged set", got, err) + } + if paths[runscAssetName] != filepath.Join(final, "runsc") { + t.Errorf("paths after commit = %v, want them under %s", paths, final) + } + if _, err := os.Lstat(staging); !os.IsNotExist(err) { + t.Errorf("staging dir remains after commit (lstat err = %v)", err) + } +} diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 8e5ad30235..09b4d7c5ba 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -35,6 +35,7 @@ import ( "go.ate.dev/substrate/cmd/atelet/internal/apivalidation" "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" "go.ate.dev/substrate/cmd/atelet/internal/credentialprovider" + "go.ate.dev/substrate/cmd/atelet/internal/filecache" "go.ate.dev/substrate/cmd/atelet/internal/snapshottracker" "go.ate.dev/substrate/cmd/atelet/internal/sparsefile" "go.ate.dev/substrate/cmd/atelet/internal/trustbundle" @@ -237,6 +238,14 @@ func main() { go newImageCacheGC(imageCache, *imageCacheDir).Run(ctx) } + if err := validateSandboxAssetCacheFlags(); err != nil { + serverboot.Fatal(ctx, "Invalid sandbox asset cache flags", err) + } + sandboxAssetCache, err := openSandboxAssetCache(ctx, *sandboxAssetCacheDir, *sandboxAssetCacheMinAge, *sandboxAssetFetchTimeout) + if err != nil { + serverboot.Fatal(ctx, "Failed to open sandbox asset cache", err) + } + // Sandbox assets go through the object-store plugin; a gs:// asset it // cannot serve, such as one in the public gs://gvisor bucket, is read // anonymously. @@ -317,6 +326,7 @@ func main() { objectstorev1.NewAssetProviderClient(objectStorePluginConn), *objectStorePluginSocket, imageCache, + sandboxAssetCache, instruments, volPlugins, csiDriverConfigGetter, @@ -527,6 +537,9 @@ type Atelet struct { workerDialer *WorkerDialer imageCache *imagecache.Store + // sandboxAssetCache holds every sandbox asset on the node once; actors get hard + // links into their own directories (see ensureSandboxAssets). + sandboxAssetCache *filecache.Store // anonGCSClient reads gs:// assets the plugin cannot serve, without credentials. anonGCSClient objectstorage.ObjectStorage // objectStorePlugin moves external snapshot files between the node and storage. @@ -560,6 +573,7 @@ func NewService( assetPlugin objectstorev1.AssetProviderClient, pluginSocket string, imageCache *imagecache.Store, + sandboxAssetCache *filecache.Store, instruments *Instruments, volumePlugins map[string]volume.VolumePluginWorkerPlane, csiDriverConfigGetter csi.CSIDriverConfigGetter, @@ -569,6 +583,7 @@ func NewService( wms := &Atelet{ workerDialer: workerDialer, imageCache: imageCache, + sandboxAssetCache: sandboxAssetCache, anonGCSClient: anonGCSClient, objectStorePlugin: objectStorePlugin, assetPlugin: assetPlugin, @@ -595,10 +610,30 @@ func (s *Atelet) Run(ctx context.Context, req *ateletpb.RunRequest) (resp *atele if err != nil { return nil, apierror.InvalidArgument("%v", err) } - assetPaths, err := s.ensureSandboxAssets(ctx, sandboxRec) + // Fetched before the reset below, into a staging dir, so a failed fetch + // leaves a live activation as it was. The staged set is swapped in only + // after the reset, together with the record that describes it, so the + // two never disagree: a failure in between removes the staged set, and + // the record keeps describing whatever the reset left. + staging := ateletpath.SandboxAssetsStagingDir(actorUID) + if err := os.RemoveAll(staging); err != nil { + return nil, wrapFileSystemErr("while removing a stale sandbox assets staging dir", err) + } + assetPaths, err := s.ensureSandboxAssets(ctx, sandboxRec, staging) if err != nil { + _ = os.RemoveAll(staging) return nil, err } + committed := false + defer func() { + if err == nil { + return + } + _ = os.RemoveAll(staging) + if committed { + _ = os.RemoveAll(ateletpath.SandboxAssetsDir(actorUID)) + } + }() s.systemInfoVolumes.Deregister(actorUID) if err := resetActorDirs(actorUID); err != nil { @@ -612,6 +647,10 @@ func (s *Atelet) Run(ctx context.Context, req *ateletpb.RunRequest) (resp *atele // Record the sandbox binaries this actor is running so a later Checkpoint // (whose request no longer carries the sandbox config) can re-fetch the same // version and pin it into the snapshot manifest. + if err := commitSandboxAssets(actorUID, staging, assetPaths); err != nil { + return nil, err + } + committed = true if err := writeSandboxRecord(actorUID, sandboxRec); err != nil { return nil, fmt.Errorf("while recording sandbox assets: %w", err) } @@ -754,7 +793,7 @@ func (s *Atelet) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest op.sandboxClass = sandboxRec.SandboxClass tAssets := time.Now() - assetPaths, err := s.ensureSandboxAssets(ctx, sandboxRec) + assetPaths, err := s.ensureSandboxAssets(ctx, sandboxRec, ateletpath.SandboxAssetsDir(actorUID)) dAssets = time.Since(tAssets) if err != nil { return nil, err @@ -1223,6 +1262,20 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res if err := resetActorDirs(actorUID); err != nil { return nil, fmt.Errorf("while resetting actor dirs: %w", err) } + // A restore may bring another runtime than the one staged here before, + // so the assets dir is emptied for it; Run swaps a staged set in + // instead (see commitSandboxAssets). + if err := resetSandboxAssetsDir(actorUID); err != nil { + return nil, err + } + // The record below is written last, so until then it describes the + // previous activation; a failed restore must not leave this one's + // asset links behind for a later Terminate to take for that record's. + defer func() { + if err != nil { + _ = os.RemoveAll(ateletpath.SandboxAssetsDir(actorUID)) + } + }() tMount := time.Now() mountErr := s.mountExternalVolumes(ctx, actorUID, req.GetSpec().GetVolumes()) @@ -1322,7 +1375,7 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res g.Go(func() (err error) { defer func() { prepErr = err }() tAssets := time.Now() - assetPaths, err = s.ensureSandboxAssets(gctx, runtimeRec) + assetPaths, err = s.ensureSandboxAssets(gctx, runtimeRec, ateletpath.SandboxAssetsDir(actorUID)) dAssets = time.Since(tAssets) if err != nil { prepFailedPhase = ateattr.SnapshotPhaseSandboxAssets @@ -1418,7 +1471,7 @@ func (s *Atelet) Terminate(ctx context.Context, req *ateletpb.TerminateRequest) if err != nil { return nil, fmt.Errorf("failed to read sandbox record during terminate (actor: %s, actorUID: %s): %w", actorRef, actorUID, err) } - paths, err := s.ensureSandboxAssets(ctx, sandboxRec) + paths, err := s.ensureSandboxAssets(ctx, sandboxRec, ateletpath.SandboxAssetsDir(actorUID)) if err != nil { return nil, fmt.Errorf("failed to ensure sandbox assets during terminate (actor: %s, actorUID: %s): %w", actorRef, actorUID, err) } @@ -1885,6 +1938,45 @@ func resetActorDirs(actorUID string) error { return nil } +// resetSandboxAssetsDir empties the actor's sandbox assets dir for Restore +// to stage into. Not part of resetActorDirs: the links an activation holds +// stay while the actor remains resumable on this node, through Checkpoint +// and its local snapshot, and keep its binaries in the cache meanwhile; +// Run replaces them through its staging swap. The entries are hard links +// into the asset cache, so removing them never touches the cached bytes. +// Owner-only, like the bundle dir: the worker runs the binaries from here +// as root. +func resetSandboxAssetsDir(actorUID string) error { + return resetDir("sandbox assets dir", ateletpath.SandboxAssetsDir(actorUID), os.RemoveAll, 0o700) +} + +// commitSandboxAssets replaces the actor's sandbox assets dir with the set +// Run staged at staging, and rewrites paths, which point under staging, to +// their final location. Nothing moves until every path has been checked, +// so a failure leaves the staged set where it was for Run's cleanup and +// the assets dir as it was, still matching the sandbox record. +func commitSandboxAssets(actorUID, staging string, paths map[string]string) error { + final := ateletpath.SandboxAssetsDir(actorUID) + moved := make(map[string]string, len(paths)) + for name, p := range paths { + rel, err := filepath.Rel(staging, p) + if err != nil || !filepath.IsLocal(rel) { + return fmt.Errorf("staged asset %q at %q is not under %q", name, p, staging) + } + moved[name] = filepath.Join(final, rel) + } + if err := os.RemoveAll(final); err != nil { + return wrapFileSystemErr("while removing previous sandbox assets dir", err) + } + if err := os.Rename(staging, final); err != nil { + return wrapFileSystemErr("while placing sandbox assets dir", err) + } + for name, p := range moved { + paths[name] = p + } + return nil +} + // resetDir empties dir with remove and recreates it with mode. what names the // directory in errors. func resetDir(what, dir string, remove func(string) error, mode os.FileMode) error { diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index 1bc2e6941e..8c1eaa1f08 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -29,6 +29,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "syscall" "testing" "time" @@ -40,7 +41,6 @@ import ( "go.ate.dev/substrate/internal/ateattr" "go.ate.dev/substrate/internal/atelet" "go.ate.dev/substrate/internal/imagecache" - "go.ate.dev/substrate/internal/nodepath" "go.ate.dev/substrate/internal/ocispec" "go.ate.dev/substrate/internal/proto/ateletpb" "go.ate.dev/substrate/internal/proto/ateworkerpb" @@ -508,44 +508,45 @@ func TestToWorkerFidelity(t *testing.T) { } } -// TestFetchAssetRejectsBadHash confirms fetchAsset validates the asset hash -// before the cache-hit os.Stat/early-return, not merely "at some point". To -// prove the ordering, it plants a real file at the exact path an invalid hash -// resolves to: a correctly-ordered fetchAsset validates first and returns an -// error, while a regression that stats first would find this file and return it -// with a nil error, failing the test. StaticFilesDir is redirected to a temp -// dir so the planted path is writable and isolated. -func TestFetchAssetRejectsBadHash(t *testing.T) { - orig := nodepath.StaticFilesDir - nodepath.StaticFilesDir = t.TempDir() - t.Cleanup(func() { nodepath.StaticFilesDir = orig }) - - // Invalid (8 chars, not 64) but separator-free, so it resolves to a normal - // filename inside the temp StaticFilesDir. - const badHash = "deadbeef" - if err := os.WriteFile(ateletpath.AssetPath(badHash), []byte("planted"), 0o755); err != nil { - t.Fatalf("planting cache file: %v", err) +// TestStageAssetRejectsBadHash confirms stageAsset validates the asset hash +// before it looks for an already-staged file, not merely "at some point". To +// prove the ordering, it plants a file at the destination: a correctly-ordered +// stageAsset validates first and returns an error, while a regression that +// checked the destination first would find this file and return it with a +// nil error, failing the test. +func TestStageAssetRejectsBadHash(t *testing.T) { + dst := filepath.Join(t.TempDir(), "asset") + if err := os.WriteFile(dst, []byte("planted"), 0o755); err != nil { + t.Fatalf("planting staged file: %v", err) } - s := &Atelet{} - _, err := s.fetchAsset(context.Background(), assetEntry{SHA256: badHash}) + s := &Atelet{sandboxAssetCache: newTestAssetCache(t)} + _, err := s.stageAsset(context.Background(), assetEntry{SHA256: "deadbeef"}, dst) if err == nil { - t.Fatal("fetchAsset returned a cache hit for an invalid hash; validation must run before the os.Stat early return") + t.Fatal("stageAsset returned the planted file for an invalid hash; validation must run before the staged check") } // The error must come from the validation step, proving it ran before the - // cache-hit stat could return the planted file. + // staged check could return the planted file. if !strings.Contains(err.Error(), "while validating asset hash") { t.Errorf("error did not come from hash validation: %v", err) } } -// fakeObjectStorage serves fixed bytes for GetObject so fetchAsset can be tested. +// fakeObjectStorage serves fixed bytes for GetObject so stageAsset can be tested. type fakeObjectStorage struct { data []byte err error + // gets, if set, counts GetObject calls. + gets *atomic.Int32 + // delay, if set, holds each GetObject so concurrent callers overlap. + delay time.Duration } func (f fakeObjectStorage) GetObject(_ context.Context, _, _ string) (io.ReadCloser, error) { + if f.gets != nil { + f.gets.Add(1) + } + time.Sleep(f.delay) if f.err != nil { return nil, f.err } @@ -554,94 +555,125 @@ func (f fakeObjectStorage) GetObject(_ context.Context, _, _ string) (io.ReadClo func (fakeObjectStorage) PutObject(_ context.Context, _, _ string, _ io.Reader) error { return nil } -// TestFetchAssetStreaming covers the streamed download: good asset cached, -// over-cap rejected, hash mismatch rejected (failures leave no cache file). -func TestFetchAssetStreaming(t *testing.T) { - origDir, origCap := nodepath.StaticFilesDir, maxAssetBytes - t.Cleanup(func() { nodepath.StaticFilesDir, maxAssetBytes = origDir, origCap }) +// TestStageAssetStreaming covers the streamed download: good asset cached +// and staged, over-cap rejected, hash mismatch rejected (failures leave no +// cache entry). +func TestStageAssetStreaming(t *testing.T) { + origCap := maxAssetBytes + t.Cleanup(func() { maxAssetBytes = origCap }) content := []byte("micro-vm kernel bytes") goodHash := fmt.Sprintf("%x", sha256.Sum256(content)) const url = "gs://test-bucket/asset" - t.Run("good asset is cached", func(t *testing.T) { - nodepath.StaticFilesDir = t.TempDir() - s := &Atelet{anonGCSClient: fakeObjectStorage{data: content}} - path, err := s.fetchAsset(context.Background(), assetEntry{URL: url, SHA256: goodHash}) + t.Run("good asset is cached and staged", func(t *testing.T) { + root := t.TempDir() + s := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{data: content}} + path, err := stageTestAsset(t, s, root, assetEntry{URL: url, SHA256: goodHash}) if err != nil { - t.Fatalf("fetchAsset: %v", err) + t.Fatalf("stageAsset: %v", err) } got, err := os.ReadFile(path) if err != nil { - t.Fatalf("reading cached asset: %v", err) + t.Fatalf("reading staged asset: %v", err) } if !bytes.Equal(got, content) { - t.Errorf("cached bytes = %q, want %q", got, content) + t.Errorf("staged bytes = %q, want %q", got, content) + } + fi, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != 0o555 { + t.Errorf("staged mode = %v, want 0555 (read-only, executable)", fi.Mode().Perm()) + } + if n := cachedAssets(t, root); n != 1 { + t.Errorf("cache holds %d assets, want 1", n) } }) t.Run("over-cap asset rejected, cache not written", func(t *testing.T) { - nodepath.StaticFilesDir = t.TempDir() + root := t.TempDir() maxAssetBytes = 4 // content is longer than this - s := &Atelet{anonGCSClient: fakeObjectStorage{data: content}} - _, err := s.fetchAsset(context.Background(), assetEntry{URL: url, SHA256: goodHash}) + s := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{data: content}} + _, err := stageTestAsset(t, s, root, assetEntry{URL: url, SHA256: goodHash}) if err == nil { - t.Fatal("fetchAsset accepted an over-cap asset") + t.Fatal("stageAsset accepted an over-cap asset") } - if _, err := os.Stat(ateletpath.AssetPath(goodHash)); !errors.Is(err, os.ErrNotExist) { - t.Errorf("over-cap download left a file at the cache path (stat err = %v)", err) + if n := cachedAssets(t, root); n != 0 { + t.Errorf("over-cap download left %d assets in the cache", n) } }) t.Run("hash mismatch rejected, cache not written", func(t *testing.T) { - nodepath.StaticFilesDir = t.TempDir() + root := t.TempDir() maxAssetBytes = origCap wrongHash := strings.Repeat("a", 64) // valid 64-hex format, wrong value - s := &Atelet{anonGCSClient: fakeObjectStorage{data: content}} - _, err := s.fetchAsset(context.Background(), assetEntry{URL: url, SHA256: wrongHash}) + s := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{data: content}} + _, err := stageTestAsset(t, s, root, assetEntry{URL: url, SHA256: wrongHash}) if err == nil { - t.Fatal("fetchAsset accepted a hash mismatch") + t.Fatal("stageAsset accepted a hash mismatch") } - if _, err := os.Stat(ateletpath.AssetPath(wrongHash)); !errors.Is(err, os.ErrNotExist) { - t.Errorf("mismatched download left a file at the cache path (stat err = %v)", err) + if n := cachedAssets(t, root); n != 0 { + t.Errorf("mismatched download left %d assets in the cache", n) } }) t.Run("missing object keeps the client's sentinel", func(t *testing.T) { - nodepath.StaticFilesDir = t.TempDir() + root := t.TempDir() maxAssetBytes = origCap // The objectstorage clients tag a missing object with ErrObjectNotFound. notFound := fmt.Errorf("%w: no such object", objectstorage.ErrObjectNotFound) - s := &Atelet{anonGCSClient: fakeObjectStorage{err: notFound}} - _, err := s.fetchAsset(context.Background(), assetEntry{URL: url, SHA256: goodHash}) + s := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{err: notFound}} + _, err := stageTestAsset(t, s, root, assetEntry{URL: url, SHA256: goodHash}) if !errors.Is(err, objectstorage.ErrObjectNotFound) { t.Errorf("missing-object error lost the client's sentinel: %v", err) } }) t.Run("malformed url is rejected", func(t *testing.T) { - nodepath.StaticFilesDir = t.TempDir() + root := t.TempDir() maxAssetBytes = origCap - s := &Atelet{anonGCSClient: fakeObjectStorage{data: content}} + s := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{data: content}} // Invalid percent-escape: url.Parse rejects it inside objectstorage.Open. - _, err := s.fetchAsset(context.Background(), assetEntry{URL: "gs://bucket/%zz", SHA256: goodHash}) + _, err := stageTestAsset(t, s, root, assetEntry{URL: "gs://bucket/%zz", SHA256: goodHash}) if err == nil { - t.Fatal("fetchAsset accepted a malformed URL") + t.Fatal("stageAsset accepted a malformed URL") } }) t.Run("network error keeps its context wrap", func(t *testing.T) { - nodepath.StaticFilesDir = t.TempDir() + root := t.TempDir() maxAssetBytes = origCap - s := &Atelet{anonGCSClient: fakeObjectStorage{err: errors.New("connection refused")}} - _, err := s.fetchAsset(context.Background(), assetEntry{URL: url, SHA256: goodHash}) + s := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{err: errors.New("connection refused")}} + _, err := stageTestAsset(t, s, root, assetEntry{URL: url, SHA256: goodHash}) if err == nil { - t.Fatal("fetchAsset accepted a failing open") + t.Fatal("stageAsset accepted a failing open") } if !strings.Contains(err.Error(), "while fetching") { t.Errorf("open failure lost its context wrap: %v", err) } }) + + t.Run("already staged asset is kept without a fetch", func(t *testing.T) { + root := t.TempDir() + maxAssetBytes = origCap + var gets atomic.Int32 + s := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{data: content, gets: &gets}} + path, err := stageTestAsset(t, s, root, assetEntry{URL: url, SHA256: goodHash}) + if err != nil { + t.Fatalf("stageAsset: %v", err) + } + // A Checkpoint or Terminate of the same activation finds the link + // Run staged and uses it as it is. + again, err := s.stageAsset(context.Background(), assetEntry{URL: url, SHA256: goodHash}, path) + if err != nil || again != path { + t.Fatalf("stageAsset onto a staged path = %q, %v; want %q", again, err, path) + } + if gets.Load() != 1 { + t.Errorf("object fetched %d times, want 1", gets.Load()) + } + }) } // TestRPCBoundariesReject confirms each of the three RPCs validates path inputs diff --git a/cmd/atelet/sandbox_asset_cache.go b/cmd/atelet/sandbox_asset_cache.go new file mode 100644 index 0000000000..dccb19363a --- /dev/null +++ b/cmd/atelet/sandbox_asset_cache.go @@ -0,0 +1,113 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "syscall" + "time" + + "github.com/spf13/pflag" + + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/cmd/atelet/internal/filecache" + "go.ate.dev/substrate/internal/nodepath" +) + +var ( + sandboxAssetCacheDir = pflag.String("sandbox-asset-cache-dir", ateletpath.SandboxAssetCacheDir, "Directory for the node-local sandbox asset cache (gVisor releases, micro-VM kernels, VMMs and guest images). Must be on the same filesystem mount as the actor state dirs: an actor's assets are hard links from the cache into its own directory.") + // TODO: no eviction pass runs on the asset cache yet, so this value is + // handed to the store and not consulted; set or unset, the cache behaves + // the same. The flag is declared ahead of the GC pass, which the next + // change adds and which will read it, so the operator surface does not + // change when it lands. + sandboxAssetCacheMinAge = pflag.Duration("sandbox-asset-cache-min-age", time.Hour, "Cached sandbox assets younger than this are never evicted. No eviction pass runs on this cache yet, so the value is recorded for when one does. A running actor's assets are protected by its links regardless; this covers an asset prewarmed for a SandboxConfig that no actor on the node has used yet.") + sandboxAssetFetchTimeout = pflag.Duration("sandbox-asset-fetch-timeout", 5*time.Minute, "Bound on one sandbox asset download (and, for a gVisor release, its extraction). Downloads run detached from the requests waiting on them, so this is the only bound on how long one can run; a Run arriving while a download hangs waits on it until then. Sized for multi-hundred-MiB micro-VM guest images on a busy node.") +) + +func validateSandboxAssetCacheFlags() error { + if *sandboxAssetCacheMinAge < 0 { + // A negative min-age inverts the veto (the cutoff lands in the + // future), making just-fetched assets evictable. + return fmt.Errorf("--sandbox-asset-cache-min-age %v must be >= 0", *sandboxAssetCacheMinAge) + } + if *sandboxAssetFetchTimeout <= 0 { + return fmt.Errorf("--sandbox-asset-fetch-timeout %v must be > 0", *sandboxAssetFetchTimeout) + } + return nil +} + +// openSandboxAssetCache opens the asset cache at dir, sweeps the debris a +// crash may have left, and empties the scratch dir asset fetches without +// an actor link into. It refuses a dir from which the actor dirs cannot be +// hard-linked: every hit is such a link, so none could be served. +func openSandboxAssetCache(ctx context.Context, dir string, minAge, fetchTimeout time.Duration) (*filecache.Store, error) { + store, err := filecache.New(dir, filecache.WithMinAge(minAge), filecache.WithFetchTimeout(fetchTimeout)) + if err != nil { + return nil, fmt.Errorf("while opening sandbox asset cache at %s: %w", dir, err) + } + if err := canHardLink(dir, nodepath.ActorsDir); err != nil { + return nil, fmt.Errorf("sandbox asset cache: %w", err) + } + stats, err := store.SweepDebris(ctx) + if err != nil { + // Leftover debris wastes space but affects no lookup, so the store + // is fully usable: log and carry on rather than failing startup. + slog.WarnContext(ctx, "Sandbox asset cache debris sweep incomplete", slog.Any("err", err)) + } + if err := resetDir("sandbox asset scratch dir", ateletpath.SandboxAssetScratchDir, os.RemoveAll, 0o700); err != nil { + return nil, err + } + slog.InfoContext(ctx, "Sandbox asset cache open", + slog.String("dir", dir), + slog.Duration("min_age", minAge), + slog.Duration("fetch_timeout", fetchTimeout), + slog.Int("tmp_removed", stats.TmpRemoved), + slog.Int("retired_removed", stats.RetiredRemoved)) + return store, nil +} + +// canHardLink fails unless a file under cacheDir can be hard-linked into +// actorsDir, which every cache hit relies on. It links a probe file and +// removes it again. Nothing short of the probe is exact: Linux refuses a +// link across two mounts even when they are bind mounts of one filesystem, +// so equal device numbers do not settle it, and a path check cannot see a +// mount at all. actorsDir is created if it does not exist yet. +func canHardLink(cacheDir, actorsDir string) error { + if err := os.MkdirAll(actorsDir, 0o700); err != nil { + return fmt.Errorf("while creating %s: %w", actorsDir, err) + } + probe, err := os.CreateTemp(cacheDir, ".linkprobe-") + if err != nil { + return fmt.Errorf("while creating a link probe in %s: %w", cacheDir, err) + } + defer os.Remove(probe.Name()) + if err := probe.Close(); err != nil { + return fmt.Errorf("while closing the link probe: %w", err) + } + link := filepath.Join(actorsDir, filepath.Base(probe.Name())) + if err := os.Link(probe.Name(), link); err != nil { + if errors.Is(err, syscall.EXDEV) { + return fmt.Errorf("%s and %s are on different mounts, so assets cannot be hard-linked into actor dirs; put the cache on the mount that holds %s: %w", cacheDir, actorsDir, actorsDir, err) + } + return fmt.Errorf("while probing a hard link from %s into %s: %w", cacheDir, actorsDir, err) + } + return os.Remove(link) +} diff --git a/cmd/atelet/sandbox_asset_cache_linux_test.go b/cmd/atelet/sandbox_asset_cache_linux_test.go new file mode 100644 index 0000000000..96501eac5c --- /dev/null +++ b/cmd/atelet/sandbox_asset_cache_linux_test.go @@ -0,0 +1,75 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux + +package main + +import ( + "errors" + "os" + "path/filepath" + "syscall" + "testing" + "time" + + "go.ate.dev/substrate/internal/nodepath" +) + +// otherMount returns a fresh directory on a mount other than the one +// t.TempDir is on: /dev/shm is a tmpfs of its own and world-writable on +// every Linux, so the probe reaches the cross-mount link as any user. An +// unusable /dev/shm fails the test rather than skipping it: this is where +// the guard is seen red for the reason it exists. +func otherMount(t *testing.T) string { + t.Helper() + dir, err := os.MkdirTemp("/dev/shm", "ate-link-probe-") + if err != nil { + t.Fatalf("creating a directory on /dev/shm: %v", err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + return dir +} + +// TestCanHardLinkRejectsAnotherMount pins the one failure the probe exists +// for: the actors dir on a different mount than the cache, which Linux +// reports as EXDEV on the link and which no path or device check can see. +func TestCanHardLinkRejectsAnotherMount(t *testing.T) { + cache := filepath.Join(t.TempDir(), "cache") + if err := os.MkdirAll(cache, 0o700); err != nil { + t.Fatal(err) + } + actors := filepath.Join(otherMount(t), "actors") + err := canHardLink(cache, actors) + if !errors.Is(err, syscall.EXDEV) { + t.Fatalf("canHardLink across mounts = %v, want EXDEV", err) + } + // The failed probe leaves nothing behind on either side. + for _, dir := range []string{cache, actors} { + if entries, err := os.ReadDir(dir); err != nil || len(entries) != 0 { + t.Errorf("%s holds %d entries after the failed probe (%v), want none", dir, len(entries), err) + } + } +} + +func TestOpenSandboxAssetCacheRejectsAnotherMount(t *testing.T) { + useTempAssetScratch(t) + orig := nodepath.ActorsDir + nodepath.ActorsDir = filepath.Join(otherMount(t), "actors") + t.Cleanup(func() { nodepath.ActorsDir = orig }) + _, err := openSandboxAssetCache(t.Context(), filepath.Join(t.TempDir(), "cache"), time.Hour, time.Minute) + if !errors.Is(err, syscall.EXDEV) { + t.Fatalf("openSandboxAssetCache with the actors dir on another mount = %v, want EXDEV", err) + } +} diff --git a/cmd/atelet/sandbox_asset_cache_test.go b/cmd/atelet/sandbox_asset_cache_test.go new file mode 100644 index 0000000000..19871a620c --- /dev/null +++ b/cmd/atelet/sandbox_asset_cache_test.go @@ -0,0 +1,102 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "path/filepath" + "testing" + "time" + + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" +) + +func TestValidateSandboxAssetCacheFlags(t *testing.T) { + origMinAge, origTimeout := *sandboxAssetCacheMinAge, *sandboxAssetFetchTimeout + t.Cleanup(func() { *sandboxAssetCacheMinAge, *sandboxAssetFetchTimeout = origMinAge, origTimeout }) + for _, tc := range []struct { + name string + minAge time.Duration + timeout time.Duration + wantErr bool + }{ + {"defaults", origMinAge, origTimeout, false}, + {"zero min age", 0, time.Minute, false}, + {"negative min age", -time.Second, time.Minute, true}, + {"zero fetch timeout", time.Minute, 0, true}, + {"negative fetch timeout", time.Minute, -time.Second, true}, + } { + t.Run(tc.name, func(t *testing.T) { + *sandboxAssetCacheMinAge, *sandboxAssetFetchTimeout = tc.minAge, tc.timeout + if err := validateSandboxAssetCacheFlags(); (err != nil) != tc.wantErr { + t.Errorf("validateSandboxAssetCacheFlags() = %v, wantErr %v", err, tc.wantErr) + } + }) + } +} + +// TestOpenSandboxAssetCacheResetsScratch pins the startup contract: the +// cache opens under the given dir, and whatever a crashed prewarm left in +// the scratch dir is gone. +func TestOpenSandboxAssetCacheResetsScratch(t *testing.T) { + useTempNodeDirs(t) + useTempAssetScratch(t) + leftover := filepath.Join(ateletpath.SandboxAssetScratchDir, "prewarm-123", "runsc") + if err := os.MkdirAll(filepath.Dir(leftover), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(leftover, []byte("stale link"), 0o555); err != nil { + t.Fatal(err) + } + + dir := filepath.Join(t.TempDir(), "cache") + store, err := openSandboxAssetCache(t.Context(), dir, time.Hour, 5*time.Minute) + if err != nil { + t.Fatalf("openSandboxAssetCache: %v", err) + } + if store == nil { + t.Fatal("openSandboxAssetCache returned no store") + } + if _, err := os.Stat(filepath.Join(dir, "entries")); err != nil { + t.Errorf("cache layout not created under %s: %v", dir, err) + } + entries, err := os.ReadDir(ateletpath.SandboxAssetScratchDir) + if err != nil { + t.Fatalf("scratch dir after open: %v", err) + } + if len(entries) != 0 { + t.Errorf("scratch dir holds %d entries after open, want it emptied", len(entries)) + } +} + +func TestCanHardLink(t *testing.T) { + root := t.TempDir() + cache, actors := filepath.Join(root, "cache"), filepath.Join(root, "actors") + if err := os.MkdirAll(cache, 0o700); err != nil { + t.Fatal(err) + } + // The actors dir need not exist yet; the probe creates it and leaves + // nothing else behind. + if err := canHardLink(cache, actors); err != nil { + t.Errorf("two dirs under one temp dir: %v", err) + } + for _, dir := range []string{cache, actors} { + if entries, err := os.ReadDir(dir); err != nil || len(entries) != 0 { + t.Errorf("%s holds %d entries after the probe (%v), want none", dir, len(entries), err) + } + } + // The cross-mount case needs a second mount a non-root user can write + // to; see sandbox_asset_cache_linux_test.go. +} diff --git a/cmd/atelet/sandbox_assets.go b/cmd/atelet/sandbox_assets.go index 12b177a398..b454cbb2c0 100644 --- a/cmd/atelet/sandbox_assets.go +++ b/cmd/atelet/sandbox_assets.go @@ -36,13 +36,12 @@ import ( "runtime" "slices" "strings" - "syscall" "time" "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" "go.ate.dev/substrate/cmd/atelet/internal/atomicfile" + "go.ate.dev/substrate/cmd/atelet/internal/filecache" "go.ate.dev/substrate/internal/apierror" - "go.ate.dev/substrate/internal/nodepath" "go.ate.dev/substrate/internal/proto/ateletpb" "go.ate.dev/substrate/internal/resources" "go.ate.dev/substrate/pkg/api/v1alpha1" @@ -157,24 +156,36 @@ func checkPauseImage(sandboxClass, pauseImage string) error { return nil } -// ensureSandboxAssets fetches every asset in the record content-addressed and -// returns a map of asset name to local path. gVisor has a single "gvisor" -// release-tarball asset or a bare "runsc" binary; the micro-VM runtime has -// several (kata-shim, cloud-hypervisor, ...). -// Assets are cached, so re-fetching at Checkpoint/Restore is a no-op once -// present. -func (s *Atelet) ensureSandboxAssets(ctx context.Context, rec *sandboxAssetsRecord) (map[string]string, error) { - if err := os.MkdirAll(nodepath.StaticFilesDir, 0o700); err != nil { - return nil, fmt.Errorf("while creating static files dir: %w", err) +// ensureSandboxAssets stages every asset in the record under dstDir, named +// by asset name, and returns a map of asset name to the local path the worker +// runs it from. gVisor has a single "gvisor" release-tarball asset (staged +// as a directory, the path is its runsc) or a bare "runsc" binary; the +// micro-VM runtime has several (kata-shim, cloud-hypervisor, ...). +// +// Each asset is served from the node's content-addressed asset cache as a +// hard link, so the download runs once per node, concurrent callers for one +// asset share it, and what an actor was handed stays valid whatever the +// cache evicts later. An asset already present under dstDir is kept as it +// is: dstDir is the actor's own, reset at Run and Restore, so a Checkpoint +// or Terminate finds exactly what its activation staged. +func (s *Atelet) ensureSandboxAssets(ctx context.Context, rec *sandboxAssetsRecord, dstDir string) (map[string]string, error) { + // Owner-only, like the actor's bundle dir: the worker runs the binaries + // as root, and nothing else needs to see them. + if err := os.MkdirAll(dstDir, 0o700); err != nil { + return nil, wrapFileSystemErr("while creating sandbox assets dir", err) } paths := make(map[string]string, len(rec.Assets)) for name, entry := range rec.Assets { + if !plainAssetName(name) { + return nil, apierror.InvalidArgument("sandbox asset name %q is not a plain file name", name) + } + dst := filepath.Join(dstDir, name) var p string var err error if name == gvisorAssetName { - p, err = s.fetchGVisorRelease(ctx, entry) + p, err = s.stageGVisorRelease(ctx, entry, dst) } else { - p, err = s.fetchAsset(ctx, entry) + p, err = s.stageAsset(ctx, entry, dst) } if err != nil { return nil, fmt.Errorf("while fetching sandbox asset %q: %w", name, err) @@ -184,6 +195,14 @@ func (s *Atelet) ensureSandboxAssets(ctx context.Context, rec *sandboxAssetsReco return paths, nil } +// plainAssetName reports whether name is one path element that names a +// file of its own under the assets dir: not empty, not "." (which Base and +// IsLocal both accept, and which would make the assets dir itself the +// destination), not "..", and without separators. +func plainAssetName(name string) bool { + return name != "" && name != "." && name == filepath.Base(name) && filepath.IsLocal(name) +} + // runscAssetNames are the assets that carry the gVisor `runsc` binary, most // preferred first. var runscAssetNames = []string{gvisorAssetName, runscAssetName} @@ -199,104 +218,126 @@ func runscPathFor(paths map[string]string) string { return "" } -// fetchAsset downloads one content-addressed asset (verifying its sha256) into -// the shared static-files cache and returns its local path. On a cache hit it -// returns immediately. -func (s *Atelet) fetchAsset(ctx context.Context, entry assetEntry) (string, error) { - if err := resources.ValidateAssetHash(entry.SHA256); err != nil { - return "", wrapFileSystemErr("while validating asset hash", err) +// assetKey is the asset's cache key: its content hash, which the key +// constructor validates before anything is looked up or staged under it. +func assetKey(entry assetEntry) (filecache.Key, error) { + key, err := filecache.SHA256Key(entry.SHA256) + if err != nil { + return filecache.Key{}, wrapFileSystemErr("while validating asset hash", err) } + return key, nil +} - localPath := ateletpath.AssetPath(entry.SHA256) - _, err := os.Stat(localPath) +// staged reports whether dst already holds the asset from this activation. +func staged(ctx context.Context, dst string) (bool, error) { + _, err := os.Lstat(dst) if err == nil { - slog.DebugContext(ctx, "Sandbox asset cache hit", slog.String("path", localPath)) - return localPath, nil - } else if !errors.Is(err, os.ErrNotExist) { - return "", wrapFileSystemErr("while stat-ing local file", err) + slog.DebugContext(ctx, "Sandbox asset already staged", slog.String("path", dst)) + return true, nil + } + if !errors.Is(err, os.ErrNotExist) { + return false, wrapFileSystemErr("while stat-ing staged asset", err) } + return false, nil +} - slog.InfoContext(ctx, "Sandbox asset cache miss; downloading", slog.String("url", entry.displayURL()), slog.String("sha256", entry.SHA256)) - t := time.Now() - tmpName, err := s.downloadVerified(ctx, entry, filepath.Base(localPath)+"-download-") +// stageAsset links one single-file asset from the cache to dst, downloading +// it (and verifying its sha256) on a cache miss, and returns dst. +func (s *Atelet) stageAsset(ctx context.Context, entry assetEntry, dst string) (string, error) { + key, err := assetKey(entry) if err != nil { return "", err } - defer os.Remove(tmpName) // no-op if successful rename later in the function - - if err := os.Chmod(tmpName, 0o755); err != nil { - return "", wrapFileSystemErr("while setting file mode", err) + if ok, err := staged(ctx, dst); err != nil || ok { + return dst, err } - if err := os.Rename(tmpName, localPath); err != nil { - return "", wrapFileSystemErr("while renaming temp file to target", err) + fetch := func(ctx context.Context, path string) error { + slog.InfoContext(ctx, "Sandbox asset cache miss; downloading", slog.String("url", entry.displayURL()), slog.String("sha256", entry.SHA256)) + t := time.Now() + if err := s.downloadVerified(ctx, entry, path); err != nil { + return err + } + // Executable for everyone: a sandbox runtime may re-execute its + // binary after dropping privileges, so the group and other bits + // must stay. They grant nothing beyond that: the cache publishes + // the file read-only (0555), the directory it is linked into is + // the owner's alone, and the content is a public release artifact. + if err := os.Chmod(path, 0o755); err != nil { + return wrapFileSystemErr("while setting file mode", err) + } + slog.InfoContext(ctx, "Sandbox asset download complete", slog.String("url", entry.displayURL()), slog.Duration("duration", time.Since(t))) + return nil } - - slog.InfoContext(ctx, "Sandbox asset download complete", slog.String("path", localPath), slog.Duration("duration", time.Since(t))) - return localPath, nil -} - -// fetchGVisorRelease downloads the gVisor release tarball (gvisor.tar.zstd, -// verifying its sha256) and extracts it into a content-addressed directory in -// the shared static-files cache, returning the local path of the extracted -// `runsc` binary. -func (s *Atelet) fetchGVisorRelease(ctx context.Context, entry assetEntry) (string, error) { - if err := resources.ValidateAssetHash(entry.SHA256); err != nil { - return "", wrapFileSystemErr("while validating asset hash", err) + if err := s.sandboxAssetCache.GetFileTo(ctx, key, dst, fetch); err != nil && !stagedMeanwhile(ctx, dst, err) { + return "", err } + return dst, nil +} - releaseDir := ateletpath.GVisorReleaseDir(entry.SHA256) - runscPath := filepath.Join(releaseDir, "runsc") - _, err := os.Stat(releaseDir) - if err == nil { - slog.DebugContext(ctx, "gVisor release cache hit", slog.String("dir", releaseDir)) - return runscPath, nil - } else if !errors.Is(err, os.ErrNotExist) { - return "", wrapFileSystemErr("while stat-ing extracted release dir", err) - } +// stagedMeanwhile reports whether err from the cache says dst exists: a +// concurrent call for the same actor, such as a Checkpoint racing a +// Terminate, staged it first. A hard link is complete the moment it exists, +// and the cache places a tree by rename, so what is there is whole. +func stagedMeanwhile(ctx context.Context, dst string, err error) bool { + if !errors.Is(err, fs.ErrExist) { + return false + } + // A fetcher's own error reaches here wrapped too, and could wrap + // ErrExist for reasons of its own; only an existing dst means staged. + if _, lerr := os.Lstat(dst); lerr != nil { + return false + } + slog.DebugContext(ctx, "Sandbox asset staged by a concurrent call", slog.String("path", dst)) + return true +} - slog.InfoContext(ctx, "gVisor release cache miss; downloading", slog.String("url", entry.displayURL()), slog.String("sha256", entry.SHA256)) - tDownload := time.Now() - tarball, err := s.downloadVerified(ctx, entry, filepath.Base(releaseDir)+"-download-") +// stageGVisorRelease links the extracted gVisor release (gvisor.tar.zstd: +// runsc plus its gvisor-bin/ helpers, which must sit next to it) from the +// cache to the directory dst, downloading, verifying and extracting the +// tarball on a cache miss, and returns the local path of runsc. +func (s *Atelet) stageGVisorRelease(ctx context.Context, entry assetEntry, dst string) (string, error) { + runscPath := filepath.Join(dst, "runsc") + key, err := assetKey(entry) if err != nil { return "", err } - defer os.Remove(tarball) - slog.InfoContext(ctx, "gVisor release download complete", slog.String("url", entry.displayURL()), slog.Duration("duration", time.Since(tDownload))) - - tmpDir, err := os.MkdirTemp(nodepath.StaticFilesDir, filepath.Base(releaseDir)+"-extract-") - if err != nil { - return "", wrapFileSystemErr("while creating extraction dir", err) + if ok, err := staged(ctx, dst); err != nil || ok { + return runscPath, err + } + fetch := func(ctx context.Context, dir string) error { + slog.InfoContext(ctx, "gVisor release cache miss; downloading", slog.String("url", entry.displayURL()), slog.String("sha256", entry.SHA256)) + tDownload := time.Now() + // The tarball is downloaded into the tree it is extracted into and + // removed before the cache publishes the tree; on a failure the + // cache discards the whole tree, tarball included. + tarball := filepath.Join(dir, ".gvisor-release-download") + if err := s.downloadVerified(ctx, entry, tarball); err != nil { + return err + } + slog.InfoContext(ctx, "gVisor release download complete", slog.String("url", entry.displayURL()), slog.Duration("duration", time.Since(tDownload))) + + slog.InfoContext(ctx, "Extracting gVisor archive", slog.String("url", entry.displayURL())) + tExtract := time.Now() + // The redacted form ends in the URL path, so a query does not hide the + // archive suffix. + if err := extractTarArchive(ctx, tarball, entry.displayURL(), dir); err != nil { + return err + } + slog.InfoContext(ctx, "gVisor archive extraction complete", slog.String("url", entry.displayURL()), slog.Duration("duration", time.Since(tExtract))) + if fi, err := os.Stat(filepath.Join(dir, "runsc")); err != nil || !fi.Mode().IsRegular() { + return fmt.Errorf("gvisor tarball %v contains no runsc binary (stat: %v)", entry.displayURL(), err) + } + return os.Remove(tarball) } - defer func() { _ = os.RemoveAll(tmpDir) }() // no-op after rename - - slog.InfoContext(ctx, "Extracting gVisor archive", slog.String("path", tarball), slog.String("url", entry.displayURL())) - tExtract := time.Now() - // The redacted form ends in the URL path, so a query does not hide the - // archive suffix. - if err := extractTarArchive(ctx, tarball, entry.displayURL(), tmpDir); err != nil { + if err := s.sandboxAssetCache.GetDirTo(ctx, key, dst, fetch); err != nil && !stagedMeanwhile(ctx, dst, err) { return "", err } - slog.InfoContext(ctx, "gVisor archive extraction complete", slog.String("url", entry.displayURL()), slog.Duration("duration", time.Since(tExtract))) - if fi, err := os.Stat(filepath.Join(tmpDir, "runsc")); err != nil || !fi.Mode().IsRegular() { - return "", fmt.Errorf("gvisor tarball %v contains no runsc binary (stat: %v)", entry.displayURL(), err) - } - if err := os.Chmod(tmpDir, 0o755); err != nil { // MkdirTemp created it 0700 - return "", wrapFileSystemErr("while setting extraction dir mode", err) - } - if err := os.Rename(tmpDir, releaseDir); err != nil { - // A concurrent fetch of the same release may have won the rename. - if errors.Is(err, syscall.EEXIST) || errors.Is(err, syscall.ENOTEMPTY) { - return runscPath, nil - } - return "", wrapFileSystemErr("while renaming extraction dir to target", err) - } return runscPath, nil } -// downloadVerified fetches entry.URL into a temp file in the static-files -// cache and returns the temp path once the size cap and sha256 both check out. -// On error the temp file is removed; on success the caller owns it (rename it -// into place or extract from it, then remove it). +// downloadVerified fetches entry.URL into a new file at path, which must not +// exist, and returns once the size cap and sha256 both check out. On error +// the file is removed. // // Every asset is fetched through the object-store plugin's AssetProvider // first, with the plugin's credentials; atelet holds none of its own and does @@ -307,53 +348,53 @@ func (s *Atelet) fetchGVisorRelease(ctx context.Context, entry assetEntry) (stri // or failed unexpectedly (see anonymousFallbackCodes). A refusal on policy or // content, and a canceled request, are final. atelet verifies the bytes it // caches whichever path fetched them, since it executes them. -func (s *Atelet) downloadVerified(ctx context.Context, entry assetEntry, tmpPrefix string) (string, error) { +func (s *Atelet) downloadVerified(ctx context.Context, entry assetEntry, path string) error { wantSum, err := hex.DecodeString(entry.SHA256) if err != nil { - return "", fmt.Errorf("while parsing sha256 hash: %w", err) + return fmt.Errorf("while parsing sha256 hash: %w", err) } u, err := url.Parse(entry.URL) if err != nil { - return "", apierror.InvalidArgument("sandbox asset URL is not a valid URL: %v", urlErrorCause(err)) + return apierror.InvalidArgument("sandbox asset URL is not a valid URL: %v", urlErrorCause(err)) } shown := redactURL(u) - tmpFile, err := os.CreateTemp(nodepath.StaticFilesDir, tmpPrefix) + // Created here, so the plugin and the anonymous download both write into + // a file atelet owns; the plugin writes it by path. + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) if err != nil { - return "", wrapFileSystemErr("while creating temp file", err) + return wrapFileSystemErr("while creating download file", err) + } + if err := f.Close(); err != nil { + return wrapFileSystemErr("while closing download file", err) } - tmpName := tmpFile.Name() ok := false defer func() { if !ok { - os.Remove(tmpName) + os.Remove(path) } }() - // The plugin writes the file by path. - if err := tmpFile.Close(); err != nil { - return "", wrapFileSystemErr("while closing temp file", err) - } // atelet's only client of its own is an anonymous GCS client, for public // buckets such as gs://gvisor, so only a gs:// asset has a fallback. anonFallback := u.Scheme == "gs" var pluginCode codes.Code if s.assetPlugin != nil { - pluginCode = s.fetchFromPlugin(ctx, entry, shown, tmpName) + pluginCode = s.fetchFromPlugin(ctx, entry, shown, path) if pluginCode == codes.OK { - if err := verifyAssetFile(tmpName, shown, wantSum); err != nil { - return "", err + if err := verifyAssetFile(path, shown, wantSum); err != nil { + return err } ok = true - return tmpName, nil + return nil } if !anonFallback || !slices.Contains(anonymousFallbackCodes, pluginCode) { slog.WarnContext(ctx, "Object-store plugin failed to fetch sandbox asset", slog.String("url", shown), slog.String("code", pluginCode.String())) - return "", s.pluginError(ctx, entry, shown, pluginCode, nil) + return s.pluginError(ctx, entry, shown, pluginCode, nil) } if ctxErr := ctx.Err(); ctxErr != nil { - return "", fmt.Errorf("sandbox asset %s: %w", shown, ctxErr) + return fmt.Errorf("sandbox asset %s: %w", shown, ctxErr) } // Internal and Unavailable mean the plugin failed rather than declined // the URL, so falling back must not hide them: warn. The plugin logs @@ -365,17 +406,17 @@ func (s *Atelet) downloadVerified(ctx context.Context, entry assetEntry, tmpPref slog.Log(ctx, level, "Object-store plugin did not serve sandbox asset; downloading it anonymously", slog.String("url", shown), slog.String("code", pluginCode.String())) } else if !anonFallback { - return "", apierror.FailedPrecondition("sandbox asset %s: no object-store plugin is configured to fetch it", shown) + return apierror.FailedPrecondition("sandbox asset %s: no object-store plugin is configured to fetch it", shown) } - if err := s.downloadAnonymous(ctx, entry, shown, tmpName, wantSum); err != nil { + if err := s.downloadAnonymous(ctx, entry, shown, path, wantSum); err != nil { if s.assetPlugin == nil { - return "", err + return err } - return "", s.pluginError(ctx, entry, shown, pluginCode, err) + return s.pluginError(ctx, entry, shown, pluginCode, err) } ok = true - return tmpName, nil + return nil } // anonymousFallbackCodes are the plugin codes after which atelet reads a gs:// diff --git a/cmd/atelet/sandbox_assets_plugin_test.go b/cmd/atelet/sandbox_assets_plugin_test.go index f1521bc563..a008246283 100644 --- a/cmd/atelet/sandbox_assets_plugin_test.go +++ b/cmd/atelet/sandbox_assets_plugin_test.go @@ -30,10 +30,8 @@ import ( "testing" "time" - "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/ateinterceptors" - "go.ate.dev/substrate/internal/nodepath" "go.ate.dev/substrate/internal/objectstoreplugin" "go.ate.dev/substrate/internal/objectstoreplugin/objectstoreplugintest" "go.ate.dev/substrate/internal/pluginsocket" @@ -97,35 +95,42 @@ func (c *countingAssetClient) FetchAsset(ctx context.Context, in *objectstorev1. // plugin on an in-memory bucket, with an anonymous client that cannot read // that bucket. type assetHerderFixture struct { + t *testing.T herder *Atelet bucket *bucketStorage anon *bucketStorage plugin *countingAssetClient staging string + // root holds the asset cache (root/cache) and the destinations fetch + // stages into (root/consumer), on one filesystem. + root string +} + +// fetch stages entry into a fresh destination and returns the staged path. +func (f *assetHerderFixture) fetch(ctx context.Context, entry assetEntry) (string, error) { + return stageTestAsset(f.t, f.herder, f.root, entry) } // newAssetHerderFixture's plugin serves assets of scheme ("gs" or "s3"), as // the in-tree plugin on that backend does. func newAssetHerderFixture(t *testing.T, scheme string) *assetHerderFixture { t.Helper() - origDir, origCap := nodepath.StaticFilesDir, maxAssetBytes - t.Cleanup(func() { nodepath.StaticFilesDir, maxAssetBytes = origDir, origCap }) + origCap := maxAssetBytes + t.Cleanup(func() { maxAssetBytes = origCap }) root := t.TempDir() - nodepath.StaticFilesDir = filepath.Join(root, "static-files") - if err := os.MkdirAll(nodepath.StaticFilesDir, 0o700); err != nil { - t.Fatal(err) - } f := &assetHerderFixture{ + t: t, bucket: &bucketStorage{objects: map[string][]byte{}}, anon: &bucketStorage{objects: map[string][]byte{}}, staging: t.TempDir(), + root: root, } plugin, err := objectstoreplugin.NewAssetPlugin(f.bucket, scheme, root, f.staging, maxAssetBytes) if err != nil { t.Fatal(err) } f.plugin = &countingAssetClient{AssetProviderClient: objectstoreplugintest.AssetClient(plugin)} - f.herder = &Atelet{anonGCSClient: f.anon, assetPlugin: f.plugin, pluginSocket: testPluginSock} + f.herder = &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: f.anon, assetPlugin: f.plugin, pluginSocket: testPluginSock} return f } @@ -138,16 +143,11 @@ func (f *assetHerderFixture) put(t *testing.T, object string, content []byte) st return fmt.Sprintf("%x", sha256.Sum256(content)) } -// assertNotCached fails if fetching left anything in the static-files cache: -// neither the cache file nor a temp file. -func assertNotCached(t *testing.T) { - t.Helper() - entries, err := os.ReadDir(nodepath.StaticFilesDir) - if err != nil { - t.Fatal(err) - } - for _, e := range entries { - t.Errorf("failed fetch left %s in the static-files cache", e.Name()) +// assertNotCached fails if fetching published anything in the asset cache. +func (f *assetHerderFixture) assertNotCached() { + f.t.Helper() + if n := cachedAssets(f.t, f.root); n != 0 { + f.t.Errorf("failed fetch left %d assets in the cache", n) } } @@ -160,7 +160,7 @@ func TestFetchAssetThroughPlugin(t *testing.T) { t.Run("non-gs scheme goes straight to the plugin", func(t *testing.T) { f := newAssetHerderFixture(t, "s3") sum := f.put(t, "kata-assets/vmlinux", content) - path, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: "s3://" + testAssetBucket + "/kata-assets/vmlinux", SHA256: sum}) + path, err := f.fetch(t.Context(), assetEntry{URL: "s3://" + testAssetBucket + "/kata-assets/vmlinux", SHA256: sum}) if err != nil { t.Fatalf("fetchAsset: %v", err) } @@ -176,7 +176,7 @@ func TestFetchAssetThroughPlugin(t *testing.T) { f := newAssetHerderFixture(t, "gs") sum := f.put(t, "kata-assets/vmlinux", content) f.anon.objects[testAssetBucket+"/kata-assets/vmlinux"] = content - path, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: "gs://" + testAssetBucket + "/kata-assets/vmlinux", SHA256: sum}) + path, err := f.fetch(t.Context(), assetEntry{URL: "gs://" + testAssetBucket + "/kata-assets/vmlinux", SHA256: sum}) if err != nil { t.Fatalf("fetchAsset: %v", err) } @@ -196,7 +196,7 @@ func TestFetchAssetThroughPlugin(t *testing.T) { } f.anon.objects["gvisor/release.tar"] = content sum := fmt.Sprintf("%x", sha256.Sum256(content)) - path, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: "gs://gvisor/release.tar", SHA256: sum}) + path, err := f.fetch(t.Context(), assetEntry{URL: "gs://gvisor/release.tar", SHA256: sum}) if err != nil { t.Fatalf("fetchAsset: %v", err) } @@ -212,7 +212,7 @@ func TestFetchAssetThroughPlugin(t *testing.T) { f := newAssetHerderFixture(t, "s3") f.anon.objects["gvisor/release.tar"] = content sum := fmt.Sprintf("%x", sha256.Sum256(content)) - path, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: "gs://gvisor/release.tar", SHA256: sum}) + path, err := f.fetch(t.Context(), assetEntry{URL: "gs://gvisor/release.tar", SHA256: sum}) if err != nil { t.Fatalf("fetchAsset: %v", err) } @@ -227,56 +227,62 @@ func TestFetchAssetThroughPlugin(t *testing.T) { t.Run("cache hit", func(t *testing.T) { f := newAssetHerderFixture(t, "s3") sum := f.put(t, "kata-assets/vmlinux", content) - cached := ateletpath.AssetPath(sum) - if err := os.WriteFile(cached, content, 0o755); err != nil { - t.Fatal(err) + entry := assetEntry{URL: "s3://" + testAssetBucket + "/kata-assets/vmlinux", SHA256: sum} + first, err := f.fetch(t.Context(), entry) + if err != nil { + t.Fatalf("fetchAsset: %v", err) } - path, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: "s3://" + testAssetBucket + "/kata-assets/vmlinux", SHA256: sum}) - if err != nil || path != cached { - t.Fatalf("fetchAsset = %q, %v; want %q", path, err, cached) + second, err := f.fetch(t.Context(), entry) + if err != nil { + t.Fatalf("fetchAsset (hit): %v", err) } - if n := f.plugin.calls.Load(); n != 0 { - t.Errorf("plugin called %d times on a cache hit", n) + if n := f.plugin.calls.Load(); n != 1 { + t.Errorf("plugin called %d times for two fetches of one asset, want 1", n) + } + fi1, err1 := os.Stat(first) + fi2, err2 := os.Stat(second) + if err1 != nil || err2 != nil || !os.SameFile(fi1, fi2) { + t.Errorf("the two staged paths are not one inode (%v, %v)", err1, err2) } }) t.Run("sha256 mismatch", func(t *testing.T) { f := newAssetHerderFixture(t, "s3") f.put(t, "kata-assets/vmlinux", content) - _, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: "s3://" + testAssetBucket + "/kata-assets/vmlinux", SHA256: strings.Repeat("a", 64)}) + _, err := f.fetch(t.Context(), assetEntry{URL: "s3://" + testAssetBucket + "/kata-assets/vmlinux", SHA256: strings.Repeat("a", 64)}) if apierror.Code(err) != codes.FailedPrecondition { t.Fatalf("fetchAsset with a wrong sha256 = %v (code %s), want %s", err, apierror.Code(err), codes.FailedPrecondition) } - assertNotCached(t) + f.assertNotCached() }) t.Run("size cap", func(t *testing.T) { f := newAssetHerderFixture(t, "s3") sum := f.put(t, "kata-assets/vmlinux", content) maxAssetBytes = int64(len(content)) - 1 - _, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: "s3://" + testAssetBucket + "/kata-assets/vmlinux", SHA256: sum}) + _, err := f.fetch(t.Context(), assetEntry{URL: "s3://" + testAssetBucket + "/kata-assets/vmlinux", SHA256: sum}) if apierror.Code(err) != codes.FailedPrecondition { t.Fatalf("fetchAsset over the cap = %v (code %s), want %s", err, apierror.Code(err), codes.FailedPrecondition) } - assertNotCached(t) + f.assertNotCached() }) t.Run("not found", func(t *testing.T) { f := newAssetHerderFixture(t, "s3") - _, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: "s3://" + testAssetBucket + "/kata-assets/missing", SHA256: strings.Repeat("a", 64)}) + _, err := f.fetch(t.Context(), assetEntry{URL: "s3://" + testAssetBucket + "/kata-assets/missing", SHA256: strings.Repeat("a", 64)}) // Not NotFound: ate-api-server reads NotFound from Terminate as // "already terminated". if apierror.Code(err) != codes.FailedPrecondition || !strings.Contains(err.Error(), "s3://"+testAssetBucket+"/kata-assets/missing") { t.Fatalf("fetchAsset of a missing object = %v (code %s), want %s naming the URL", err, apierror.Code(err), codes.FailedPrecondition) } - assertNotCached(t) + f.assertNotCached() }) t.Run("snapshot in the same bucket is refused", func(t *testing.T) { f := newAssetHerderFixture(t, "gs") sum := f.put(t, "root/atespaces/team-a/actors/uid1/snapshots/snap1/checkpoint.img.zstd", []byte("another actor's memory")) f.anon.objects[testAssetBucket+"/root/atespaces/team-a/actors/uid1/snapshots/snap1/checkpoint.img.zstd"] = []byte("another actor's memory") - _, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: "gs://" + testAssetBucket + "/root/atespaces/team-a/actors/uid1/snapshots/snap1/checkpoint.img.zstd", SHA256: sum}) + _, err := f.fetch(t.Context(), assetEntry{URL: "gs://" + testAssetBucket + "/root/atespaces/team-a/actors/uid1/snapshots/snap1/checkpoint.img.zstd", SHA256: sum}) if apierror.Code(err) != codes.FailedPrecondition || !strings.Contains(err.Error(), "snapshot location") { t.Fatalf("fetchAsset of a snapshot file = %v (code %s), want %s about the snapshot location", err, apierror.Code(err), codes.FailedPrecondition) } @@ -286,7 +292,7 @@ func TestFetchAssetThroughPlugin(t *testing.T) { if n := f.anon.gets.Load(); n != 0 { t.Errorf("a refused snapshot file was tried anonymously %d times", n) } - assertNotCached(t) + f.assertNotCached() }) t.Run("connection error is Unavailable", func(t *testing.T) { @@ -297,11 +303,11 @@ func TestFetchAssetThroughPlugin(t *testing.T) { } t.Cleanup(func() { conn.Close() }) f.herder.assetPlugin = objectstorev1.NewAssetProviderClient(conn) - _, err = f.herder.fetchAsset(t.Context(), assetEntry{URL: "s3://" + testAssetBucket + "/kata-assets/vmlinux", SHA256: strings.Repeat("a", 64)}) + _, err = f.fetch(t.Context(), assetEntry{URL: "s3://" + testAssetBucket + "/kata-assets/vmlinux", SHA256: strings.Repeat("a", 64)}) if apierror.Code(err) != codes.Unavailable { t.Fatalf("fetchAsset with the plugin down = %v (code %s), want %s", err, apierror.Code(err), codes.Unavailable) } - assertNotCached(t) + f.assertNotCached() }) t.Run("plugin without AssetProvider", func(t *testing.T) { @@ -327,7 +333,7 @@ func TestFetchAssetThroughPlugin(t *testing.T) { t.Cleanup(func() { conn.Close() }) f.herder.assetPlugin = objectstorev1.NewAssetProviderClient(conn) f.herder.pluginSocket = sock - _, err = f.herder.fetchAsset(t.Context(), assetEntry{URL: "s3://" + testAssetBucket + "/kata-assets/vmlinux", SHA256: strings.Repeat("a", 64)}) + _, err = f.fetch(t.Context(), assetEntry{URL: "s3://" + testAssetBucket + "/kata-assets/vmlinux", SHA256: strings.Repeat("a", 64)}) if apierror.Code(err) != codes.FailedPrecondition { t.Fatalf("fetchAsset with an old plugin = %v (code %s), want %s", err, apierror.Code(err), codes.FailedPrecondition) } @@ -380,7 +386,7 @@ func TestFetchAssetAnonymousFallback(t *testing.T) { f.anon.objects[object] = content // A partial write the plugin leaves behind is replaced. f.herder.assetPlugin = fakeAssetClient{data: []byte("partial plugin write, longer than the asset"), err: pluginErr(code)} - path, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: gsURL, SHA256: sum}) + path, err := f.fetch(t.Context(), assetEntry{URL: gsURL, SHA256: sum}) if err != nil { t.Fatalf("fetchAsset: %v", err) } @@ -414,7 +420,7 @@ func TestFetchAssetAnonymousFallback(t *testing.T) { logs := captureLogs(t) f := newAssetHerderFixture(t, "s3") f.herder.assetPlugin = fakeAssetClient{err: pluginErr(code)} - _, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: gsURL, SHA256: sum}) + _, err := f.fetch(t.Context(), assetEntry{URL: gsURL, SHA256: sum}) if err == nil { t.Fatal("fetchAsset succeeded") } @@ -443,7 +449,7 @@ func TestFetchAssetAnonymousFallback(t *testing.T) { t.Errorf("%s carries the plugin's status text:\n%s", name, text) } } - assertNotCached(t) + f.assertNotCached() }) for _, u := range []string{"s3://" + object, "https://storage.example/" + object} { @@ -451,13 +457,13 @@ func TestFetchAssetAnonymousFallback(t *testing.T) { f := newAssetHerderFixture(t, "s3") f.anon.objects[object] = content f.herder.assetPlugin = fakeAssetClient{err: pluginErr(code)} - if _, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: u, SHA256: sum}); err == nil { + if _, err := f.fetch(t.Context(), assetEntry{URL: u, SHA256: sum}); err == nil { t.Fatal("fetchAsset succeeded") } if n := f.anon.gets.Load(); n != 0 { t.Errorf("anonymous reads = %d, want 0", n) } - assertNotCached(t) + f.assertNotCached() }) } } @@ -467,7 +473,7 @@ func TestFetchAssetAnonymousFallback(t *testing.T) { f := newAssetHerderFixture(t, "s3") f.anon.objects[object] = content f.herder.assetPlugin = fakeAssetClient{err: pluginErr(code)} - _, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: gsURL, SHA256: sum}) + _, err := f.fetch(t.Context(), assetEntry{URL: gsURL, SHA256: sum}) if err == nil { t.Fatal("fetchAsset succeeded") } @@ -477,30 +483,38 @@ func TestFetchAssetAnonymousFallback(t *testing.T) { if strings.Contains(err.Error(), "anonymous") || strings.Contains(err.Error(), "SECRET") { t.Errorf("error %q mentions an anonymous read or the plugin's text", err) } - assertNotCached(t) + f.assertNotCached() }) } - t.Run("canceled request does not fall back", func(t *testing.T) { + t.Run("canceled caller does not abort the download", func(t *testing.T) { f := newAssetHerderFixture(t, "s3") f.anon.objects[object] = content ctx, cancel := context.WithCancel(t.Context()) + // The caller's context is canceled while the plugin is being asked. + // The download runs detached from it in the asset cache, so the + // plugin's refusal still falls back to the anonymous read, and the + // caller either sees its cancellation or the finished fetch. f.herder.assetPlugin = cancelingAssetClient{cancel: cancel, err: pluginErr(codes.Unavailable)} - _, err := f.herder.fetchAsset(ctx, assetEntry{URL: gsURL, SHA256: sum}) - if !errors.Is(err, context.Canceled) { - t.Fatalf("fetchAsset = %v, want context.Canceled", err) + _, err := f.fetch(ctx, assetEntry{URL: gsURL, SHA256: sum}) + if err != nil && !errors.Is(err, context.Canceled) { + t.Fatalf("fetchAsset = %v, want context.Canceled or success", err) } - if n := f.anon.gets.Load(); n != 0 { - t.Errorf("anonymous reads = %d, want 0", n) + // A second caller joins or hits what the first one started: one + // anonymous read serves both. + if _, err := f.fetch(t.Context(), assetEntry{URL: gsURL, SHA256: sum}); err != nil { + t.Fatalf("fetchAsset after a canceled caller: %v", err) + } + if n := f.anon.gets.Load(); n != 1 { + t.Errorf("anonymous reads = %d, want 1", n) } - assertNotCached(t) }) t.Run("no plugin reads gs anonymously", func(t *testing.T) { f := newAssetHerderFixture(t, "s3") f.anon.objects[object] = content f.herder.assetPlugin = nil - path, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: gsURL, SHA256: sum}) + path, err := f.fetch(t.Context(), assetEntry{URL: gsURL, SHA256: sum}) if err != nil { t.Fatalf("fetchAsset: %v", err) } @@ -515,25 +529,25 @@ func TestFetchAssetAnonymousFallback(t *testing.T) { t.Run("no plugin, gs anonymous failure", func(t *testing.T) { f := newAssetHerderFixture(t, "s3") f.herder.assetPlugin = nil - _, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: gsURL, SHA256: sum}) + _, err := f.fetch(t.Context(), assetEntry{URL: gsURL, SHA256: sum}) if err == nil || !strings.Contains(err.Error(), "object not found") || strings.Contains(err.Error(), "object-store plugin") { t.Fatalf("fetchAsset = %v, want the anonymous read's error alone", err) } - assertNotCached(t) + f.assertNotCached() }) t.Run("no plugin, non-gs is refused", func(t *testing.T) { f := newAssetHerderFixture(t, "s3") f.anon.objects[object] = content f.herder.assetPlugin = nil - _, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: "s3://" + object, SHA256: sum}) + _, err := f.fetch(t.Context(), assetEntry{URL: "s3://" + object, SHA256: sum}) if apierror.Code(err) != codes.FailedPrecondition || !strings.Contains(err.Error(), "no object-store plugin") { t.Fatalf("fetchAsset = %v (code %s), want %s naming the missing plugin", err, apierror.Code(err), codes.FailedPrecondition) } if n := f.anon.gets.Load(); n != 0 { t.Errorf("anonymous reads = %d, want 0", n) } - assertNotCached(t) + f.assertNotCached() }) } @@ -565,10 +579,10 @@ func TestFetchAssetVerifiesPluginOutput(t *testing.T) { f := newAssetHerderFixture(t, "s3") maxAssetBytes = tc.cap f.herder.assetPlugin = fakeAssetClient{data: tc.data} - if _, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: "s3://" + testAssetBucket + "/kata-assets/vmlinux", SHA256: sum}); err == nil { + if _, err := f.fetch(t.Context(), assetEntry{URL: "s3://" + testAssetBucket + "/kata-assets/vmlinux", SHA256: sum}); err == nil { t.Fatal("fetchAsset accepted bytes the plugin wrote without checking them") } - assertNotCached(t) + f.assertNotCached() }) } } @@ -615,7 +629,7 @@ func TestFetchAssetKeepsCredentialsOutOfErrors(t *testing.T) { // what ate-api-server receives. _, err := ateinterceptors.InternalServerUnaryInterceptor(t.Context(), nil, &grpc.UnaryServerInfo{FullMethod: "/test"}, func(ctx context.Context, _ any) (any, error) { - return f.herder.fetchAsset(ctx, assetEntry{URL: signed, SHA256: sum}) + return f.fetch(ctx, assetEntry{URL: signed, SHA256: sum}) }) if err == nil { t.Fatal("fetchAsset succeeded") @@ -645,7 +659,7 @@ func TestFetchAssetLogsPluginCodeAndRedactedURL(t *testing.T) { logs := captureLogs(t) f := newAssetHerderFixture(t, "s3") f.herder.assetPlugin = fakeAssetClient{err: status.Error(codes.PermissionDenied, "denied SECRET")} - _, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: "https://acct.blob.example/assets/vmlinux?sig=SECRET", SHA256: strings.Repeat("a", 64)}) + _, err := f.fetch(t.Context(), assetEntry{URL: "https://acct.blob.example/assets/vmlinux?sig=SECRET", SHA256: strings.Repeat("a", 64)}) if err == nil { t.Fatal("fetchAsset succeeded") } diff --git a/cmd/atelet/sandbox_assets_test.go b/cmd/atelet/sandbox_assets_test.go index a99e709eda..f2fc9a64ae 100644 --- a/cmd/atelet/sandbox_assets_test.go +++ b/cmd/atelet/sandbox_assets_test.go @@ -16,18 +16,30 @@ package main import ( "archive/tar" + "bytes" "compress/gzip" "context" + "crypto/sha256" + "errors" + "fmt" "io" + "io/fs" "os" "path/filepath" "runtime" + "strconv" "strings" + "sync" + "sync/atomic" "testing" "github.com/google/go-cmp/cmp" "github.com/klauspost/compress/zstd" + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/cmd/atelet/internal/filecache" + "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/proto/ateletpb" + "google.golang.org/grpc/codes" ) func TestExtractTarArchive(t *testing.T) { @@ -237,3 +249,282 @@ func TestRecordFromRequest(t *testing.T) { }) } } + +// newTestAssetCacheAt opens an asset cache under root/cache. Destinations a +// test links into should live under the same root, on the cache's +// filesystem. +func newTestAssetCacheAt(t *testing.T, root string) *filecache.Store { + t.Helper() + store, err := filecache.New(filepath.Join(root, "cache")) + if err != nil { + t.Fatalf("filecache.New: %v", err) + } + return store +} + +// newTestAssetCache opens an asset cache in a fresh temp dir. +func newTestAssetCache(t *testing.T) *filecache.Store { + t.Helper() + return newTestAssetCacheAt(t, t.TempDir()) +} + +// cachedAssets returns the number of assets published in the cache under +// root (see newTestAssetCacheAt): one directory per asset under entries/. +func cachedAssets(t *testing.T, root string) int { + t.Helper() + entries, err := os.ReadDir(filepath.Join(root, "cache", "entries")) + if err != nil { + t.Fatal(err) + } + return len(entries) +} + +// useTempAssetScratch points prewarm's scratch dir at a temp dir. Not +// parallel-safe: the path is process-global. +func useTempAssetScratch(t *testing.T) { + t.Helper() + orig := ateletpath.SandboxAssetScratchDir + ateletpath.SandboxAssetScratchDir = t.TempDir() + t.Cleanup(func() { ateletpath.SandboxAssetScratchDir = orig }) +} + +var stageSeq struct { + sync.Mutex + n int +} + +// stageTestAsset stages entry through s into a fresh destination under +// root (the cache's filesystem) and returns the staged path. +func stageTestAsset(t *testing.T, s *Atelet, root string, entry assetEntry) (string, error) { + t.Helper() + stageSeq.Lock() + stageSeq.n++ + n := stageSeq.n + stageSeq.Unlock() + dir := filepath.Join(root, "consumer", strconv.Itoa(n)) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + return s.stageAsset(t.Context(), entry, filepath.Join(dir, "asset")) +} + +// gvisorReleaseTarball builds an uncompressed gVisor release tarball: runsc +// next to a gvisor-bin/ helper, as the real release ships them. +func gvisorReleaseTarball(t *testing.T) []byte { + t.Helper() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for _, f := range []struct { + name, body string + mode int64 + }{ + {"runsc", "runsc bytes", 0o755}, + {"gvisor-bin/containerd-shim", "shim bytes", 0o755}, + } { + if err := tw.WriteHeader(&tar.Header{Name: f.name, Mode: f.mode, Size: int64(len(f.body))}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(f.body)); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +// TestStageGVisorRelease stages the release tree through the asset cache: +// one download and extraction serves every actor, each gets its own +// directory of links to the cached files, and the tarball itself is not +// published with them. +func TestStageGVisorRelease(t *testing.T) { + tarball := gvisorReleaseTarball(t) + entry := assetEntry{URL: "gs://gvisor/release.tar", SHA256: fmt.Sprintf("%x", sha256.Sum256(tarball))} + root := t.TempDir() + var gets atomic.Int32 + s := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{data: tarball, gets: &gets}} + + dst1 := filepath.Join(root, "actor-1", gvisorAssetName) + if err := os.MkdirAll(filepath.Dir(dst1), 0o755); err != nil { + t.Fatal(err) + } + runsc1, err := s.stageGVisorRelease(t.Context(), entry, dst1) + if err != nil { + t.Fatalf("stageGVisorRelease: %v", err) + } + if runsc1 != filepath.Join(dst1, "runsc") { + t.Errorf("runsc path = %q, want it inside %q", runsc1, dst1) + } + for rel, want := range map[string]string{"runsc": "runsc bytes", "gvisor-bin/containerd-shim": "shim bytes"} { + got, err := os.ReadFile(filepath.Join(dst1, rel)) + if err != nil || string(got) != want { + t.Errorf("%s = %q, %v; want %q", rel, got, err, want) + } + fi, err := os.Stat(filepath.Join(dst1, rel)) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != 0o555 { + t.Errorf("%s mode = %v, want 0555", rel, fi.Mode().Perm()) + } + } + if _, err := os.Lstat(filepath.Join(dst1, ".gvisor-release-download")); !errors.Is(err, os.ErrNotExist) { + t.Errorf("the downloaded tarball was staged with the tree (lstat err = %v)", err) + } + + // A second actor links the same cached tree: no second download. + dst2 := filepath.Join(root, "actor-2", gvisorAssetName) + if err := os.MkdirAll(filepath.Dir(dst2), 0o755); err != nil { + t.Fatal(err) + } + runsc2, err := s.stageGVisorRelease(t.Context(), entry, dst2) + if err != nil { + t.Fatalf("stageGVisorRelease (hit): %v", err) + } + fi1, err1 := os.Stat(runsc1) + fi2, err2 := os.Stat(runsc2) + if err1 != nil || err2 != nil || !os.SameFile(fi1, fi2) { + t.Errorf("the two actors' runsc are not one inode (%v, %v)", err1, err2) + } + if gets.Load() != 1 { + t.Errorf("release downloaded %d times, want 1", gets.Load()) + } + if n := cachedAssets(t, root); n != 1 { + t.Errorf("cache holds %d assets, want 1", n) + } + + // Re-staging onto the first actor's existing tree keeps it. + again, err := s.stageGVisorRelease(t.Context(), entry, dst1) + if err != nil || again != runsc1 { + t.Errorf("stageGVisorRelease onto a staged tree = %q, %v; want %q", again, err, runsc1) + } + + // A tarball without runsc is rejected and nothing is published. + var noRunsc bytes.Buffer + tw := tar.NewWriter(&noRunsc) + if err := tw.WriteHeader(&tar.Header{Name: "README", Mode: 0o644, Size: 2}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte("hi")); err != nil { + t.Fatal(err) + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + bad := assetEntry{URL: "gs://gvisor/bad.tar", SHA256: fmt.Sprintf("%x", sha256.Sum256(noRunsc.Bytes()))} + s = &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{data: noRunsc.Bytes()}} + if _, err := s.stageGVisorRelease(t.Context(), bad, filepath.Join(root, "actor-3", gvisorAssetName)); err == nil || !strings.Contains(err.Error(), "no runsc binary") { + t.Errorf("stageGVisorRelease of a tarball without runsc = %v, want the missing-runsc error", err) + } + if n := cachedAssets(t, root); n != 1 { + t.Errorf("cache holds %d assets after the rejected tarball, want 1", n) + } +} + +// TestEnsureSandboxAssetsConcurrentSameActor pins that two calls for one +// actor at once, as a Checkpoint racing a Terminate, both succeed with the +// same paths: the loser of each link finds the winner's complete result. +func TestEnsureSandboxAssetsConcurrentSameActor(t *testing.T) { + tarball, kernel := gvisorReleaseTarball(t), []byte("kernel bytes") + root := t.TempDir() + storage := &bucketStorage{objects: map[string][]byte{"gvisor/release.tar": tarball, "assets/vmlinux": kernel}} + s := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: storage} + // A tree asset and a single-file asset, so both link-out paths race. + rec := &sandboxAssetsRecord{ + SandboxClass: "gvisor", + Assets: map[string]assetEntry{ + gvisorAssetName: {URL: "gs://gvisor/release.tar", SHA256: fmt.Sprintf("%x", sha256.Sum256(tarball))}, + "kata-kernel": {URL: "gs://assets/vmlinux", SHA256: fmt.Sprintf("%x", sha256.Sum256(kernel))}, + }, + } + dstDir := filepath.Join(root, "actor", "sandbox-assets") + + const n = 8 + var wg sync.WaitGroup + results := make([]map[string]string, n) + errs := make([]error, n) + for i := range n { + wg.Go(func() { results[i], errs[i] = s.ensureSandboxAssets(t.Context(), rec, dstDir) }) + } + wg.Wait() + for i := range n { + if errs[i] != nil { + t.Fatalf("call %d: %v", i, errs[i]) + } + if results[i][gvisorAssetName] != filepath.Join(dstDir, gvisorAssetName, "runsc") || results[i]["kata-kernel"] != filepath.Join(dstDir, "kata-kernel") { + t.Errorf("call %d paths = %v", i, results[i]) + } + } + if _, err := os.Stat(filepath.Join(dstDir, gvisorAssetName, "gvisor-bin", "containerd-shim")); err != nil { + t.Errorf("staged tree incomplete: %v", err) + } + siblings, err := os.ReadDir(dstDir) + if err != nil { + t.Fatal(err) + } + if len(siblings) != 2 { + t.Errorf("sandbox assets dir holds %d entries, want the 2 assets only", len(siblings)) + } +} + +// TestStageAssetFetchErrorWrappingErrExistIsNotStaged pins that only an +// existing destination counts as staged by a concurrent call: a fetcher +// failing with an error that happens to wrap ErrExist still fails the +// stage, rather than reporting success with nothing at the destination. +func TestStageAssetFetchErrorWrappingErrExistIsNotStaged(t *testing.T) { + root := t.TempDir() + s := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{err: fmt.Errorf("object already there: %w", fs.ErrExist)}} + entry := assetEntry{URL: "gs://bucket/runsc", SHA256: strings.Repeat("d", 64)} + dst := filepath.Join(root, "actor", "runsc") + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + t.Fatal(err) + } + if _, err := s.stageAsset(t.Context(), entry, dst); err == nil { + t.Fatal("stageAsset reported success with nothing staged") + } + if _, err := os.Lstat(dst); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("destination exists after a failed fetch (lstat err = %v)", err) + } +} + +// TestEnsureSandboxAssetsRejectsBadAssetNames pins that an asset name that +// is not one plain path element is refused before anything is fetched or +// staged. "." is the case that slips past Base and IsLocal: it would make +// the assets dir itself the destination, which staged would then report as +// present without a fetch. +func TestEnsureSandboxAssetsRejectsBadAssetNames(t *testing.T) { + content := []byte("kernel bytes") + sha := fmt.Sprintf("%x", sha256.Sum256(content)) + for _, name := range []string{".", "..", "", "kata/kernel", "/kata-kernel", "./kata-kernel"} { + t.Run(fmt.Sprintf("%q", name), func(t *testing.T) { + root := t.TempDir() + var gets atomic.Int32 + s := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{data: content, gets: &gets}} + dstDir := filepath.Join(root, "actor", "sandbox-assets") + rec := &sandboxAssetsRecord{SandboxClass: "microvm", Assets: map[string]assetEntry{name: {URL: "gs://assets/vmlinux", SHA256: sha}}} + _, err := s.ensureSandboxAssets(t.Context(), rec, dstDir) + if apierror.Code(err) != codes.InvalidArgument { + t.Fatalf("ensureSandboxAssets with asset name %q = %v (code %s), want %s", name, err, apierror.Code(err), codes.InvalidArgument) + } + if gets.Load() != 0 { + t.Errorf("asset fetched %d times for a refused name", gets.Load()) + } + if entries, _ := os.ReadDir(dstDir); len(entries) != 0 { + t.Errorf("refused name left %d entries in the assets dir", len(entries)) + } + }) + } + + // A plain name is staged as the file it names. + root := t.TempDir() + s := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{data: content}} + dstDir := filepath.Join(root, "actor", "sandbox-assets") + paths, err := s.ensureSandboxAssets(t.Context(), &sandboxAssetsRecord{SandboxClass: "microvm", Assets: map[string]assetEntry{"kata-kernel": {URL: "gs://assets/vmlinux", SHA256: sha}}}, dstDir) + if err != nil { + t.Fatalf("ensureSandboxAssets: %v", err) + } + if fi, err := os.Stat(paths["kata-kernel"]); err != nil || !fi.Mode().IsRegular() || paths["kata-kernel"] != filepath.Join(dstDir, "kata-kernel") { + t.Errorf("kata-kernel staged at %q (%v), want a regular file at %s", paths["kata-kernel"], err, filepath.Join(dstDir, "kata-kernel")) + } +} diff --git a/cmd/atelet/sandbox_prewarm.go b/cmd/atelet/sandbox_prewarm.go index 12a722c4a2..f1ee58068b 100644 --- a/cmd/atelet/sandbox_prewarm.go +++ b/cmd/atelet/sandbox_prewarm.go @@ -20,6 +20,7 @@ import ( "fmt" "log/slog" "math/rand/v2" + "os" "runtime" "sync" "time" @@ -28,6 +29,7 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" "go.ate.dev/substrate/internal/imagecache" "go.ate.dev/substrate/pkg/api/v1alpha1" listersv1alpha1 "go.ate.dev/substrate/pkg/client/listers/api/v1alpha1" @@ -67,13 +69,13 @@ var prewarmFinished = func(string, error) {} // Prewarming through the same method as the Run/Restore path keeps the two // fetches from ever diverging on cache layout or validation. type sandboxAssetFetcher interface { - ensureSandboxAssets(ctx context.Context, rec *sandboxAssetsRecord) (map[string]string, error) + ensureSandboxAssets(ctx context.Context, rec *sandboxAssetsRecord, dstDir string) (map[string]string, error) } -// sandboxPrewarmer downloads SandboxConfig assets into the node's -// content-addressed static-files cache before any actor asks for them, so the -// fetch inside the first Run/Restore on the node is a cache hit instead of a -// download+extract on the critical path. +// sandboxPrewarmer downloads SandboxConfig assets into the node's asset +// cache before any actor asks for them, so the fetch inside the first +// Run/Restore on the node is a cache hit instead of a download+extract on +// the critical path. type sandboxPrewarmer struct { assets sandboxAssetFetcher images *imagecache.Store @@ -117,7 +119,7 @@ func newSandboxPrewarmer(assets sandboxAssetFetcher, images *imagecache.Store, l // is logged and left to the on-demand fetch in ensureSandboxAssets and the // pull inside prepareOCIBundles, which remain the correctness path. // -// TODO: the static-files cache is never pruned, and prewarming every config +// TODO: the asset cache is never pruned, and prewarming every config // revision makes stale releases accumulate faster. Add a GC that removes // assets referenced by no current SandboxConfig and no on-node actor record. func startSandboxAssetPrewarm(ctx context.Context, informer cache.SharedIndexInformer, assets sandboxAssetFetcher, images *imagecache.Store, microvmCapable bool) (*sandboxPrewarmer, error) { @@ -254,11 +256,14 @@ func (p *sandboxPrewarmer) process(ctx context.Context, name string) { p.queue.Forget(name) } -// prewarm fetches every asset of one SandboxConfig into the static-files -// cache and its pause image into the image cache. Racing an on-demand -// ensureSandboxAssets for the same assets is safe: both paths install -// content-addressed files via atomic rename, and the image cache collapses -// concurrent pulls of one digest. +// prewarm fetches every asset of one SandboxConfig into the asset cache +// and its pause image into the image cache. Racing an on-demand +// ensureSandboxAssets for the same assets is safe: the asset cache runs one +// download per asset and serves every caller from it, and the image cache +// collapses concurrent pulls of one digest. +// +// Prewarm has no actor to link the assets into, so it stages them into a +// scratch directory it removes again; the cache keeps the entries. func (p *sandboxPrewarmer) prewarm(ctx context.Context, cfg *v1alpha1.SandboxConfig) error { ctx, cancel := context.WithTimeout(ctx, prewarmTimeout) defer cancel() @@ -295,7 +300,13 @@ func (p *sandboxPrewarmer) prewarm(ctx context.Context, cfg *v1alpha1.SandboxCon assetErr = nil case assetErr == nil: wg.Go(func() { - _, assetErr = p.assets.ensureSandboxAssets(ctx, rec) + scratch, err := os.MkdirTemp(ateletpath.SandboxAssetScratchDir, "prewarm-") + if err != nil { + assetErr = fmt.Errorf("while creating prewarm scratch dir: %w", err) + return + } + defer func() { _ = os.RemoveAll(scratch) }() + _, assetErr = p.assets.ensureSandboxAssets(ctx, rec, scratch) }) } wg.Wait() diff --git a/cmd/atelet/sandbox_prewarm_test.go b/cmd/atelet/sandbox_prewarm_test.go index 90439e08b9..3d44de47b4 100644 --- a/cmd/atelet/sandbox_prewarm_test.go +++ b/cmd/atelet/sandbox_prewarm_test.go @@ -40,7 +40,6 @@ import ( "github.com/google/go-containerregistry/pkg/v1/remote" "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" "go.ate.dev/substrate/internal/imagecache" - "go.ate.dev/substrate/internal/nodepath" "go.ate.dev/substrate/pkg/api/v1alpha1" "go.ate.dev/substrate/pkg/client/clientset/versioned/fake" "go.ate.dev/substrate/pkg/client/informers/externalversions" @@ -168,9 +167,7 @@ func TestPrewarmEnqueueFilters(t *testing.T) { // prewarm is requeued with backoff, and a config deleted between enqueue and // processing is forgotten without retries. func TestPrewarmProcessRetries(t *testing.T) { - origDir := nodepath.StaticFilesDir - nodepath.StaticFilesDir = t.TempDir() - t.Cleanup(func() { nodepath.StaticFilesDir = origDir }) + useTempAssetScratch(t) ctx := context.Background() cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256([]byte("runsc")))) @@ -181,7 +178,7 @@ func TestPrewarmProcessRetries(t *testing.T) { if err := indexer.Add(cfg); err != nil { t.Fatalf("indexer.Add: %v", err) } - herder := &Atelet{anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}} + herder := &Atelet{sandboxAssetCache: newTestAssetCache(t), anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}} p := newSandboxPrewarmer(herder, nil, listersv1alpha1.NewSandboxConfigLister(indexer), false) p.queue.Add(cfg.Name) @@ -202,9 +199,7 @@ func TestPrewarmProcessRetries(t *testing.T) { // filter as gVisor may be micro-VM by the time the worker resolves it, and a // node without KVM must skip it rather than download guest assets. func TestPrewarmProcessReappliesClassGate(t *testing.T) { - origDir := nodepath.StaticFilesDir - nodepath.StaticFilesDir = t.TempDir() - t.Cleanup(func() { nodepath.StaticFilesDir = origDir }) + useTempAssetScratch(t) ctx := context.Background() // Enqueued while gVisor, edited to micro-VM before the worker ran. @@ -217,7 +212,7 @@ func TestPrewarmProcessReappliesClassGate(t *testing.T) { } // Any fetch would fail and requeue, so NumRequeues distinguishes // "skipped" from "attempted and failed". - herder := &Atelet{anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}} + herder := &Atelet{sandboxAssetCache: newTestAssetCache(t), anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}} p := newSandboxPrewarmer(herder, nil, listersv1alpha1.NewSandboxConfigLister(indexer), false) p.queue.Add(cfg.Name) @@ -248,9 +243,7 @@ func TestMicrovmNodeCapable(t *testing.T) { // lands in the image cache, and an asset fetch failure does not stop the // pull (the two live in different backends). func TestPrewarmPauseImage(t *testing.T) { - origDir := nodepath.StaticFilesDir - nodepath.StaticFilesDir = t.TempDir() - t.Cleanup(func() { nodepath.StaticFilesDir = origDir }) + useTempAssetScratch(t) srv := httptest.NewServer(registry.New(registry.Logger(log.New(io.Discard, "", 0)))) defer srv.Close() @@ -276,20 +269,20 @@ func TestPrewarmPauseImage(t *testing.T) { cfg.Spec.Versions[0].PauseImage = pauseRef p := &sandboxPrewarmer{ - assets: &Atelet{anonGCSClient: fakeObjectStorage{data: content}}, + assets: &Atelet{sandboxAssetCache: newTestAssetCache(t), anonGCSClient: fakeObjectStorage{data: content}}, images: okStore, } if err := p.prewarm(ctx, cfg); err != nil { t.Fatalf("prewarm: %v", err) } - // A different asset hash misses the shared static-files cache, so the - // failing object storage is actually consulted — and must not keep the - // pause image from being pulled. + // A different asset hash misses the asset cache, so the failing object + // storage is actually consulted — and must not keep the pause image from + // being pulled. failCfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256([]byte("other runsc")))) failCfg.Spec.Versions[0].PauseImage = pauseRef p = &sandboxPrewarmer{ - assets: &Atelet{anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}}, + assets: &Atelet{sandboxAssetCache: newTestAssetCache(t), anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}}, images: failStore, } if err := p.prewarm(ctx, failCfg); err == nil { @@ -334,14 +327,14 @@ func (hangingObjectStorage) PutObject(_ context.Context, _, _ string, _ io.Reade // prewarmTimeout: the queue has one worker, so an attempt that never returned // would block every other config's prewarm. func TestPrewarmTimeout(t *testing.T) { - origDir, origTimeout := nodepath.StaticFilesDir, prewarmTimeout - nodepath.StaticFilesDir = t.TempDir() + useTempAssetScratch(t) + origTimeout := prewarmTimeout prewarmTimeout = 50 * time.Millisecond - t.Cleanup(func() { nodepath.StaticFilesDir, prewarmTimeout = origDir, origTimeout }) + t.Cleanup(func() { prewarmTimeout = origTimeout }) cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256([]byte("hung runsc")))) cfg.Spec.Versions[0].PauseImage = "" - p := &sandboxPrewarmer{assets: &Atelet{anonGCSClient: hangingObjectStorage{}}} + p := &sandboxPrewarmer{assets: &Atelet{sandboxAssetCache: newTestAssetCache(t), anonGCSClient: hangingObjectStorage{}}} done := make(chan error, 1) go func() { done <- p.prewarm(context.Background(), cfg) }() @@ -360,9 +353,7 @@ func TestPrewarmTimeout(t *testing.T) { // whose default version is Disabled, is nothing to do rather than a retryable // failure. func TestPrewarmDefaultVersionOnly(t *testing.T) { - origDir := nodepath.StaticFilesDir - nodepath.StaticFilesDir = t.TempDir() - t.Cleanup(func() { nodepath.StaticFilesDir = origDir }) + useTempAssetScratch(t) ctx := context.Background() content := []byte("runsc binary bytes") @@ -376,17 +367,18 @@ func TestPrewarmDefaultVersionOnly(t *testing.T) { Assets: map[string]map[string]v1alpha1.AssetFile{runtime.GOARCH: {runscAssetName: {URL: "gs://bucket/other-runsc", SHA256: otherSHA}}}, }}, cfg.Spec.Versions...) - p := &sandboxPrewarmer{assets: &Atelet{anonGCSClient: fakeObjectStorage{data: content}}} + root := t.TempDir() + p := &sandboxPrewarmer{assets: &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{data: content}}} if err := p.prewarm(ctx, cfg); err != nil { t.Fatalf("prewarm: %v", err) } - if _, err := os.Stat(ateletpath.AssetPath(otherSHA)); !errors.Is(err, os.ErrNotExist) { - t.Errorf("non-default version asset was prewarmed (stat err = %v)", err) + if n := cachedAssets(t, root); n != 1 { + t.Errorf("cache holds %d assets after prewarm, want 1 (the default version's only)", n) } // Any fetch would fail, so a nil error means nothing was attempted. cfg.Spec.DefaultVersion = "missing" - p = &sandboxPrewarmer{assets: &Atelet{anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}}} + p = &sandboxPrewarmer{assets: &Atelet{sandboxAssetCache: newTestAssetCache(t), anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}}} if err := p.prewarm(ctx, cfg); err != nil { t.Errorf("prewarm of a config without its default version: %v", err) } @@ -400,10 +392,10 @@ func TestPrewarmDefaultVersionOnly(t *testing.T) { // TestSandboxAssetPrewarmDownloads runs the whole path: a SandboxConfig in a // fake clientset flows through the informer into the prewarm worker, which -// lands the asset in the static-files cache without any Run/Restore request. +// lands the asset in the asset cache without any Run/Restore request. func TestSandboxAssetPrewarmDownloads(t *testing.T) { - origDir, origJitter, origFinished := nodepath.StaticFilesDir, prewarmMaxJitter, prewarmFinished - nodepath.StaticFilesDir = t.TempDir() + useTempAssetScratch(t) + origJitter, origFinished := prewarmMaxJitter, prewarmFinished prewarmMaxJitter = 0 // The worker's own prewarm call returns only when the asset has landed // and the pause-image pull has finished: it waits for the pull on a live @@ -420,7 +412,7 @@ func TestSandboxAssetPrewarmDownloads(t *testing.T) { } } t.Cleanup(func() { - nodepath.StaticFilesDir, prewarmMaxJitter, prewarmFinished = origDir, origJitter, origFinished + prewarmMaxJitter, prewarmFinished = origJitter, origFinished }) host := imageVolumeTestRegistry(t) @@ -441,7 +433,8 @@ func TestSandboxAssetPrewarmDownloads(t *testing.T) { if err != nil { t.Fatalf("imagecache.New: %v", err) } - herder := &Atelet{anonGCSClient: fakeObjectStorage{data: content}} + root := t.TempDir() + herder := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{data: content}} // Handler first, informer start second, mirroring main: atelet startup // must never wait on this informer's sync, and the initial List replays // the pre-existing config into the handler as an Add. @@ -467,8 +460,10 @@ func TestSandboxAssetPrewarmDownloads(t *testing.T) { case <-time.After(30 * time.Second): t.Fatal("prewarm never finished") } - wantPath := ateletpath.AssetPath(sha) - if _, err := os.Stat(wantPath); err != nil { - t.Fatalf("asset not prewarmed to %s: %v", wantPath, err) + if n := cachedAssets(t, root); n != 1 { + t.Fatalf("cache holds %d assets after prewarm, want 1", n) + } + if left, err := os.ReadDir(ateletpath.SandboxAssetScratchDir); err != nil || len(left) != 0 { + t.Errorf("prewarm left %d scratch dirs behind (%v)", len(left), err) } } diff --git a/cmd/atelet/snapshot_transfer_test.go b/cmd/atelet/snapshot_transfer_test.go index e77712ebb5..73423ae490 100644 --- a/cmd/atelet/snapshot_transfer_test.go +++ b/cmd/atelet/snapshot_transfer_test.go @@ -59,6 +59,7 @@ func newPluginHerder(t *testing.T, store objectstorage.ObjectStorage) *Atelet { objectStorePlugin: objectstoreplugintest.NodeClient(plugin), snapshotScratchDir: t.TempDir(), snapshots: snapshottracker.New(t.Context(), clock.RealClock{}), + sandboxAssetCache: newTestAssetCache(t), } } diff --git a/cmd/atelet/worker_sandbox_syncer.go b/cmd/atelet/worker_sandbox_syncer.go index 0d25863ff9..1307cf0d5f 100644 --- a/cmd/atelet/worker_sandbox_syncer.go +++ b/cmd/atelet/worker_sandbox_syncer.go @@ -45,6 +45,7 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" "go.ate.dev/substrate/internal/hardware" "go.ate.dev/substrate/internal/resources" "go.ate.dev/substrate/pkg/api/v1alpha1" @@ -96,6 +97,9 @@ type workerSandboxSyncer struct { // pods holds this node's worker pods, indexed by podUIDIndex. pods cache.Indexer assets sandboxAssetFetcher + // scratchDir is where the runsc probe stages the binary it runs, read + // from ateletpath once here so a test can point a syncer elsewhere. + scratchDir string // queue carries pod UIDs. A workqueue never hands the same key to two // workers at once, so a pod's file is never written and its directory // removed at the same time. @@ -122,9 +126,10 @@ type workerSandboxSyncer struct { func newWorkerSandboxSyncer(resolver *workerSandboxResolver, pods cache.Indexer, assets sandboxAssetFetcher, workersDir string) *workerSandboxSyncer { return &workerSandboxSyncer{ - resolver: resolver, - pods: pods, - assets: assets, + resolver: resolver, + pods: pods, + assets: assets, + scratchDir: ateletpath.SandboxAssetScratchDir, // Backoff doubles from 1s for informer lag and unresolved pools, and // caps at 5m where uncapped retries for transient asset fetch or probe // failures stay until they recover. diff --git a/cmd/atelet/worker_sandbox_syncer_test.go b/cmd/atelet/worker_sandbox_syncer_test.go index d335d2e473..fc0ebfcd01 100644 --- a/cmd/atelet/worker_sandbox_syncer_test.go +++ b/cmd/atelet/worker_sandbox_syncer_test.go @@ -58,11 +58,14 @@ func testCPUFeatures() []string { } } +// fakeAssetFetcher answers ensureSandboxAssets from fn. The paths it returns +// are not under dstDir: the syncer only runs what it is handed, so where the +// fake says a binary is does not matter. type fakeAssetFetcher struct { fn func(ctx context.Context, rec *sandboxAssetsRecord) (map[string]string, error) } -func (f *fakeAssetFetcher) ensureSandboxAssets(ctx context.Context, rec *sandboxAssetsRecord) (map[string]string, error) { +func (f *fakeAssetFetcher) ensureSandboxAssets(ctx context.Context, rec *sandboxAssetsRecord, _ string) (map[string]string, error) { return f.fn(ctx, rec) } @@ -165,6 +168,7 @@ func newTestSyncer(t *testing.T, objs ...any) *workerSandboxSyncer { workerPools: listersv1alpha1.NewWorkerPoolLister(pools), configs: listersv1alpha1.NewSandboxConfigLister(configs), }, pods, newTestAssetFetcher(), t.TempDir()) + s.scratchDir = t.TempDir() // the runsc probe stages the binary there s.probeCPUIdentity = func() (string, string, error) { return testCPUVendor, testCPUModel, nil } s.runCPUFeatures = func(context.Context, string) ([]string, error) { return testCPUFeatures(), nil } // Tests that check the jitter set it. @@ -455,7 +459,7 @@ func TestWorkerSandboxSyncerProcess(t *testing.T) { if rec.Assets[gvisorAssetName].SHA256 != "sha-v2" { return nil, errors.New("temporary download failure on restorable version") } - return newTestAssetFetcher().ensureSandboxAssets(ctx, rec) + return newTestAssetFetcher().ensureSandboxAssets(ctx, rec, "") }, }, priorRetries: syncMaxRetries, @@ -761,6 +765,7 @@ func TestStartWorkerSandboxSyncer(t *testing.T) { ateFactory.Shutdown() }) + useTempAssetScratch(t) s, err := startWorkerSandboxSyncer(ctx, podFactory.Core().V1().Pods(), ateFactory.Api().V1alpha1().WorkerPools(), diff --git a/cmd/atelet/worker_sandbox_versions.go b/cmd/atelet/worker_sandbox_versions.go index 76c4c2d7fe..fefdffb667 100644 --- a/cmd/atelet/worker_sandbox_versions.go +++ b/cmd/atelet/worker_sandbox_versions.go @@ -321,7 +321,14 @@ func (s *workerSandboxSyncer) gvisorCPUFeatures(ctx context.Context, rec *sandbo // the fetch for the same reason it bounds prewarming. ctx, cancel := context.WithTimeout(ctx, prewarmTimeout) defer cancel() - paths, err := s.assets.ensureSandboxAssets(ctx, rec) + // The probe has no actor to stage the binary for: it links it into a + // scratch dir it removes again, and the cache keeps the entry. + scratch, err := os.MkdirTemp(s.scratchDir, "cpu-features-") + if err != nil { + return "", fmt.Errorf("while creating cpu-features scratch dir: %w", err) + } + defer func() { _ = os.RemoveAll(scratch) }() + paths, err := s.assets.ensureSandboxAssets(ctx, rec, scratch) if err != nil { return "", err } diff --git a/cmd/atelet/worker_sandbox_versions_test.go b/cmd/atelet/worker_sandbox_versions_test.go index ad593f8858..c2870ebc01 100644 --- a/cmd/atelet/worker_sandbox_versions_test.go +++ b/cmd/atelet/worker_sandbox_versions_test.go @@ -550,7 +550,7 @@ func TestGVisorCPUFeaturesMemoizationAndErrors(t *testing.T) { if url := rec.Assets[gvisorAssetName].URL; strings.HasPrefix(url, "gs://other-bucket/") { return nil, fmt.Errorf("fetched a duplicate runtime from %s", url) } - return newTestAssetFetcher().ensureSandboxAssets(ctx, rec) + return newTestAssetFetcher().ensureSandboxAssets(ctx, rec, "") }, } got, err := planAndBuild(s, workerPod("pod-a", "pool")) diff --git a/cmd/ateworker-gvisor/fidelity_test.go b/cmd/ateworker-gvisor/fidelity_test.go index 3a68ec4a26..d9c6fdca4b 100644 --- a/cmd/ateworker-gvisor/fidelity_test.go +++ b/cmd/ateworker-gvisor/fidelity_test.go @@ -73,6 +73,7 @@ func TestRPCsRejectRootfsBeforeTouchingTheSandbox(t *testing.T) { DurableDirVolumeMountsDir: "/node/actors/actor-a/durable-dirs", SystemInfoVolumeRootsDir: "/node/actors/actor-a/system-info", VolumesDir: "/node/actors/actor-a/volumes", + SandboxAssetsDir: "/node/actors/actor-a/sandbox-assets", } rootfs := ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS diff --git a/cmd/ateworker-gvisor/main.go b/cmd/ateworker-gvisor/main.go index 31d0ab15c8..bd57d47d18 100644 --- a/cmd/ateworker-gvisor/main.go +++ b/cmd/ateworker-gvisor/main.go @@ -559,9 +559,18 @@ func containerNames(containers []*ateworkerpb.Container) []string { } // validateActorDirs rejects a request whose actor directories are unusable. -// validateRunscPath ensures we only execute runsc from the static files dir -func validateRunscPath(p string) error { - if errs := resources.ValidateRuntimeAssetPath(nodepath.StaticFilesDir, p, field.NewPath("runsc_path")); len(errs) > 0 { +// validateRunscPath ensures we only execute runsc from the actor's own +// sandbox assets dir, where atelet staged it. The dir is derived from the +// actor UID here and the request's sandbox_assets_dir must match it, so a +// request cannot name a root of its own choosing. actorDirs must have passed +// validateActorDirs. +func validateRunscPath(actorUID string, actorDirs *ateworkerpb.ActorDirs, p string) error { + root := actorDirs.GetSandboxAssetsDir() + errs := resources.ValidateSandboxAssetsDir(actorUID, root, field.NewPath("actor_dirs", "sandbox_assets_dir")) + if len(errs) == 0 { + errs = resources.ValidateRuntimeAssetPath(root, p, field.NewPath("runsc_path")) + } + if len(errs) > 0 { return resources.ToAPIError(errs) } return nil @@ -595,7 +604,7 @@ func (s *AteWorkerService) RunWorkload(ctx context.Context, req *ateworkerpb.Run if err := validateActorDirs(req.GetActorDirs()); err != nil { return nil, err } - if err := validateRunscPath(req.GetRunscPath()); err != nil { + if err := validateRunscPath(req.GetActorUid(), req.GetActorDirs(), req.GetRunscPath()); err != nil { return nil, err } if !s.locks.Lock(ctx, req.GetActorUid()) { @@ -733,7 +742,7 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke if err := validateFidelity(req.GetFidelity()); err != nil { return nil, err } - if err := validateRunscPath(req.GetRunscPath()); err != nil { + if err := validateRunscPath(req.GetActorUid(), req.GetActorDirs(), req.GetRunscPath()); err != nil { return nil, err } if !s.locks.Lock(ctx, req.GetActorUid()) { @@ -962,7 +971,7 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb if err := validateFidelity(req.GetFidelity()); err != nil { return nil, err } - if err := validateRunscPath(req.GetRunscPath()); err != nil { + if err := validateRunscPath(req.GetActorUid(), req.GetActorDirs(), req.GetRunscPath()); err != nil { return nil, err } if !s.locks.Lock(ctx, req.GetActorUid()) { @@ -1174,7 +1183,7 @@ func (s *AteWorkerService) TerminateWorkload(ctx context.Context, req *ateworker if err := validateActorDirs(req.GetActorDirs()); err != nil { return nil, err } - if err := validateRunscPath(req.GetRunscPath()); err != nil { + if err := validateRunscPath(req.GetActorUid(), req.GetActorDirs(), req.GetRunscPath()); err != nil { return nil, err } if !s.locks.Lock(ctx, req.GetActorUid()) { diff --git a/cmd/ateworker-gvisor/main_test.go b/cmd/ateworker-gvisor/main_test.go index 5061629ca2..d7c598797a 100644 --- a/cmd/ateworker-gvisor/main_test.go +++ b/cmd/ateworker-gvisor/main_test.go @@ -78,6 +78,7 @@ func TestRPCsRejectUntrustedRunscPath(t *testing.T) { DurableDirVolumeMountsDir: "/node/actors/actor-a/durable-dirs", SystemInfoVolumeRootsDir: "/node/actors/actor-a/system-info", VolumesDir: "/node/actors/actor-a/volumes", + SandboxAssetsDir: "/node/actors/actor-a/sandbox-assets", } for name, call := range map[string]func() error{ "RunWorkload": func() error { @@ -104,10 +105,9 @@ func TestRPCsRejectUntrustedRunscPath(t *testing.T) { } func TestRestoreWorkloadRejectsUnsupportedFidelity(t *testing.T) { - staticFilesDir := nodepath.StaticFilesDir - nodepath.StaticFilesDir = t.TempDir() - t.Cleanup(func() { nodepath.StaticFilesDir = staticFilesDir }) - runscPath := filepath.Join(nodepath.StaticFilesDir, "runsc") + // runsc must sit in the actor's own sandbox assets dir. + sandboxAssetsDir := t.TempDir() + runscPath := filepath.Join(sandboxAssetsDir, "runsc") if err := os.WriteFile(runscPath, nil, 0o755); err != nil { t.Fatal(err) } @@ -119,6 +119,7 @@ func TestRestoreWorkloadRejectsUnsupportedFidelity(t *testing.T) { DurableDirVolumeMountsDir: "/node/actors/actor-a/durable-dirs", SystemInfoVolumeRootsDir: "/node/actors/actor-a/system-info", VolumesDir: "/node/actors/actor-a/volumes", + SandboxAssetsDir: sandboxAssetsDir, } // A zero service cannot acquire locks, configure networking, or open log pipes. s := &AteWorkerService{} @@ -250,3 +251,49 @@ func decodeRecord(t *testing.T, b []byte) map[string]any { } return m } + +// TestValidateRunscPathRequiresTheActorsOwnDir pins that the root the runsc +// path is checked against is derived from the actor UID, not taken from the +// request: a request cannot name a root of its own choosing. +func TestValidateRunscPathRequiresTheActorsOwnDir(t *testing.T) { + orig := nodepath.ActorsDir + nodepath.ActorsDir = t.TempDir() + t.Cleanup(func() { nodepath.ActorsDir = orig }) + stage := func(t *testing.T, actorUID string) string { + t.Helper() + dir := nodepath.ActorSandboxAssetsDir(actorUID) + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + runsc := filepath.Join(dir, "runsc") + if err := os.WriteFile(runsc, nil, 0o555); err != nil { + t.Fatal(err) + } + return runsc + } + own, other := stage(t, "actor-a"), stage(t, "actor-b") + dirs := func(sandboxAssetsDir string) *ateworkerpb.ActorDirs { + return &ateworkerpb.ActorDirs{SandboxAssetsDir: sandboxAssetsDir} + } + for _, tc := range []struct { + name string + dir string + runsc string + wantErr bool + }{ + {"runsc in the actor's own dir", filepath.Dir(own), own, false}, + {"runsc outside the actor's dir", filepath.Dir(own), "/bin/sh", true}, + {"another actor's dir named as the root", filepath.Dir(other), other, true}, + {"the root named as the root", "/", "/bin/sh", true}, + } { + t.Run(tc.name, func(t *testing.T) { + err := validateRunscPath("actor-a", dirs(tc.dir), tc.runsc) + if (err != nil) != tc.wantErr { + t.Errorf("validateRunscPath(actor-a, %q, %q) = %v, wantErr %v", tc.dir, tc.runsc, err, tc.wantErr) + } + if err != nil && apierror.Code(err) != codes.InvalidArgument { + t.Errorf("code = %v, want %v", apierror.Code(err), codes.InvalidArgument) + } + }) + } +} diff --git a/cmd/ateworker-gvisor/runsc_test.go b/cmd/ateworker-gvisor/runsc_test.go index 553c7de701..a263fbab1a 100644 --- a/cmd/ateworker-gvisor/runsc_test.go +++ b/cmd/ateworker-gvisor/runsc_test.go @@ -310,6 +310,7 @@ func TestRestoreWorkloadRejectsEmptyRestoreDir(t *testing.T) { DurableDirVolumeMountsDir: "/node/actors/actor-a/durable-dirs", SystemInfoVolumeRootsDir: "/node/actors/actor-a/system-info", VolumesDir: "/node/actors/actor-a/volumes", + SandboxAssetsDir: "/node/actors/actor-a/sandbox-assets", }, }, }, diff --git a/cmd/ateworker-microvm/internal/kata/cleanup_linux.go b/cmd/ateworker-microvm/internal/kata/cleanup_linux.go index 0c2634275c..03e1e4d12b 100644 --- a/cmd/ateworker-microvm/internal/kata/cleanup_linux.go +++ b/cmd/ateworker-microvm/internal/kata/cleanup_linux.go @@ -128,14 +128,16 @@ func CleanupSandboxState(ctx context.Context, id string) { // isSandboxProcess reports whether a process is a VMM or virtiofsd serving the // sandbox id. Both carry id in their socket paths. atelet stages their binaries -// under content-addressed names, so the executable's location identifies them -// rather than its name. +// under the actor's own directory, and the sandbox id is the actor UID, so an +// executable under an actor's directory settles it: this sandbox's if it is +// id's directory, another sandbox's otherwise, whatever the command line +// says. Only a binary from elsewhere is judged by its name. func isSandboxProcess(id, cmdline, exe string) bool { if !strings.Contains(cmdline, id) { return false } - if strings.HasPrefix(exe, nodepath.StaticFilesDir+"/") { - return true + if strings.HasPrefix(exe, nodepath.ActorsDir+"/") { + return strings.HasPrefix(exe, nodepath.ActorSandboxAssetsDir(id)+"/") } argv0 := strings.SplitN(cmdline, "\x00", 2)[0] return strings.Contains(argv0, "cloud-hypervisor") || strings.Contains(argv0, "virtiofsd") diff --git a/cmd/ateworker-microvm/internal/kata/cleanup_linux_test.go b/cmd/ateworker-microvm/internal/kata/cleanup_linux_test.go index 8180307ce8..0ec05da040 100644 --- a/cmd/ateworker-microvm/internal/kata/cleanup_linux_test.go +++ b/cmd/ateworker-microvm/internal/kata/cleanup_linux_test.go @@ -25,7 +25,10 @@ import ( func TestIsSandboxProcess(t *testing.T) { const id = "0e16c0fa-5c06-4b48-8742-4068635c08c1" - staged := nodepath.StaticFilesDir + "/runsc-ed81e9b2" + // atelet stages each actor's VMM under that actor's own directory. + staged := nodepath.ActorsDir + "/" + id + "/sandbox-assets/cloud-hypervisor" + const other = "7d3b1a9c-2e4f-4a6b-9c8d-0f1e2d3c4b5a" + otherStaged := nodepath.ActorsDir + "/" + other + "/sandbox-assets/cloud-hypervisor" argv := func(args ...string) string { return strings.Join(args, "\x00") + "\x00" } for _, tc := range []struct { name string @@ -36,7 +39,8 @@ func TestIsSandboxProcess(t *testing.T) { {"staged VMM", argv(staged, "--api-socket", "/run/vc/vm/"+id+"/clh-api.sock"), staged, true}, {"staged virtiofsd", argv(staged, "--socket-path=/run/vc/vm/"+id+"/virtiofsd.sock"), staged, true}, {"VMM by name", argv("cloud-hypervisor", "--api-socket", "/run/vc/vm/"+id+"/clh-api.sock"), "/usr/bin/cloud-hypervisor", true}, - {"another sandbox", argv(staged, "--api-socket", "/run/vc/vm/other/clh-api.sock"), staged, false}, + {"another sandbox", argv(otherStaged, "--api-socket", "/run/vc/vm/"+other+"/clh-api.sock"), otherStaged, false}, + {"another sandbox's VMM naming the id", argv(otherStaged, "--api-socket", "/run/vc/vm/"+other+"/clh-api.sock", "--log", "/tmp/"+id), otherStaged, false}, {"unrelated process naming the id", argv("/bin/sh", "-c", "echo "+id), "/bin/busybox", false}, } { t.Run(tc.name, func(t *testing.T) { diff --git a/cmd/ateworker-microvm/main.go b/cmd/ateworker-microvm/main.go index 6f1a34ed6c..df3131257d 100644 --- a/cmd/ateworker-microvm/main.go +++ b/cmd/ateworker-microvm/main.go @@ -439,11 +439,19 @@ func validateFidelity(fidelity ateworkerpb.SnapshotFidelity) error { return nil } -// validateRuntimeAssetPaths ensures we only run assets from the static files dir -func validateRuntimeAssetPaths(paths map[string]string) error { +// validateRuntimeAssetPaths ensures we only run assets from the actor's own +// sandbox assets dir, where atelet staged them. The dir is derived from the +// actor UID here and the request's sandbox_assets_dir must match it, so a +// request cannot name a root of its own choosing. actorDirs must have passed +// validateActorDirs. +func validateRuntimeAssetPaths(actorUID string, actorDirs *ateworkerpb.ActorDirs, paths map[string]string) error { + root := actorDirs.GetSandboxAssetsDir() + if errs := resources.ValidateSandboxAssetsDir(actorUID, root, field.NewPath("actor_dirs", "sandbox_assets_dir")); len(errs) > 0 { + return resources.ToAPIError(errs) + } var errs field.ErrorList for name, p := range paths { - errs = append(errs, resources.ValidateRuntimeAssetPath(nodepath.StaticFilesDir, p, field.NewPath("runtime_asset_paths").Key(name))...) + errs = append(errs, resources.ValidateRuntimeAssetPath(root, p, field.NewPath("runtime_asset_paths").Key(name))...) } if len(errs) > 0 { return resources.ToAPIError(errs) diff --git a/cmd/ateworker-microvm/restore.go b/cmd/ateworker-microvm/restore.go index efa86e976e..b7c7ff1032 100644 --- a/cmd/ateworker-microvm/restore.go +++ b/cmd/ateworker-microvm/restore.go @@ -111,7 +111,7 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb if err := validateFidelity(req.GetFidelity()); err != nil { return nil, err } - if err := validateRuntimeAssetPaths(req.GetRuntimeAssetPaths()); err != nil { + if err := validateRuntimeAssetPaths(req.GetActorUid(), req.GetActorDirs(), req.GetRuntimeAssetPaths()); err != nil { return nil, err } if !s.locks.Lock(ctx, req.GetActorUid()) { @@ -234,6 +234,12 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb // caller from their tar. func (s *AteWorkerService) restoreMemoryFidelity(ctx context.Context, p actorBootParams, scope ateworkerpb.SnapshotFidelity, restoreDir string, preserveRestoreDir bool, tStart time.Time) (retErr error) { actorUID := p.actorUID + // Checked before anything is prepared or cleaned up: the snapshot's + // kernel and guest image paths are repointed at these below. + kernel, image := p.assetPaths[assetKernel], p.assetPaths[assetImage] + if kernel == "" || image == "" { + return fmt.Errorf("ateworker-microvm requires %q and %q asset paths", assetKernel, assetImage) + } rr := s.resolveRuntime(p.assetPaths) egress, err := s.tunnel.PrepareEgress(ctx, p.attribution(), p.egressGateway) @@ -242,10 +248,11 @@ func (s *AteWorkerService) restoreMemoryFidelity(ctx context.Context, p actorBoo } s.cleanupSandboxState(ctx, actorUID) - // Repoint the snapshot's vsock socket to this actor's VMDir (the disk + kernel - // paths are content-addressed/per-actor and already line up on the same node). - if err := rewriteSnapshotSocketPaths(restoreDir, actorUID); err != nil { - return fmt.Errorf("while rewriting snapshot socket paths: %w", err) + // Repoint the snapshot's per-actor paths to this actor's: its VMDir + // sockets and files, and the kernel and guest image atelet staged under + // this actor's own directory. + if err := rewriteSnapshotPaths(restoreDir, actorUID, kernel, image); err != nil { + return fmt.Errorf("while rewriting snapshot paths: %w", err) } srcID := actorUID if b, rerr := os.ReadFile(filepath.Join(restoreDir, baseIDFile)); rerr == nil { @@ -281,7 +288,7 @@ func (s *AteWorkerService) restoreMemoryFidelity(ctx context.Context, p actorBoo // SharedDir(id)//rootfs from the LOCAL OCI bundle (atelet re-unpacked // the golden image) and start the one virtiofsd serving the tree. The fs // sockets in the snapshot config are repointed to this VMDir by - // rewriteSnapshotSocketPaths above. Cross-node consistency relies on a + // rewriteSnapshotPaths above. Cross-node consistency relies on a // deterministic unpack of the same image at the same /rootfs path // (plus, for merged rootfs, the upper re-materialized from the tar). containers := p.containers @@ -538,13 +545,17 @@ func maybeDropStagedMemoryImage(ctx context.Context, restoreDir, memMode string, dropStagedMemoryImage(ctx, restoreDir) } -// rewriteSnapshotSocketPaths repoints the snapshot config.json's per-VMDir paths from -// the source actor's VMDir to the restoring actor's: the hybrid-vsock socket, the -// File serial console, and each virtio-fs socket, so the sockets/files we create are -// the ones CH reopens. The kernel and the kata image are content-addressed static -// files with identical paths on every node, so they need no rewrite, and the overlay -// has no per-actor disk to repoint. -func rewriteSnapshotSocketPaths(snapshotDir, id string) error { +// rewriteSnapshotPaths repoints the snapshot config.json's per-actor paths from +// the source actor's to the restoring actor's. The file is CH's own: at +// snapshot time it serializes the config the VM booted with, and at restore +// it reopens every path in it, so each path must exist on this node for +// this actor. That covers the hybrid-vsock socket, the File serial console, +// and each virtio-fs socket under the VMDir, and the kernel and the guest +// image, which atelet stages under each actor's own directory and removes +// with the actor, so the source actor's copies may be gone. How atelet +// identifies the runtime (asset URLs today, a SandboxConfig hash later) +// changes none of this. The overlay has no per-actor disk to repoint. +func rewriteSnapshotPaths(snapshotDir, id, kernel, image string) error { cfgPath := filepath.Join(snapshotDir, "config.json") b, err := os.ReadFile(cfgPath) if err != nil { @@ -602,6 +613,39 @@ func rewriteSnapshotSocketPaths(snapshotDir, id string) error { } } } + // Every VM this worker boots has a payload with the kernel and a pmem + // device with the guest image (see buildVMConfig), and both paths are + // per-actor, so a snapshot config without them is broken: skipping the + // rewrite would hand CH the source actor's paths. + payload, ok := cfg["payload"].(map[string]any) + if !ok { + return fmt.Errorf("snapshot config %q has no payload", cfgPath) + } + if got, _ := payload["kernel"].(string); got != kernel { + payload["kernel"] = kernel + changed = true + } + pmems, _ := cfg["pmem"].([]any) + guestImage := false + for _, pm := range pmems { + pmm, ok := pm.(map[string]any) + if !ok { + return fmt.Errorf("snapshot config %q has a malformed pmem device", cfgPath) + } + switch pmID, _ := pmm["id"].(string); pmID { + case guestImagePmemID: + guestImage = true + if got, _ := pmm["file"].(string); got != image { + pmm["file"] = image + changed = true + } + default: + return fmt.Errorf("snapshot config %q has pmem device with unknown id %q", cfgPath, pmID) + } + } + if !guestImage { + return fmt.Errorf("snapshot config %q has no %q pmem device", cfgPath, guestImagePmemID) + } if !changed { // Same-actor resume: nothing to rewrite, so leave a preserved snapshot untouched. return nil diff --git a/cmd/ateworker-microvm/restore_test.go b/cmd/ateworker-microvm/restore_test.go index b95ddd2384..8d6fd10ca3 100644 --- a/cmd/ateworker-microvm/restore_test.go +++ b/cmd/ateworker-microvm/restore_test.go @@ -22,8 +22,10 @@ import ( "encoding/json" "os" "path/filepath" + "strings" "syscall" "testing" + "time" "google.golang.org/grpc/codes" @@ -31,6 +33,7 @@ import ( "go.ate.dev/substrate/cmd/ateworker-microvm/internal/kata" "go.ate.dev/substrate/internal/actorlock" "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/nodepath" "go.ate.dev/substrate/internal/proto/ateworkerpb" ) @@ -46,6 +49,8 @@ func writeSnapshotConfig(t *testing.T, fsDevices []map[string]any) string { "console": map[string]any{"mode": "File", "file": "/run/vc/vm/golden/console.log"}, "serial": map[string]any{"mode": "File", "file": "/run/vc/vm/golden/serial.log"}, "fs": fsDevices, + "payload": map[string]any{"kernel": "/var/lib/ate/actors/golden/sandbox-assets/kata-kernel", "cmdline": "console=hvc0"}, + "pmem": []map[string]any{{"id": guestImagePmemID, "file": "/var/lib/ate/actors/golden/sandbox-assets/kata-image", "discard_writes": true}}, } b, err := json.Marshal(cfg) if err != nil { @@ -57,6 +62,28 @@ func writeSnapshotConfig(t *testing.T, fsDevices []map[string]any) string { return dir } +// mutateSnapshotConfig rewrites dir's config.json through mutate. +func mutateSnapshotConfig(t *testing.T, dir string, mutate func(cfg map[string]any)) { + t.Helper() + cfgPath := filepath.Join(dir, "config.json") + b, err := os.ReadFile(cfgPath) + if err != nil { + t.Fatal(err) + } + var cfg map[string]any + if err := json.Unmarshal(b, &cfg); err != nil { + t.Fatal(err) + } + mutate(cfg) + b, err = json.Marshal(cfg) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(cfgPath, b, 0o600); err != nil { + t.Fatal(err) + } +} + // readFsSockets returns the rewritten config's fs sockets keyed by device tag. func readFsSockets(t *testing.T, dir string) map[string]string { t.Helper() @@ -82,15 +109,20 @@ func readFsSockets(t *testing.T, dir string) map[string]string { return got } -func TestRewriteSnapshotSocketPaths(t *testing.T) { +func TestRewriteSnapshotPaths(t *testing.T) { const id = "actor-uid" + // The restoring actor's own staged kernel and guest image. + const ( + kernel = "/var/lib/ate/actors/actor-uid/sandbox-assets/kata-kernel" + image = "/var/lib/ate/actors/actor-uid/sandbox-assets/kata-image" + ) t.Run("overlay lower only", func(t *testing.T) { dir := writeSnapshotConfig(t, []map[string]any{ {"tag": kata.FsTag, "socket": "/run/vc/vm/golden/virtiofsd.sock"}, }) - if err := rewriteSnapshotSocketPaths(dir, id); err != nil { - t.Fatalf("rewriteSnapshotSocketPaths: %v", err) + if err := rewriteSnapshotPaths(dir, id, kernel, image); err != nil { + t.Fatalf("rewriteSnapshotPaths: %v", err) } if got, want := readFsSockets(t, dir)[kata.FsTag], kata.VirtiofsdSocketPath(id); got != want { t.Errorf("%s socket = %q, want %q", kata.FsTag, got, want) @@ -105,18 +137,65 @@ func TestRewriteSnapshotSocketPaths(t *testing.T) { dir := writeSnapshotConfig(t, []map[string]any{ {"tag": tag, "socket": "/run/vc/vm/golden/other.sock"}, }) - if err := rewriteSnapshotSocketPaths(dir, id); err == nil { - t.Fatalf("rewriteSnapshotSocketPaths accepted fs tag %q, want an error", tag) + if err := rewriteSnapshotPaths(dir, id, kernel, image); err == nil { + t.Fatalf("rewriteSnapshotPaths accepted fs tag %q, want an error", tag) + } + } + }) + + // A snapshot config that lacks or mangles the two per-actor boot paths + // is broken: skipping their rewrite would restore with the source + // actor's, so each shape is an error. + for name, mutate := range map[string]func(cfg map[string]any){ + "missing payload": func(cfg map[string]any) { delete(cfg, "payload") }, + "malformed payload": func(cfg map[string]any) { cfg["payload"] = "vmlinux" }, + "malformed pmem entry": func(cfg map[string]any) { cfg["pmem"] = []any{"not a device"} }, + "missing pmem list": func(cfg map[string]any) { delete(cfg, "pmem") }, + "missing guest image device": func(cfg map[string]any) { cfg["pmem"] = []any{} }, + } { + t.Run(name+" is an error", func(t *testing.T) { + dir := writeSnapshotConfig(t, []map[string]any{ + {"tag": kata.FsTag, "socket": "/run/vc/vm/golden/virtiofsd.sock"}, + }) + mutateSnapshotConfig(t, dir, mutate) + if err := rewriteSnapshotPaths(dir, id, kernel, image); err == nil { + t.Fatalf("rewriteSnapshotPaths accepted a config with %s, want an error", name) } + }) + } + + t.Run("unknown pmem device is an error", func(t *testing.T) { + dir := writeSnapshotConfig(t, []map[string]any{ + {"tag": kata.FsTag, "socket": "/run/vc/vm/golden/virtiofsd.sock"}, + }) + cfgPath := filepath.Join(dir, "config.json") + b, err := os.ReadFile(cfgPath) + if err != nil { + t.Fatal(err) + } + var cfg map[string]any + if err := json.Unmarshal(b, &cfg); err != nil { + t.Fatal(err) + } + cfg["pmem"] = []map[string]any{{"id": "scratch-disk", "file": "/somewhere/else"}} + b, err = json.Marshal(cfg) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(cfgPath, b, 0o600); err != nil { + t.Fatal(err) + } + if err := rewriteSnapshotPaths(dir, id, kernel, image); err == nil { + t.Fatal("rewriteSnapshotPaths accepted a pmem device it does not know, want an error") } }) - t.Run("vsock and console devices are repointed", func(t *testing.T) { + t.Run("vsock, console, kernel and image are repointed", func(t *testing.T) { dir := writeSnapshotConfig(t, []map[string]any{ {"tag": kata.FsTag, "socket": "/run/vc/vm/golden/virtiofsd.sock"}, }) - if err := rewriteSnapshotSocketPaths(dir, id); err != nil { - t.Fatalf("rewriteSnapshotSocketPaths: %v", err) + if err := rewriteSnapshotPaths(dir, id, kernel, image); err != nil { + t.Fatalf("rewriteSnapshotPaths: %v", err) } b, err := os.ReadFile(filepath.Join(dir, "config.json")) if err != nil { @@ -126,6 +205,11 @@ func TestRewriteSnapshotSocketPaths(t *testing.T) { Vsock struct{ Socket string } `json:"vsock"` Console struct{ File string } `json:"console"` Serial struct{ File string } `json:"serial"` + Payload struct{ Kernel string } `json:"payload"` + Pmem []struct { + ID string `json:"id"` + File string `json:"file"` + } `json:"pmem"` } if err := json.Unmarshal(b, &cfg); err != nil { t.Fatalf("parsing rewritten config: %v", err) @@ -133,6 +217,14 @@ func TestRewriteSnapshotSocketPaths(t *testing.T) { if want := kata.VsockSocketPath(id); cfg.Vsock.Socket != want { t.Errorf("vsock socket = %q, want %q", cfg.Vsock.Socket, want) } + // The source actor's staged kernel and image may be gone; the + // snapshot must boot from this actor's. + if cfg.Payload.Kernel != kernel { + t.Errorf("payload kernel = %q, want %q", cfg.Payload.Kernel, kernel) + } + if len(cfg.Pmem) != 1 || cfg.Pmem[0].ID != guestImagePmemID || cfg.Pmem[0].File != image { + t.Errorf("pmem = %+v, want the %q device at %q", cfg.Pmem, guestImagePmemID, image) + } if want := kata.ConsoleLogPath(id); cfg.Console.File != want { t.Errorf("console file = %q, want %q", cfg.Console.File, want) } @@ -158,8 +250,8 @@ func TestRewriteSnapshotSocketPaths(t *testing.T) { t.Fatalf("linking config.json: %v", err) } - if err := rewriteSnapshotSocketPaths(dir, id); err != nil { - t.Fatalf("rewriteSnapshotSocketPaths: %v", err) + if err := rewriteSnapshotPaths(dir, id, kernel, image); err != nil { + t.Fatalf("rewriteSnapshotPaths: %v", err) } if got, want := readFsSockets(t, dir)[kata.FsTag], kata.VirtiofsdSocketPath(id); got != want { t.Errorf("%s socket = %q, want %q", kata.FsTag, got, want) @@ -193,6 +285,8 @@ func TestRewriteSnapshotSocketPaths(t *testing.T) { "fs": []map[string]any{ {"tag": kata.FsTag, "socket": kata.VirtiofsdSocketPath(id)}, }, + "payload": map[string]any{"kernel": kernel, "cmdline": "console=hvc0"}, + "pmem": []map[string]any{{"id": guestImagePmemID, "file": image, "discard_writes": true}}, } b, err := json.Marshal(cfg) if err != nil { @@ -207,8 +301,8 @@ func TestRewriteSnapshotSocketPaths(t *testing.T) { } inoBefore := fiBefore.Sys().(*syscall.Stat_t).Ino - if err := rewriteSnapshotSocketPaths(dir, id); err != nil { - t.Fatalf("rewriteSnapshotSocketPaths: %v", err) + if err := rewriteSnapshotPaths(dir, id, kernel, image); err != nil { + t.Fatalf("rewriteSnapshotPaths: %v", err) } fiAfter, err := os.Stat(cfgPath) @@ -247,6 +341,7 @@ func TestRestoreWorkloadRejectsEmptyRestoreDir(t *testing.T) { DurableDirVolumeMountsDir: "/node/actors/actor-a/durable-dirs", SystemInfoVolumeRootsDir: "/node/actors/actor-a/system-info", VolumesDir: "/node/actors/actor-a/volumes", + SandboxAssetsDir: "/node/actors/actor-a/sandbox-assets", }, }, }, @@ -418,6 +513,7 @@ func TestRPCsRejectUntrustedRuntimeAssetPaths(t *testing.T) { DurableDirVolumeMountsDir: "/node/actors/actor-a/durable-dirs", SystemInfoVolumeRootsDir: "/node/actors/actor-a/system-info", VolumesDir: "/node/actors/actor-a/volumes", + SandboxAssetsDir: "/node/actors/actor-a/sandbox-assets", } assets := map[string]string{assetCH: "/bin/sh"} for name, call := range map[string]func() error{ @@ -435,3 +531,66 @@ func TestRPCsRejectUntrustedRuntimeAssetPaths(t *testing.T) { } } } + +// TestValidateRuntimeAssetPathsRequiresTheActorsOwnDir pins that the root +// the asset paths are checked against is derived from the actor UID, not +// taken from the request. +func TestValidateRuntimeAssetPathsRequiresTheActorsOwnDir(t *testing.T) { + orig := nodepath.ActorsDir + nodepath.ActorsDir = t.TempDir() + t.Cleanup(func() { nodepath.ActorsDir = orig }) + stage := func(t *testing.T, actorUID string) string { + t.Helper() + dir := nodepath.ActorSandboxAssetsDir(actorUID) + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, assetCH), nil, 0o555); err != nil { + t.Fatal(err) + } + return dir + } + own, other := stage(t, "actor-a"), stage(t, "actor-b") + for _, tc := range []struct { + name string + dir string + asset string + wantErr bool + }{ + {"asset in the actor's own dir", own, filepath.Join(own, assetCH), false}, + {"asset outside the actor's dir", own, "/bin/sh", true}, + {"another actor's dir named as the root", other, filepath.Join(other, assetCH), true}, + {"the root named as the root", "/", "/bin/sh", true}, + } { + t.Run(tc.name, func(t *testing.T) { + err := validateRuntimeAssetPaths("actor-a", &ateworkerpb.ActorDirs{SandboxAssetsDir: tc.dir}, map[string]string{assetCH: tc.asset}) + if (err != nil) != tc.wantErr { + t.Errorf("validateRuntimeAssetPaths(actor-a, %q, %q) = %v, wantErr %v", tc.dir, tc.asset, err, tc.wantErr) + } + }) + } +} + +// TestRestoreMemoryFidelityRequiresKernelAndImage pins that a memory restore +// without the kernel or the guest image among its asset paths fails before +// it prepares or cleans up anything: the snapshot's boot paths are repointed +// at them, so there is nothing to restore with. +func TestRestoreMemoryFidelityRequiresKernelAndImage(t *testing.T) { + // A zero service cannot prepare egress or clean up sandbox state; the + // check must come first for this test to return at all. + s := &AteWorkerService{} + for name, paths := range map[string]map[string]string{ + "no assets": nil, + "no kernel": {assetImage: "/node/actors/actor-a/sandbox-assets/kata-image"}, + "no image": {assetKernel: "/node/actors/actor-a/sandbox-assets/kata-kernel"}, + "empty path": {assetKernel: "", assetImage: "/node/actors/actor-a/sandbox-assets/kata-image"}, + } { + t.Run(name, func(t *testing.T) { + p := actorBootParams{actorUID: "actor-a", assetPaths: paths} + err := s.restoreMemoryFidelity(t.Context(), p, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, t.TempDir(), false, time.Now()) + if err == nil || !strings.Contains(err.Error(), assetKernel) || !strings.Contains(err.Error(), assetImage) { + t.Fatalf("restoreMemoryFidelity with %s = %v, want an error naming %q and %q", name, err, assetKernel, assetImage) + } + }) + } +} diff --git a/cmd/ateworker-microvm/run.go b/cmd/ateworker-microvm/run.go index 55748ff804..8d2d8d0460 100644 --- a/cmd/ateworker-microvm/run.go +++ b/cmd/ateworker-microvm/run.go @@ -119,6 +119,10 @@ const ( assetKernel = "kata-kernel" assetImage = "kata-image" assetVirtiofsd = "virtiofsd" + + // guestImagePmemID names the virtio-pmem device that carries the guest + // image, in the VM config and so in every snapshot's config.json. + guestImagePmemID = "guest-image" ) // kataAgentPath is where the kata guest image keeps the agent binary. buildVMConfig @@ -235,7 +239,7 @@ func (s *AteWorkerService) RunWorkload(ctx context.Context, req *ateworkerpb.Run if err := validateActorDirs(req.GetActorDirs()); err != nil { return nil, err } - if err := validateRuntimeAssetPaths(req.GetRuntimeAssetPaths()); err != nil { + if err := validateRuntimeAssetPaths(req.GetActorUid(), req.GetActorDirs(), req.GetRuntimeAssetPaths()); err != nil { return nil, err } if !s.locks.Lock(ctx, req.GetActorUid()) { @@ -818,7 +822,7 @@ func buildVMConfig(id, kernel, image, kparams, consoleLog string, memMiB, vcpus Payload: ch.PayloadConfig{Kernel: kernel, Cmdline: cmdline}, // The guest image is a virtio-pmem device (/dev/pmem0), mounted read-only // with DAX, so its file data stays in the host page cache instead of being copied into guest RAM. - Pmem: []ch.PmemConfig{{ID: "guest-image", File: image, DiscardWrites: true}}, + Pmem: []ch.PmemConfig{{ID: guestImagePmemID, File: image, DiscardWrites: true}}, Fs: buildFsConfigs(id), Platform: &ch.PlatformConfig{NumPciSegments: 2}, Rng: &ch.RngConfig{Src: "/dev/urandom"}, diff --git a/internal/nodepath/nodepath.go b/internal/nodepath/nodepath.go index 2eb3431cbd..7a4d563f85 100644 --- a/internal/nodepath/nodepath.go +++ b/internal/nodepath/nodepath.go @@ -27,8 +27,13 @@ const BasePath = "/var/lib/ate" // ActorsDir is the parent of the per-actor directories atelet prepares. var ActorsDir = filepath.Join(BasePath, "actors") -// StaticFilesDir holds things like runsc binaries. -var StaticFilesDir = filepath.Join(BasePath, "static-files") +// ActorSandboxAssetsDir is where atelet stages an actor's sandbox runtime +// assets, hard-linked from its asset cache, and the only location a worker +// runs a sandbox binary from. Both sides derive it from the actor UID, so a +// worker can check that the directory a request names is the actor's own. +func ActorSandboxAssetsDir(actorUID string) string { + return filepath.Join(ActorsDir, actorUID, "sandbox-assets") +} // WorkerSupportSocket is the node-local atelet socket used by atunnel // to request credentials for the worker's current actor assignment. diff --git a/internal/proto/ateworkerpb/ateworker.pb.go b/internal/proto/ateworkerpb/ateworker.pb.go index 5d1331f7d4..3cf28e003c 100644 --- a/internal/proto/ateworkerpb/ateworker.pb.go +++ b/internal/proto/ateworkerpb/ateworker.pb.go @@ -176,9 +176,15 @@ type ActorDirs struct { // Holds one subdirectory per system-info volume. SystemInfoVolumeRootsDir string `protobuf:"bytes,6,opt,name=system_info_volume_roots_dir,json=systemInfoVolumeRootsDir,proto3" json:"system_info_volume_roots_dir,omitempty"` // Holds one mount point per CSI volume. - VolumesDir string `protobuf:"bytes,7,opt,name=volumes_dir,json=volumesDir,proto3" json:"volumes_dir,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + VolumesDir string `protobuf:"bytes,7,opt,name=volumes_dir,json=volumesDir,proto3" json:"volumes_dir,omitempty"` + // Holds the sandbox runtime assets staged for this actor (runsc, or the + // micro-VM kernel, VMM and guest image), hard-linked from atelet's asset + // cache. The only location a sandbox runtime may run a binary from; the + // worker derives the same path from actor_uid and rejects a request + // naming any other. + SandboxAssetsDir string `protobuf:"bytes,8,opt,name=sandbox_assets_dir,json=sandboxAssetsDir,proto3" json:"sandbox_assets_dir,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ActorDirs) Reset() { @@ -260,6 +266,13 @@ func (x *ActorDirs) GetVolumesDir() string { return "" } +func (x *ActorDirs) GetSandboxAssetsDir() string { + if x != nil { + return x.SandboxAssetsDir + } + return "" +} + type TerminateWorkloadRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Atespace string `protobuf:"bytes,1,opt,name=atespace,proto3" json:"atespace,omitempty"` @@ -1986,7 +1999,7 @@ var File_ateworker_proto protoreflect.FileDescriptor const file_ateworker_proto_rawDesc = "" + "\n" + - "\x0fateworker.proto\x12\tateworker\"\xb7\x02\n" + + "\x0fateworker.proto\x12\tateworker\"\xe5\x02\n" + "\tActorDirs\x12\x19\n" + "\broot_dir\x18\x01 \x01(\tR\arootDir\x12$\n" + "\x0eoci_bundle_dir\x18\x02 \x01(\tR\fociBundleDir\x12%\n" + @@ -1996,7 +2009,8 @@ const file_ateworker_proto_rawDesc = "" + "\x1ddurable_dir_volume_mounts_dir\x18\x05 \x01(\tR\x19durableDirVolumeMountsDir\x12>\n" + "\x1csystem_info_volume_roots_dir\x18\x06 \x01(\tR\x18systemInfoVolumeRootsDir\x12\x1f\n" + "\vvolumes_dir\x18\a \x01(\tR\n" + - "volumesDir\"\xdb\x02\n" + + "volumesDir\x12,\n" + + "\x12sandbox_assets_dir\x18\b \x01(\tR\x10sandboxAssetsDir\"\xdb\x02\n" + "\x18TerminateWorkloadRequest\x12\x1a\n" + "\batespace\x18\x01 \x01(\tR\batespace\x12\x1d\n" + "\n" + diff --git a/internal/proto/ateworkerpb/ateworker.proto b/internal/proto/ateworkerpb/ateworker.proto index 8a379e5c62..8fb4825449 100644 --- a/internal/proto/ateworkerpb/ateworker.proto +++ b/internal/proto/ateworkerpb/ateworker.proto @@ -124,6 +124,12 @@ message ActorDirs { string system_info_volume_roots_dir = 6; // Holds one mount point per CSI volume. string volumes_dir = 7; + // Holds the sandbox runtime assets staged for this actor (runsc, or the + // micro-VM kernel, VMM and guest image), hard-linked from atelet's asset + // cache. The only location a sandbox runtime may run a binary from; the + // worker derives the same path from actor_uid and rejects a request + // naming any other. + string sandbox_assets_dir = 8; } message TerminateWorkloadRequest { diff --git a/internal/resources/validate.go b/internal/resources/validate.go index 7376fb6b1b..aeefd1215c 100644 --- a/internal/resources/validate.go +++ b/internal/resources/validate.go @@ -29,6 +29,7 @@ import ( "github.com/distribution/reference" "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/nodepath" "go.ate.dev/substrate/internal/proto/ateworkerpb" "go.ate.dev/substrate/internal/volumepath" "go.ate.dev/substrate/pkg/sandboxclass" @@ -121,6 +122,7 @@ func ValidateActorDirs(actorDirs *ateworkerpb.ActorDirs, fldPath *field.Path) fi {"durable_dir_volume_mounts_dir", actorDirs.GetDurableDirVolumeMountsDir()}, {"system_info_volume_roots_dir", actorDirs.GetSystemInfoVolumeRootsDir()}, {"volumes_dir", actorDirs.GetVolumesDir()}, + {"sandbox_assets_dir", actorDirs.GetSandboxAssetsDir()}, } { errs = append(errs, validateAbsDir(actorDir.path, fldPath.Child(actorDir.name))...) } @@ -189,12 +191,24 @@ func ValidateRuntimeAssetPath(root, p string, fldPath *field.Path) field.ErrorLi return nil } +// ValidateSandboxAssetsDir requires dir, the sandbox_assets_dir a request +// names, to be the actor's own, nodepath.ActorSandboxAssetsDir(actorUID), so +// a request cannot point a worker at binaries staged for another actor or +// anywhere else on the node. The actor UID is checked first: it becomes a +// path element, and must be a plain label rather than a traversal. +func ValidateSandboxAssetsDir(actorUID, dir string, fldPath *field.Path) field.ErrorList { + if errs := content.IsDNS1123Label(actorUID); len(errs) > 0 { + return field.ErrorList{field.Invalid(fldPath, dir, fmt.Sprintf("actor_uid %q is not a valid label: %s", actorUID, strings.Join(errs, "; ")))} + } + if want := nodepath.ActorSandboxAssetsDir(actorUID); dir != want { + return field.ErrorList{field.Invalid(fldPath, dir, fmt.Sprintf("must be the actor's own sandbox assets dir %s", want))} + } + return nil +} + // ValidateAssetHash ensures a sandbox asset's SHA-256 hash is exactly 64 hex -// characters before it is used to build the on-disk asset path -// (static-files/asset-) and, on a cache hit, returned for worker to -// execute. Without this, a hash containing path separators or ".." could -// point the cache-hit early return (and the download target) at an arbitrary -// binary outside the static-files dir. +// characters before it becomes the asset's cache key, so a malformed hash is +// rejected as the request's fault rather than failing deep inside the cache. func ValidateAssetHash(sha256Hash string) error { if len(sha256Hash) != 64 { return fmt.Errorf("invalid asset sha256 hash: want 64 hex chars, got %d", len(sha256Hash)) diff --git a/internal/resources/validate_test.go b/internal/resources/validate_test.go index d702097dd0..172cce0fea 100644 --- a/internal/resources/validate_test.go +++ b/internal/resources/validate_test.go @@ -23,6 +23,7 @@ import ( "testing" "go.ate.dev/substrate/internal/apierror" + "go.ate.dev/substrate/internal/nodepath" "go.ate.dev/substrate/internal/proto/ateworkerpb" "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" @@ -322,6 +323,7 @@ func testActorDirs() *ateworkerpb.ActorDirs { DurableDirVolumeMountsDir: "/node/actors/a/durable-dir", SystemInfoVolumeRootsDir: "/node/actors/a/system-info", VolumesDir: "/node/actors/a/volumes", + SandboxAssetsDir: "/node/actors/a/sandbox-assets", } } @@ -652,3 +654,29 @@ func TestValidateHostPort(t *testing.T) { {name: "unbracketed ipv6", value: "fd00::1:443", wantErr: true}, }) } + +func TestValidateSandboxAssetsDir(t *testing.T) { + orig := nodepath.ActorsDir + nodepath.ActorsDir = "/node/actors" + t.Cleanup(func() { nodepath.ActorsDir = orig }) + for _, tc := range []struct { + name string + actorUID string + dir string + wantErr bool + }{ + {"the actor's own dir", "actor-a", "/node/actors/actor-a/sandbox-assets", false}, + {"another actor's dir", "actor-a", "/node/actors/actor-b/sandbox-assets", true}, + {"a dir elsewhere", "actor-a", "/usr/bin", true}, + {"the root", "actor-a", "/", true}, + {"a traversing uid", "../etc", "/node/etc/sandbox-assets", true}, + {"an empty uid", "", "/node/actors/sandbox-assets", true}, + } { + t.Run(tc.name, func(t *testing.T) { + errs := ValidateSandboxAssetsDir(tc.actorUID, tc.dir, field.NewPath("actor_dirs", "sandbox_assets_dir")) + if (len(errs) > 0) != tc.wantErr { + t.Errorf("ValidateSandboxAssetsDir(%q, %q) = %v, wantErr %v", tc.actorUID, tc.dir, errs, tc.wantErr) + } + }) + } +} From e3a75a5205fc46b877d301e57ff35ca91e5dfc53 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Fri, 16 Oct 2026 16:32:21 +0000 Subject: [PATCH 3/6] Send and require snapshot object storage URI on all checkpoints (#2752) Populate CheckpointRequest.Snapshot.Object on both immediate and deferred checkpoints so atelet knows where to upload a paused actor's local snapshot in the background. PauseActor derives the URI from the persisted snapshot UUID rather than recording a pending DurableSnapshot on the actor, so an in-progress pause stays distinguishable from an in-progress suspend. Both workflows validate the template's storage location before marking the actor SUSPENDING. Part of https://github.com/agent-substrate/substrate/issues/2069 and https://github.com/agent-substrate/substrate/issues/2571 - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- .../controlapi/workflow_pause_test.go | 9 +-- .../internal/controlapi/workflow_suspend.go | 30 ++++++---- .../controlapi/workflow_suspend_test.go | 60 +++++++++++++++---- .../controlapi/workflow_testutil_test.go | 9 +++ cmd/atelet/internal/apivalidation/validate.go | 9 --- .../internal/apivalidation/validation_test.go | 14 ++--- .../apivalidation/zz_generated.validation.go | 20 +++++-- cmd/atelet/lifecycle_test.go | 8 +-- internal/proto/ateletpb/atelet.pb.go | 12 ++-- internal/proto/ateletpb/atelet.proto | 12 ++-- 10 files changed, 117 insertions(+), 66 deletions(-) diff --git a/cmd/ateapi/internal/controlapi/workflow_pause_test.go b/cmd/ateapi/internal/controlapi/workflow_pause_test.go index 92a2b50204..5f01d2f1da 100644 --- a/cmd/ateapi/internal/controlapi/workflow_pause_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_pause_test.go @@ -601,13 +601,13 @@ func TestPauseActor_AteletPaused_ForwardsPreferredFidelity(t *testing.T) { t.Run(tc.preferred.String(), func(t *testing.T) { ctx := context.Background() persistence := newTestPersistence(t) - seedSuspendTemplate(t, ctx, persistence, &ateapipb.SnapshotConfig{PreferredFidelity: tc.preferred}) + tmpl := seedSuspendTemplate(t, ctx, persistence, &ateapipb.SnapshotConfig{PreferredFidelity: tc.preferred}) w, atelet := newWireCaptureWorkflow(t, persistence) inst, reader := newTestInstruments(t) w.instruments = inst actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"} - storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ + created := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, ActorTemplate: suspendTestTemplateRef(), Status: &ateapipb.ActorStatus{ @@ -637,8 +637,9 @@ func TestPauseActor_AteletPaused_ForwardsPreferredFidelity(t *testing.T) { if got := req.GetSnapshot().GetGeneration(); got != 1 { t.Errorf("CheckpointRequest.Snapshot.Generation = %d, want 1, the in-progress snapshot's", got) } - if got := req.GetSnapshot().GetObject(); got != nil { - t.Errorf("CheckpointRequest.Snapshot.Object = %v, want nil for a node-local checkpoint", got) + wantURI := mustActorSnapshotURI(t, tmpl, created, "local-snap-1").String() + if got := req.GetSnapshot().GetObject().GetSnapshotUri(); got != wantURI { + t.Errorf("CheckpointRequest.Snapshot.Object.SnapshotUri = %q, want %q", got, wantURI) } if got := req.GetUploadConfig().GetMode(); got != ateletpb.UploadMode_UPLOAD_MODE_DEFERRED { t.Errorf("CheckpointRequest.UploadConfig.Mode = %v, want UPLOAD_MODE_DEFERRED", got) diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend.go b/cmd/ateapi/internal/controlapi/workflow_suspend.go index c4af95ee13..f4c19f8662 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend.go @@ -215,14 +215,10 @@ func (w *ActorWorkflow) ensureMarkedSuspending(ctx context.Context, actorRef res } // Fail here rather than at checkpoint time if the template's location - // cannot produce a usable URI: nothing has been written yet. An async - // suspend keeps its snapshot on the node, so it needs no URI. - var uri resources.SnapshotURI - if mode == suspendSync { - // TODO: Always populate the in-progress URI. - if uri, err = newInProgressSnapshotURI(actorTemplate, actor, snapshotName); err != nil { - return nil, err - } + // cannot produce a usable URI: nothing has been written yet. + uri, err := newInProgressSnapshotURI(actorTemplate, actor, snapshotName) + if err != nil { + return nil, err } storedActor, err := w.store.UpdateActor(ctx, actorRef, store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error { toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_SUSPENDING @@ -338,10 +334,16 @@ func (w *ActorWorkflow) ensureAteletSuspended(ctx context.Context, actorRef reso // into the snapshot manifest. inProgressSnap := snapshotAtLatestGeneration(actor.GetStatus()) fidelity := fidelityToAtelet(preferredFidelity(actor.GetMetadata().GetAtespace(), actorTemplate)) - // An async suspend's snapshot has no durable copy, so it stays on the node. - var object *ateletpb.ObjectStorage - if uri := inProgressSnap.GetDurableSnapshot().GetObject().GetSnapshotUri(); uri != "" { - object = &ateletpb.ObjectStorage{SnapshotUri: uri} + // A sync suspend records its URI on the in-progress durable snapshot; + // an async one has no durable snapshot record yet, so derive the URI + // from the persisted snapshot UUID. + snapshotURI := inProgressSnap.GetDurableSnapshot().GetObject().GetSnapshotUri() + if mode == suspendAsync { + uri, err := newInProgressSnapshotURI(actorTemplate, actor, inProgressSnap.GetUuid()) + if err != nil { + return "", err + } + snapshotURI = uri.String() } req := &ateletpb.CheckpointRequest{ WorkerPodUid: assignment.GetWorkerPodUid(), @@ -354,7 +356,9 @@ func (w *ActorWorkflow) ensureAteletSuspended(ctx context.Context, actorRef reso SnapshotUuid: inProgressSnap.GetUuid(), Generation: inProgressSnap.GetGeneration(), Fidelity: fidelity, - Object: object, + Object: &ateletpb.ObjectStorage{ + SnapshotUri: snapshotURI, + }, }, UploadConfig: &ateletpb.UploadConfig{ Mode: mode.uploadMode(), diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go index 47a8252fb3..0439a04f38 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go @@ -1152,7 +1152,7 @@ func TestSuspendAndPause_CharacteristicMatrix(t *testing.T) { ateletVolumes := ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES immediate := ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE checkpointSync := ateletCall{checkpoints: 1, fidelity: ateletVolumes, uploadMode: immediate, object: true} - checkpointAsync := ateletCall{checkpoints: 1, fidelity: ateletVolumes, uploadMode: ateletpb.UploadMode_UPLOAD_MODE_DEFERRED} + checkpointAsync := ateletCall{checkpoints: 1, fidelity: ateletVolumes, uploadMode: ateletpb.UploadMode_UPLOAD_MODE_DEFERRED, object: true} promote := ateletCall{promotes: 1, fidelity: ateletVolumes, object: true} // rejected is a FailedPrecondition outcome: no atelet call, and the // record left as want describes it. @@ -1277,7 +1277,7 @@ func TestSuspendAndPause_CharacteristicMatrix(t *testing.T) { // wireTestAssignment names. ctx := context.Background() persistence := newTestPersistence(t) - seedSuspendTemplate(t, ctx, persistence, &ateapipb.SnapshotConfig{StorageLocation: testStorageLocation, PreferredFidelity: volumes}) + tmpl := seedSuspendTemplate(t, ctx, persistence, &ateapipb.SnapshotConfig{StorageLocation: testStorageLocation, PreferredFidelity: volumes}) actorRef := resources.ActorRef{Atespace: atespace, Name: fmt.Sprintf("row%d-%s", i, mode.name)} created := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, @@ -1308,7 +1308,7 @@ func TestSuspendAndPause_CharacteristicMatrix(t *testing.T) { if got.state == ateapipb.ActorState_ACTOR_STATE_SUSPENDED && stored.GetStatus().GetWorkerAssignment() != nil { t.Errorf("WorkerAssignment = %v, want cleared on SUSPENDED", stored.GetStatus().GetWorkerAssignment()) } - assertRequestNamesLatestSnapshot(t, atelet, stored) + assertRequestNamesLatestSnapshot(t, atelet, tmpl, stored) }) } }) @@ -1363,6 +1363,44 @@ func TestSuspendAndPause_FinalizeWithoutAssignment(t *testing.T) { } } +// TestSuspendAndPause_InvalidStorageLocationRejected verifies that both +// workflows validate the template's storage location in MarkSuspending before +// writing SUSPENDING, leaving the actor RUNNING untouched. +func TestSuspendAndPause_InvalidStorageLocationRejected(t *testing.T) { + for _, mode := range suspendModes { + t.Run(mode.name, func(t *testing.T) { + ctx := context.Background() + persistence := newTestPersistence(t) + seedSuspendTemplate(t, ctx, persistence, &ateapipb.SnapshotConfig{StorageLocation: "not-a-uri"}) + w := newMarkOnlyWorkflow(persistence) + actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"} + storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, + ActorTemplate: suspendTestTemplateRef(), + Status: &ateapipb.ActorStatus{ + State: ateapipb.ActorState_ACTOR_STATE_RUNNING, + AssignedNode: "node-1", + WorkerAssignment: wireTestAssignment(), + }, + }) + + if _, err := mode.run(w, ctx, actorRef); err == nil { + t.Fatalf("%s: expected error for invalid storage location, got nil", mode.name) + } + stored := mustReadActor(t, ctx, persistence, actorRef) + if got := stored.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RUNNING { + t.Errorf("%s: stored state = %v, want RUNNING untouched", mode.name, got) + } + if got := stored.GetStatus().GetLastAssignedGeneration(); got != 0 { + t.Errorf("%s: last_assigned_generation = %d, want 0 untouched", mode.name, got) + } + if got := stored.GetStatus().GetSnapshots(); len(got) != 0 { + t.Errorf("%s: snapshots = %v, want empty", mode.name, got) + } + }) + } +} + // describeAteletCall summarizes the suspend RPCs atelet received: how many // of each, and the fidelity, upload mode, and object destination requested. func describeAteletCall(atelet *capturingAtelet) ateletCall { @@ -1396,9 +1434,9 @@ func describeSnapshots(status *ateapipb.ActorStatus) []snapshotOutcome { } // assertRequestNamesLatestSnapshot verifies the suspend RPC atelet received, -// if any, names the snapshot at the stored actor's latest generation, and that -// an object destination is that snapshot's durable copy. -func assertRequestNamesLatestSnapshot(t *testing.T, atelet *capturingAtelet, stored *ateapipb.Actor) { +// if any, names the snapshot at the stored actor's latest generation and its +// object storage URI. +func assertRequestNamesLatestSnapshot(t *testing.T, atelet *capturingAtelet, tmpl *ateapipb.ActorTemplate, stored *ateapipb.Actor) { t.Helper() var snap *ateletpb.Snapshot if req := atelet.checkpointRequest(); req != nil { @@ -1412,9 +1450,11 @@ func assertRequestNamesLatestSnapshot(t *testing.T, atelet *capturingAtelet, sto if got, want := snap.GetSnapshotUuid(), latest.GetUuid(); got != want { t.Errorf("request Snapshot.SnapshotUuid = %q, want the latest generation's %q", got, want) } - if obj := snap.GetObject(); obj != nil { - if got, want := obj.GetSnapshotUri(), latest.GetDurableSnapshot().GetObject().GetSnapshotUri(); got != want { - t.Errorf("request Snapshot.Object.SnapshotUri = %q, want the latest generation's durable copy %q", got, want) - } + wantURI := latest.GetDurableSnapshot().GetObject().GetSnapshotUri() + if wantURI == "" { + wantURI = mustActorSnapshotURI(t, tmpl, stored, latest.GetUuid()).String() + } + if got := snap.GetObject().GetSnapshotUri(); got != wantURI { + t.Errorf("request Snapshot.Object.SnapshotUri = %q, want %q", got, wantURI) } } diff --git a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go index a28a68c710..2361997237 100644 --- a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go @@ -31,6 +31,7 @@ import ( "go.ate.dev/substrate/pkg/proto/ateapipb" sdkmetric "go.opentelemetry.io/otel/sdk/metric" "google.golang.org/grpc/codes" + "google.golang.org/protobuf/proto" ) // testWorkerUID derives a stable pod UID from a pod name, for Workers seeded @@ -232,6 +233,14 @@ func seedSuspendTemplate(t *testing.T, ctx context.Context, st store.Interface, t.Helper() ref := suspendTestTemplateRef() storetest.MustCreateAtespace(t, ctx, st, ref.GetAtespace()) + if snapshotConfig.GetStorageLocation() == "" { + if snapshotConfig == nil { + snapshotConfig = &ateapipb.SnapshotConfig{} + } else { + snapshotConfig = proto.CloneOf(snapshotConfig) + } + snapshotConfig.StorageLocation = testStorageLocation + } created, err := st.CreateActorTemplate(ctx, &ateapipb.ActorTemplate{ Metadata: &ateapipb.ResourceMetadata{Atespace: ref.GetAtespace(), Name: ref.GetName()}, SnapshotConfig: snapshotConfig, diff --git a/cmd/atelet/internal/apivalidation/validate.go b/cmd/atelet/internal/apivalidation/validate.go index 17097a8959..bc67bff8dd 100644 --- a/cmd/atelet/internal/apivalidation/validate.go +++ b/cmd/atelet/internal/apivalidation/validate.go @@ -115,15 +115,6 @@ func ValidateCustom_PromoteSnapshotRequest_Atespace(_ context.Context, _ operati return nil } -// ValidateCustom_CheckpointRequest requires snapshot.object when -// upload_config.mode is UPLOAD_MODE_IMMEDIATE. -func ValidateCustom_CheckpointRequest(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *ateletpb.CheckpointRequest) field.ErrorList { - if value.GetUploadConfig().GetMode() == ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE && value.GetSnapshot() != nil && value.GetSnapshot().GetObject() == nil { - return field.ErrorList{field.Required(fldPath.Child("snapshot", "object"), "required when upload_config.mode is UPLOAD_MODE_IMMEDIATE")} - } - return nil -} - func ValidateCustom_ObjectStorage_SnapshotUri(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { return validateSnapshotURI(fldPath, *value) } diff --git a/cmd/atelet/internal/apivalidation/validation_test.go b/cmd/atelet/internal/apivalidation/validation_test.go index 2e81af37e9..5179da67b0 100644 --- a/cmd/atelet/internal/apivalidation/validation_test.go +++ b/cmd/atelet/internal/apivalidation/validation_test.go @@ -522,7 +522,6 @@ func TestValidateCheckpointRequest(t *testing.T) { deferred := func(name string) func(*ateletpb.CheckpointRequest) { return func(r *ateletpb.CheckpointRequest) { r.Snapshot.SnapshotUuid = name - r.Snapshot.Object = nil r.UploadConfig = &ateletpb.UploadConfig{Mode: ateletpb.UploadMode_UPLOAD_MODE_DEFERRED} } } @@ -535,13 +534,8 @@ func TestValidateCheckpointRequest(t *testing.T) { name: "valid immediate", obj: valid(), }, { - name: "valid deferred local only", + name: "valid deferred", obj: valid(deferred("pause-snap-1")), - }, { - name: "valid deferred with object storage", - obj: valid(func(r *ateletpb.CheckpointRequest) { - r.UploadConfig = &ateletpb.UploadConfig{Mode: ateletpb.UploadMode_UPLOAD_MODE_DEFERRED} - }), }, { name: "valid volumes fidelity", obj: valid(func(r *ateletpb.CheckpointRequest) { @@ -613,6 +607,12 @@ func TestValidateCheckpointRequest(t *testing.T) { name: "immediate upload without object storage", obj: valid(func(r *ateletpb.CheckpointRequest) { r.Snapshot.Object = nil }), want: field.ErrorList{field.Required(field.NewPath("snapshot", "object"), "")}, + }, { + name: "deferred upload without object storage", + obj: valid(deferred("pause-snap-1"), func(r *ateletpb.CheckpointRequest) { + r.Snapshot.Object = nil + }), + want: field.ErrorList{field.Required(field.NewPath("snapshot", "object"), "")}, }, { name: "missing snapshot_uuid", obj: valid(deferred("")), diff --git a/cmd/atelet/internal/apivalidation/zz_generated.validation.go b/cmd/atelet/internal/apivalidation/zz_generated.validation.go index 333cac6d1c..0b4679768d 100644 --- a/cmd/atelet/internal/apivalidation/zz_generated.validation.go +++ b/cmd/atelet/internal/apivalidation/zz_generated.validation.go @@ -472,11 +472,6 @@ func Validate_CheckpointRequest( ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *ateletpb.CheckpointRequest) (errs field.ErrorList) { - // custom validation - if e := ValidateCustom_CheckpointRequest(ctx, op, fldPath, obj, oldObj); len(e) != 0 { - errs = append(errs, e...) - } - { // field ateletpb.CheckpointRequest.WorkerPodUid fn := func( fldPath *field.Path, @@ -718,6 +713,21 @@ func Validate_CheckpointRequest( if earlyReturn { return // do not proceed } + func() { // cohort = "object" + earlyReturn := false + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "object", + func(o *ateletpb.Snapshot) *ateletpb.ObjectStorage { return o.Object }, ateDeepEqual, validate.RequiredPointer).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "object", + func(o *ateletpb.Snapshot) *ateletpb.ObjectStorage { return o.Object }, ateDeepEqual, validate.OptionalPointer).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + }() // call the type's validation function errs = append(errs, Validate_Snapshot(ctx, op, fldPath, obj, oldObj)...) return diff --git a/cmd/atelet/lifecycle_test.go b/cmd/atelet/lifecycle_test.go index 48c5e5d332..1eb2e595b4 100644 --- a/cmd/atelet/lifecycle_test.go +++ b/cmd/atelet/lifecycle_test.go @@ -462,12 +462,8 @@ func TestRestoreUsesRequestSandboxAssets(t *testing.T) { ActorTemplateName: "counter", WorkerPodUid: workerPodUID, Spec: spec, - Snapshot: &ateletpb.Snapshot{ - SnapshotUuid: snapshotName, - Generation: 1, - Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, - }, - UploadConfig: &ateletpb.UploadConfig{Mode: ateletpb.UploadMode_UPLOAD_MODE_DEFERRED}, + Snapshot: lifecycleSnapshot(actorUID, snapshotName, 1, ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY), + UploadConfig: &ateletpb.UploadConfig{Mode: ateletpb.UploadMode_UPLOAD_MODE_DEFERRED}, }); err != nil { t.Fatalf("Checkpoint: %v", err) } diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index 4bef1e0548..22a6e53386 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -3156,12 +3156,12 @@ type Snapshot struct { // +k8s:minimum=1 // +k8s:maximum=3 # keep this in sync with the SnapshotFidelity enum Fidelity SnapshotFidelity `protobuf:"varint,3,opt,name=fidelity,proto3,enum=atelet.SnapshotFidelity" json:"fidelity,omitempty"` - // The snapshot's durable copy in object storage. Required on every - // request: atelet uploads there, or reads from there when it holds no - // usable local copy. + // The snapshot's durable copy in object storage. Required on Checkpoint + // and PromoteSnapshot: atelet uploads there, or reads from there when it + // holds no usable local copy. // - // TODO: Temporarily, this is optional while we haven't implemented deferred - // uploads. + // TODO: Temporarily, this is optional on RestoreRequest while the control + // plane does not set it when restoring a paused actor. // // +k8s:optional Object *ObjectStorage `protobuf:"bytes,4,opt,name=object,proto3" json:"object,omitempty"` @@ -3427,7 +3427,6 @@ func (x *UploadConfig) GetMode() UploadMode { return UploadMode_UPLOAD_MODE_UNSPECIFIED } -// +k8s:customValidation # immediate upload requires snapshot.object type CheckpointRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // The UID of the worker pod whose the worker hosts the actor. @@ -3469,6 +3468,7 @@ type CheckpointRequest struct { // workload. // // +k8s:required + // +k8s:subfield(object)=+k8s:required Snapshot *Snapshot `protobuf:"bytes,8,opt,name=snapshot,proto3" json:"snapshot,omitempty"` // When the snapshot becomes durable. // diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index 94fbbdefbd..fb6b8123c9 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -914,12 +914,12 @@ message Snapshot { // +k8s:maximum=3 # keep this in sync with the SnapshotFidelity enum SnapshotFidelity fidelity = 3; - // The snapshot's durable copy in object storage. Required on every - // request: atelet uploads there, or reads from there when it holds no - // usable local copy. + // The snapshot's durable copy in object storage. Required on Checkpoint + // and PromoteSnapshot: atelet uploads there, or reads from there when it + // holds no usable local copy. // - // TODO: Temporarily, this is optional while we haven't implemented deferred - // uploads. + // TODO: Temporarily, this is optional on RestoreRequest while the control + // plane does not set it when restoring a paused actor. // // +k8s:optional ObjectStorage object = 4; @@ -998,7 +998,6 @@ enum UploadMode { // Keep this in sync with the maximums on fields of this type. } -// +k8s:customValidation # immediate upload requires snapshot.object message CheckpointRequest { // The UID of the worker pod whose the worker hosts the actor. // @@ -1046,6 +1045,7 @@ message CheckpointRequest { // workload. // // +k8s:required + // +k8s:subfield(object)=+k8s:required Snapshot snapshot = 8; // When the snapshot becomes durable. From 3a2ca62357485c4228fae466614ec98cf2caef69 Mon Sep 17 00:00:00 2001 From: Luiz Oliveira Date: Fri, 16 Oct 2026 16:45:44 +0000 Subject: [PATCH 4/6] Align volume and env message names across ateapi, atelet, and ateom pbs (#2666) Fixes #2419 1. In `internal/proto/ateletpb/atelet.proto`: - Rename `EnvEntry` -> `EnvVar`. - Rename `DurableDirVolume` -> `DurableDirVolumeSource` and `SystemInfoVolume` -> `SystemInfoVolumeSource`. 2. In `internal/proto/ateompb/ateom.proto`: - Rename `Container.csi_volume_mounts` -> `external_volume_mounts`. - Rename `ActorDirs.volumes_dir` -> `external_volume_mounts_dir`. - Consolidate `VolumeMount`, `DurableDirVolumeMount`, `SystemInfoVolumeMount`, and `ImageVolumeMount` into a single `VolumeMount { string name = 1; string mount_path = 2; }` message (matching `ateapi.VolumeMount` and `atelet.VolumeMount`). --- .../internal/controlapi/workload_spec.go | 6 +- .../internal/controlapi/workload_spec_test.go | 14 +- .../internal/apivalidation/validation_test.go | 28 +- .../internal/apivalidation/workloadspec.go | 6 +- .../apivalidation/zz_generated.validation.go | 48 +- cmd/atelet/internal/ateletpath/ateletpath.go | 8 +- .../internal/ateletpath/ateletpath_test.go | 2 +- cmd/atelet/lifecycle_test.go | 2 +- cmd/atelet/main.go | 36 +- cmd/atelet/main_test.go | 42 +- cmd/atelet/systeminfovolume.go | 6 +- cmd/atelet/systeminfovolume_test.go | 12 +- cmd/atelet/volumes.go | 4 +- cmd/atelet/volumes_test.go | 28 +- cmd/ateworker-gvisor/durable.go | 4 +- cmd/ateworker-gvisor/durable_test.go | 24 +- cmd/ateworker-gvisor/fidelity_test.go | 2 +- cmd/ateworker-gvisor/main_test.go | 4 +- cmd/ateworker-gvisor/runsc.go | 2 +- cmd/ateworker-gvisor/runsc_test.go | 2 +- cmd/ateworker-microvm/csi.go | 2 +- cmd/ateworker-microvm/csi_test.go | 12 +- cmd/ateworker-microvm/durable.go | 2 +- cmd/ateworker-microvm/durable_test.go | 14 +- cmd/ateworker-microvm/restore_test.go | 4 +- cmd/ateworker-microvm/run.go | 10 +- .../ateinterceptors/ateinterceptors_test.go | 2 +- internal/ocispec/microvm.go | 2 +- internal/ocispec/ocispec.go | 10 +- internal/ocispec/ocispec_test.go | 14 +- internal/ocispec/parity_test.go | 10 +- internal/proto/ateletpb/atelet.pb.go | 114 ++--- internal/proto/ateletpb/atelet.proto | 18 +- internal/proto/ateworkerpb/ateworker.pb.go | 411 +++++------------- internal/proto/ateworkerpb/ateworker.proto | 54 +-- internal/protoredact/protoredact_test.go | 10 +- internal/resources/validate.go | 2 +- internal/resources/validate_test.go | 2 +- 38 files changed, 382 insertions(+), 591 deletions(-) diff --git a/cmd/ateapi/internal/controlapi/workload_spec.go b/cmd/ateapi/internal/controlapi/workload_spec.go index 790d6b086a..be06a1ec56 100644 --- a/cmd/ateapi/internal/controlapi/workload_spec.go +++ b/cmd/ateapi/internal/controlapi/workload_spec.go @@ -58,11 +58,11 @@ func workloadSpecFromActorTemplate(actorTemplate *ateapipb.ActorTemplate, actor case vol.GetDurableDir() != nil: workloadSpec.Volumes = append(workloadSpec.Volumes, &ateletpb.Volume{ Name: vol.GetName(), - DurableDir: &ateletpb.DurableDirVolume{}, + DurableDir: &ateletpb.DurableDirVolumeSource{}, }) case vol.GetSystemInfo() != nil: - ateletSystemInfo := &ateletpb.SystemInfoVolume{} + ateletSystemInfo := &ateletpb.SystemInfoVolumeSource{} for _, dataSource := range vol.GetSystemInfo().GetDataSources() { switch { case dataSource.GetActorMetadata() != nil: @@ -130,7 +130,7 @@ func workloadSpecFromActorTemplate(actorTemplate *ateapipb.ActorTemplate, actor Resources: ctrResources, } for _, env := range ctr.GetEnv() { - ateletCtr.Env = append(ateletCtr.Env, &ateletpb.EnvEntry{ + ateletCtr.Env = append(ateletCtr.Env, &ateletpb.EnvVar{ Name: env.GetName(), Value: env.GetValue(), }) diff --git a/cmd/ateapi/internal/controlapi/workload_spec_test.go b/cmd/ateapi/internal/controlapi/workload_spec_test.go index f5adef117a..35032f5c2f 100644 --- a/cmd/ateapi/internal/controlapi/workload_spec_test.go +++ b/cmd/ateapi/internal/controlapi/workload_spec_test.go @@ -51,7 +51,7 @@ func TestWorkloadSpecFromActorTemplate(t *testing.T) { Volumes: []*ateletpb.Volume{ { Name: "home", - DurableDir: &ateletpb.DurableDirVolume{}, + DurableDir: &ateletpb.DurableDirVolumeSource{}, }, }, Containers: []*ateletpb.Container{ @@ -100,7 +100,7 @@ func TestWorkloadSpecFromActorTemplate(t *testing.T) { Volumes: []*ateletpb.Volume{ { Name: "system-info", - SystemInfo: &ateletpb.SystemInfoVolume{ + SystemInfo: &ateletpb.SystemInfoVolumeSource{ DataSources: []*ateletpb.SystemInfoDataSource{ {ActorMetadata: &ateletpb.ActorMetadataDataSource{ Items: []*ateletpb.ActorMetadataItem{ @@ -147,7 +147,7 @@ func TestWorkloadSpecFromActorTemplate(t *testing.T) { Volumes: []*ateletpb.Volume{ { Name: "home", - DurableDir: &ateletpb.DurableDirVolume{}, + DurableDir: &ateletpb.DurableDirVolumeSource{}, }, { Name: "agent", @@ -196,7 +196,7 @@ func TestWorkloadSpecFromActorTemplate(t *testing.T) { Volumes: []*ateletpb.Volume{ { Name: "system-info", - SystemInfo: &ateletpb.SystemInfoVolume{ + SystemInfo: &ateletpb.SystemInfoVolumeSource{ DataSources: []*ateletpb.SystemInfoDataSource{ {TrustBundle: &ateletpb.TrustBundleDataSource{Names: []string{"egress-trust"}, Path: "trust/ca.pem"}}, }, @@ -236,7 +236,7 @@ func TestWorkloadSpecFromActorTemplate(t *testing.T) { Volumes: []*ateletpb.Volume{ { Name: "home", - DurableDir: &ateletpb.DurableDirVolume{}, + DurableDir: &ateletpb.DurableDirVolumeSource{}, }, }, Containers: []*ateletpb.Container{ @@ -265,7 +265,7 @@ func TestWorkloadSpecFromActorTemplate(t *testing.T) { Volumes: []*ateletpb.Volume{ { Name: "home", - DurableDir: &ateletpb.DurableDirVolume{}, + DurableDir: &ateletpb.DurableDirVolumeSource{}, }, }, Containers: []*ateletpb.Container{{Name: "main", Image: "main"}}, @@ -290,7 +290,7 @@ func TestWorkloadSpecFromActorTemplate(t *testing.T) { Containers: []*ateletpb.Container{{ Name: "main", Image: "main", - Env: []*ateletpb.EnvEntry{ + Env: []*ateletpb.EnvVar{ {Name: "LITERAL", Value: "plain"}, {Name: "EMPTY", Value: ""}, }, diff --git a/cmd/atelet/internal/apivalidation/validation_test.go b/cmd/atelet/internal/apivalidation/validation_test.go index 5179da67b0..88d5d17949 100644 --- a/cmd/atelet/internal/apivalidation/validation_test.go +++ b/cmd/atelet/internal/apivalidation/validation_test.go @@ -1437,7 +1437,7 @@ func TestValidateWorkloadSpec(t *testing.T) { } valid := func(mutate ...func(*ateletpb.WorkloadSpec)) *ateletpb.WorkloadSpec { s := &ateletpb.WorkloadSpec{ - Volumes: []*ateletpb.Volume{{Name: "data", DurableDir: &ateletpb.DurableDirVolume{}}}, + Volumes: []*ateletpb.Volume{{Name: "data", DurableDir: &ateletpb.DurableDirVolumeSource{}}}, Containers: []*ateletpb.Container{ctr("main"), ctr("sidecar")}, } for _, m := range mutate { @@ -1470,7 +1470,7 @@ func TestValidateWorkloadSpec(t *testing.T) { obj: valid(func(s *ateletpb.WorkloadSpec) { s.Volumes = nil for i := range 33 { - s.Volumes = append(s.Volumes, &ateletpb.Volume{Name: fmt.Sprintf("v%d", i), DurableDir: &ateletpb.DurableDirVolume{}}) + s.Volumes = append(s.Volumes, &ateletpb.Volume{Name: fmt.Sprintf("v%d", i), DurableDir: &ateletpb.DurableDirVolumeSource{}}) } }), want: field.ErrorList{field.TooMany(field.NewPath("volumes"), 33, 32).WithOrigin("maxItems")}, @@ -1522,7 +1522,7 @@ func TestValidateWorkloadSpec(t *testing.T) { // asserted at their paths under the volume. func TestValidateVolume(t *testing.T) { durable := func(mutate ...func(*ateletpb.Volume)) *ateletpb.Volume { - v := &ateletpb.Volume{Name: "data", DurableDir: &ateletpb.DurableDirVolume{}} + v := &ateletpb.Volume{Name: "data", DurableDir: &ateletpb.DurableDirVolumeSource{}} for _, m := range mutate { m(v) } @@ -1543,7 +1543,7 @@ func TestValidateVolume(t *testing.T) { return &ateletpb.Volume{Name: "data", Image: &ateletpb.ImageVolumeSource{Reference: ref}} } systemInfo := func(ds ...*ateletpb.SystemInfoDataSource) *ateletpb.Volume { - return &ateletpb.Volume{Name: "data", SystemInfo: &ateletpb.SystemInfoVolume{DataSources: ds}} + return &ateletpb.Volume{Name: "data", SystemInfo: &ateletpb.SystemInfoVolumeSource{DataSources: ds}} } bundle := func(name, path string) *ateletpb.SystemInfoDataSource { return &ateletpb.SystemInfoDataSource{TrustBundle: &ateletpb.TrustBundleDataSource{Names: []string{name}, Path: path}} @@ -1654,7 +1654,7 @@ func TestValidateVolume(t *testing.T) { want: field.ErrorList{field.Invalid(field.NewPath("image", "reference"), nil, "")}, }, - // SystemInfoVolume and SystemInfoDataSource. + // SystemInfoVolumeSource and SystemInfoDataSource. { name: "valid system info", obj: systemInfo(bundle("podcert", "a.pem"), bundle("podcert", "b.pem"), metadata(item(fieldName, "name"))), @@ -1774,7 +1774,7 @@ func TestValidateContainer(t *testing.T) { Image: testDigestImage, Command: []string{"/bin/app"}, Args: []string{"--serve"}, - Env: []*ateletpb.EnvEntry{{Name: "PORT", Value: "8080"}}, + Env: []*ateletpb.EnvVar{{Name: "PORT", Value: "8080"}}, VolumeMounts: []*ateletpb.VolumeMount{ {Name: "data", MountPath: "/data"}, {Name: "data", MountPath: "/mnt/data"}, @@ -1785,7 +1785,7 @@ func TestValidateContainer(t *testing.T) { } return c } - env := func(mutate func(*ateletpb.EnvEntry)) *ateletpb.Container { + env := func(mutate func(*ateletpb.EnvVar)) *ateletpb.Container { return valid(func(c *ateletpb.Container) { mutate(c.Env[0]) }) } // mount replaces the mounts with a single one, so the nesting check @@ -1860,31 +1860,31 @@ func TestValidateContainer(t *testing.T) { want: field.ErrorList{field.TooLong(field.NewPath("args").Index(0), nil, 4096).WithOrigin("maxLength")}, }, - // EnvEntry. + // EnvVar. { name: "duplicate env names", obj: valid(func(c *ateletpb.Container) { - c.Env = append(c.Env, &ateletpb.EnvEntry{Name: "PORT", Value: "9"}) + c.Env = append(c.Env, &ateletpb.EnvVar{Name: "PORT", Value: "9"}) }), want: field.ErrorList{field.Duplicate(field.NewPath("env").Index(1), nil)}, }, { name: "env: missing name", - obj: env(func(e *ateletpb.EnvEntry) { e.Name = "" }), + obj: env(func(e *ateletpb.EnvVar) { e.Name = "" }), want: field.ErrorList{field.Required(envPath.Child("name"), "")}, }, { name: "env: name with equals sign", - obj: env(func(e *ateletpb.EnvEntry) { e.Name = "A=B" }), + obj: env(func(e *ateletpb.EnvVar) { e.Name = "A=B" }), want: field.ErrorList{field.Invalid(envPath.Child("name"), nil, "")}, }, { name: "env: name too long", - obj: env(func(e *ateletpb.EnvEntry) { e.Name = strings.Repeat("N", 257) }), + obj: env(func(e *ateletpb.EnvVar) { e.Name = strings.Repeat("N", 257) }), want: field.ErrorList{field.TooLong(envPath.Child("name"), nil, 256).WithOrigin("maxLength")}, }, { name: "env: empty value is allowed", - obj: env(func(e *ateletpb.EnvEntry) { e.Value = "" }), + obj: env(func(e *ateletpb.EnvVar) { e.Value = "" }), }, { name: "env: value too long", - obj: env(func(e *ateletpb.EnvEntry) { e.Value = strings.Repeat("v", 32769) }), + obj: env(func(e *ateletpb.EnvVar) { e.Value = strings.Repeat("v", 32769) }), want: field.ErrorList{field.TooLong(envPath.Child("value"), nil, 32768).WithOrigin("maxLength")}, }, diff --git a/cmd/atelet/internal/apivalidation/workloadspec.go b/cmd/atelet/internal/apivalidation/workloadspec.go index e3cfc456cf..ab8a808547 100644 --- a/cmd/atelet/internal/apivalidation/workloadspec.go +++ b/cmd/atelet/internal/apivalidation/workloadspec.go @@ -59,7 +59,7 @@ func ValidateCustom_Container_Image(_ context.Context, _ operation.Operation, fl return resources.ValidatePinnedImage(fldPath, *value) } -func ValidateCustom_EnvEntry_Name(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { +func ValidateCustom_EnvVar_Name(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { return resources.ValidateEnvVarName(fldPath, *value) } @@ -117,11 +117,11 @@ func ValidateCustom_TrustBundleDataSource_Path(_ context.Context, _ operation.Op return resources.ValidateProjectedPath(fldPath, *value) } -// ValidateCustom_SystemInfoVolume_DataSources allows at most one +// ValidateCustom_SystemInfoVolumeSource_DataSources allows at most one // actor_metadata entry and requires every projected file path to be unique // across all data sources: atelet writes them in order into one tree, so a // repeated path silently clobbers the earlier file. -func ValidateCustom_SystemInfoVolume_DataSources(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ []*ateletpb.SystemInfoDataSource) field.ErrorList { +func ValidateCustom_SystemInfoVolumeSource_DataSources(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ []*ateletpb.SystemInfoDataSource) field.ErrorList { var errs field.ErrorList seen := sets.New[string]() sawMetadata := false diff --git a/cmd/atelet/internal/apivalidation/zz_generated.validation.go b/cmd/atelet/internal/apivalidation/zz_generated.validation.go index 0b4679768d..10cc29c62e 100644 --- a/cmd/atelet/internal/apivalidation/zz_generated.validation.go +++ b/cmd/atelet/internal/apivalidation/zz_generated.validation.go @@ -926,7 +926,7 @@ func Validate_Container( { // field ateletpb.Container.Env fn := func( fldPath *field.Path, - obj, oldObj []*ateletpb.EnvEntry, + obj, oldObj []*ateletpb.EnvVar, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -936,7 +936,7 @@ func Validate_Container( } // call field-attached validations earlyReturn := false - if e := validate.PtrSliceNoNils[ateletpb.EnvEntry](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + if e := validate.PtrSliceNoNils[ateletpb.EnvVar](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { errs = append(errs, e...) earlyReturn = true } @@ -952,18 +952,18 @@ func Validate_Container( } // lists with map semantics require unique keys if e := validate.PtrSliceUnique(ctx, op, fldPath, obj, oldObj, - func(a *ateletpb.EnvEntry, b *ateletpb.EnvEntry) bool { return a.Name == b.Name }); len(e) != 0 { + func(a *ateletpb.EnvVar, b *ateletpb.EnvVar) bool { return a.Name == b.Name }); len(e) != 0 { errs = append(errs, e...) } // iterate the list and call the type's validation function if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, - func(a *ateletpb.EnvEntry, b *ateletpb.EnvEntry) bool { return a.Name == b.Name }, ateDeepEqual, Validate_EnvEntry); len(e) != 0 { + func(a *ateletpb.EnvVar, b *ateletpb.EnvVar) bool { return a.Name == b.Name }, ateDeepEqual, Validate_EnvVar); len(e) != 0 { errs = append(errs, e...) } return } oldVal := safe.Field(oldObj, - func(oldObj *ateletpb.Container) []*ateletpb.EnvEntry { + func(oldObj *ateletpb.Container) []*ateletpb.EnvVar { return oldObj.Env }) errs = append(errs, fn(fldPath.Child("env"), obj.Env, oldVal, oldObj != nil)...) @@ -1262,13 +1262,13 @@ func Validate_EgressGateway( return errs } -// Validate_EnvEntry validates an instance of EnvEntry according +// Validate_EnvVar validates an instance of EnvVar according // to declarative validation rules in the API schema. -func Validate_EnvEntry( +func Validate_EnvVar( ctx context.Context, op operation.Operation, fldPath *field.Path, - obj, oldObj *ateletpb.EnvEntry) (errs field.ErrorList) { + obj, oldObj *ateletpb.EnvVar) (errs field.ErrorList) { - { // field ateletpb.EnvEntry.Name + { // field ateletpb.EnvVar.Name fn := func( fldPath *field.Path, obj, oldObj *string, @@ -1289,7 +1289,7 @@ func Validate_EnvEntry( return // do not proceed } // custom validation - if e := ValidateCustom_EnvEntry_Name(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + if e := ValidateCustom_EnvVar_Name(ctx, op, fldPath, obj, oldObj); len(e) != 0 { errs = append(errs, e...) } if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 256); len(e) != 0 { @@ -1298,13 +1298,13 @@ func Validate_EnvEntry( return } oldVal := safe.Field(oldObj, - func(oldObj *ateletpb.EnvEntry) *string { + func(oldObj *ateletpb.EnvVar) *string { return &oldObj.Name }) errs = append(errs, fn(fldPath.Child("name"), &obj.Name, oldVal, oldObj != nil)...) } - { // field ateletpb.EnvEntry.Value + { // field ateletpb.EnvVar.Value fn := func( fldPath *field.Path, obj, oldObj *string, @@ -1329,7 +1329,7 @@ func Validate_EnvEntry( return } oldVal := safe.Field(oldObj, - func(oldObj *ateletpb.EnvEntry) *string { + func(oldObj *ateletpb.EnvVar) *string { return &oldObj.Value }) errs = append(errs, fn(fldPath.Child("value"), &obj.Value, oldVal, oldObj != nil)...) @@ -3712,13 +3712,13 @@ func Validate_SystemInfoDataSource( return errs } -// Validate_SystemInfoVolume validates an instance of SystemInfoVolume according +// Validate_SystemInfoVolumeSource validates an instance of SystemInfoVolumeSource according // to declarative validation rules in the API schema. -func Validate_SystemInfoVolume( +func Validate_SystemInfoVolumeSource( ctx context.Context, op operation.Operation, fldPath *field.Path, - obj, oldObj *ateletpb.SystemInfoVolume) (errs field.ErrorList) { + obj, oldObj *ateletpb.SystemInfoVolumeSource) (errs field.ErrorList) { - { // field ateletpb.SystemInfoVolume.DataSources + { // field ateletpb.SystemInfoVolumeSource.DataSources fn := func( fldPath *field.Path, obj, oldObj []*ateletpb.SystemInfoDataSource, @@ -3746,7 +3746,7 @@ func Validate_SystemInfoVolume( return // do not proceed } // custom validation - if e := ValidateCustom_SystemInfoVolume_DataSources(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + if e := ValidateCustom_SystemInfoVolumeSource_DataSources(ctx, op, fldPath, obj, oldObj); len(e) != 0 { errs = append(errs, e...) } // iterate the list and call the type's validation function @@ -3756,7 +3756,7 @@ func Validate_SystemInfoVolume( return } oldVal := safe.Field(oldObj, - func(oldObj *ateletpb.SystemInfoVolume) []*ateletpb.SystemInfoDataSource { + func(oldObj *ateletpb.SystemInfoVolumeSource) []*ateletpb.SystemInfoDataSource { return oldObj.DataSources }) errs = append(errs, fn(fldPath.Child("data_sources"), obj.DataSources, oldVal, oldObj != nil)...) @@ -4296,7 +4296,7 @@ func Validate_Volume( { // field ateletpb.Volume.DurableDir fn := func( fldPath *field.Path, - obj, oldObj *ateletpb.DurableDirVolume, + obj, oldObj *ateletpb.DurableDirVolumeSource, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -4315,7 +4315,7 @@ func Validate_Volume( return } oldVal := safe.Field(oldObj, - func(oldObj *ateletpb.Volume) *ateletpb.DurableDirVolume { + func(oldObj *ateletpb.Volume) *ateletpb.DurableDirVolumeSource { return oldObj.DurableDir }) errs = append(errs, fn(fldPath.Child("durable_dir"), obj.DurableDir, oldVal, oldObj != nil)...) @@ -4354,7 +4354,7 @@ func Validate_Volume( { // field ateletpb.Volume.SystemInfo fn := func( fldPath *field.Path, - obj, oldObj *ateletpb.SystemInfoVolume, + obj, oldObj *ateletpb.SystemInfoVolumeSource, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -4371,11 +4371,11 @@ func Validate_Volume( return // do not proceed } // call the type's validation function - errs = append(errs, Validate_SystemInfoVolume(ctx, op, fldPath, obj, oldObj)...) + errs = append(errs, Validate_SystemInfoVolumeSource(ctx, op, fldPath, obj, oldObj)...) return } oldVal := safe.Field(oldObj, - func(oldObj *ateletpb.Volume) *ateletpb.SystemInfoVolume { + func(oldObj *ateletpb.Volume) *ateletpb.SystemInfoVolumeSource { return oldObj.SystemInfo }) errs = append(errs, fn(fldPath.Child("system_info"), obj.SystemInfo, oldVal, oldObj != nil)...) diff --git a/cmd/atelet/internal/ateletpath/ateletpath.go b/cmd/atelet/internal/ateletpath/ateletpath.go index 1711abca83..88cdc56429 100644 --- a/cmd/atelet/internal/ateletpath/ateletpath.go +++ b/cmd/atelet/internal/ateletpath/ateletpath.go @@ -199,16 +199,16 @@ func RestoreStateDir(actorUID string) string { ) } -func VolumesDir(actorUID string) string { +func ExternalVolumeMountsDir(actorUID string) string { return filepath.Join( ActorPath(actorUID), "volumes", ) } -func VolumeHostPath(actorUID, volumeName string) string { +func ExternalVolumeHostPath(actorUID, volumeName string) string { return filepath.Join( - VolumesDir(actorUID), + ExternalVolumeMountsDir(actorUID), volumeName, ) } @@ -223,7 +223,7 @@ func ActorDirs(actorUID string) *ateworkerpb.ActorDirs { RestoreDir: RestoreStateDir(actorUID), DurableDirVolumeMountsDir: DurableDirVolumeMountsDir(actorUID), SystemInfoVolumeRootsDir: SystemInfoVolumeRootsDir(actorUID), - VolumesDir: VolumesDir(actorUID), + ExternalVolumeMountsDir: ExternalVolumeMountsDir(actorUID), SandboxAssetsDir: SandboxAssetsDir(actorUID), } } diff --git a/cmd/atelet/internal/ateletpath/ateletpath_test.go b/cmd/atelet/internal/ateletpath/ateletpath_test.go index fdb0beacd8..5968e3f2af 100644 --- a/cmd/atelet/internal/ateletpath/ateletpath_test.go +++ b/cmd/atelet/internal/ateletpath/ateletpath_test.go @@ -32,7 +32,7 @@ func TestActorDirs(t *testing.T) { "restore_dir": actorDirs.GetRestoreDir(), "durable_dir_volume_mounts_dir": actorDirs.GetDurableDirVolumeMountsDir(), "system_info_volume_roots_dir": actorDirs.GetSystemInfoVolumeRootsDir(), - "volumes_dir": actorDirs.GetVolumesDir(), + "external_volume_mounts_dir": actorDirs.GetExternalVolumeMountsDir(), "sandbox_assets_dir": actorDirs.GetSandboxAssetsDir(), } seen := map[string]string{} diff --git a/cmd/atelet/lifecycle_test.go b/cmd/atelet/lifecycle_test.go index 1eb2e595b4..4d2f2e0abf 100644 --- a/cmd/atelet/lifecycle_test.go +++ b/cmd/atelet/lifecycle_test.go @@ -557,7 +557,7 @@ func TestActivationFailureBeforeRegistration(t *testing.T) { if tc.failure == "reset" { // Reset refuses to remove a populated external-volume directory, // but has already removed the system-info roots by then. - dir := ateletpath.VolumeHostPath(actorUID, "mounted") + dir := ateletpath.ExternalVolumeHostPath(actorUID, "mounted") if err := os.MkdirAll(dir, 0o750); err != nil { t.Fatal(err) } diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 09b4d7c5ba..b93b09e6d5 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -1754,10 +1754,10 @@ func buildWorkerWorkloadSpec(spec *ateletpb.WorkloadSpec, containerSpecs []*atew out := &ateworkerpb.WorkloadSpec{} for i, ctr := range spec.GetContainers() { - var ddMounts []*ateworkerpb.DurableDirVolumeMount - var csiMounts []*ateworkerpb.VolumeMount - var siMounts []*ateworkerpb.SystemInfoVolumeMount - var imgMounts []*ateworkerpb.ImageVolumeMount + var ddMounts []*ateworkerpb.VolumeMount + var extMounts []*ateworkerpb.VolumeMount + var siMounts []*ateworkerpb.VolumeMount + var imgMounts []*ateworkerpb.VolumeMount for _, vm := range ctr.GetVolumeMounts() { volName := vm.GetName() vol, ok := volumes[volName] @@ -1767,24 +1767,24 @@ func buildWorkerWorkloadSpec(spec *ateletpb.WorkloadSpec, containerSpecs []*atew switch { case vol.GetDurableDir() != nil: - ddMounts = append(ddMounts, &ateworkerpb.DurableDirVolumeMount{ - VolumeName: volName, - MountPath: vm.GetMountPath(), + ddMounts = append(ddMounts, &ateworkerpb.VolumeMount{ + Name: volName, + MountPath: vm.GetMountPath(), }) case vol.GetExternal() != nil: - csiMounts = append(csiMounts, &ateworkerpb.VolumeMount{ - VolumeName: volName, - MountPath: vm.GetMountPath(), + extMounts = append(extMounts, &ateworkerpb.VolumeMount{ + Name: volName, + MountPath: vm.GetMountPath(), }) case vol.GetSystemInfo() != nil: - siMounts = append(siMounts, &ateworkerpb.SystemInfoVolumeMount{ - VolumeName: volName, - MountPath: vm.GetMountPath(), + siMounts = append(siMounts, &ateworkerpb.VolumeMount{ + Name: volName, + MountPath: vm.GetMountPath(), }) case vol.GetImage() != nil: - imgMounts = append(imgMounts, &ateworkerpb.ImageVolumeMount{ - VolumeName: volName, - MountPath: vm.GetMountPath(), + imgMounts = append(imgMounts, &ateworkerpb.VolumeMount{ + Name: volName, + MountPath: vm.GetMountPath(), }) default: return nil, fmt.Errorf("container %q mounts volume %q with no source set", ctr.GetName(), volName) @@ -1793,7 +1793,7 @@ func buildWorkerWorkloadSpec(spec *ateletpb.WorkloadSpec, containerSpecs []*atew container := &ateworkerpb.Container{ Name: ctr.GetName(), DurableDirVolumeMounts: ddMounts, - CsiVolumeMounts: csiMounts, + ExternalVolumeMounts: extMounts, SystemInfoVolumeMounts: siMounts, ImageVolumeMounts: imgMounts, WakeupProbe: toWorkerWakeupProbe(ctr.GetWakeupProbe()), @@ -1920,7 +1920,7 @@ func resetActorDirs(actorUID string) error { // Do not call RemoveAll on volume directories in case the unmount failed. // We do not want to delete mount content. - volumesDir := ateletpath.VolumesDir(actorUID) + volumesDir := ateletpath.ExternalVolumeMountsDir(actorUID) entries, err := os.ReadDir(volumesDir) if err != nil && !os.IsNotExist(err) { return wrapFileSystemErr("while reading volumes dir", err) diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index 8c1eaa1f08..d040c0bf2d 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -286,7 +286,7 @@ func TestPrepareOCIBundlesFillsContainerSpec(t *testing.T) { Image: image, Command: []string{"/bin/app"}, Args: []string{"serve"}, - Env: []*ateletpb.EnvEntry{{Name: "GREETING", Value: "hello"}}, + Env: []*ateletpb.EnvVar{{Name: "GREETING", Value: "hello"}}, SecurityContext: &ateletpb.SecurityContext{ Capabilities: &ateletpb.Capabilities{Add: []string{"NET_ADMIN"}, Drop: []string{"KILL"}}, }, @@ -796,10 +796,10 @@ func TestBuildWorkerWorkloadSpecForwardsContainerSpecs(t *testing.T) { func TestBuildWorkerWorkloadSpecForwardsDurableDirMounts(t *testing.T) { in := &ateletpb.WorkloadSpec{ Volumes: []*ateletpb.Volume{ - {Name: "data", DurableDir: &ateletpb.DurableDirVolume{}}, - {Name: "cache", DurableDir: &ateletpb.DurableDirVolume{}}, + {Name: "data", DurableDir: &ateletpb.DurableDirVolumeSource{}}, + {Name: "cache", DurableDir: &ateletpb.DurableDirVolumeSource{}}, {Name: "scratch", External: &ateletpb.ExternalVolumeSource{}}, - {Name: "system-info", SystemInfo: &ateletpb.SystemInfoVolume{}}, + {Name: "system-info", SystemInfo: &ateletpb.SystemInfoVolumeSource{}}, }, Containers: []*ateletpb.Container{ { @@ -826,21 +826,21 @@ func TestBuildWorkerWorkloadSpecForwardsDurableDirMounts(t *testing.T) { Containers: []*ateworkerpb.Container{ { Name: "main", - DurableDirVolumeMounts: []*ateworkerpb.DurableDirVolumeMount{ - {VolumeName: "data", MountPath: "/home/counter"}, - {VolumeName: "cache", MountPath: "/var/cache"}, + DurableDirVolumeMounts: []*ateworkerpb.VolumeMount{ + {Name: "data", MountPath: "/home/counter"}, + {Name: "cache", MountPath: "/var/cache"}, }, - CsiVolumeMounts: []*ateworkerpb.VolumeMount{ - {VolumeName: "scratch", MountPath: "/scratch"}, + ExternalVolumeMounts: []*ateworkerpb.VolumeMount{ + {Name: "scratch", MountPath: "/scratch"}, }, - SystemInfoVolumeMounts: []*ateworkerpb.SystemInfoVolumeMount{ - {VolumeName: "system-info", MountPath: "/run/ate"}, + SystemInfoVolumeMounts: []*ateworkerpb.VolumeMount{ + {Name: "system-info", MountPath: "/run/ate"}, }, }, { Name: "sidecar", - DurableDirVolumeMounts: []*ateworkerpb.DurableDirVolumeMount{ - {VolumeName: "data", MountPath: "/shared"}, + DurableDirVolumeMounts: []*ateworkerpb.VolumeMount{ + {Name: "data", MountPath: "/shared"}, }, }, {Name: "no-volumes"}, @@ -865,7 +865,7 @@ func TestBuildWorkerWorkloadSpecValidation(t *testing.T) { name: "missing volume definition", in: &ateletpb.WorkloadSpec{ Volumes: []*ateletpb.Volume{ - {Name: "data", DurableDir: &ateletpb.DurableDirVolume{}}, + {Name: "data", DurableDir: &ateletpb.DurableDirVolumeSource{}}, }, Containers: []*ateletpb.Container{ { @@ -899,7 +899,7 @@ func TestBuildWorkerWorkloadSpecValidation(t *testing.T) { name: "duplicate volume names", in: &ateletpb.WorkloadSpec{ Volumes: []*ateletpb.Volume{ - {Name: "data", DurableDir: &ateletpb.DurableDirVolume{}}, + {Name: "data", DurableDir: &ateletpb.DurableDirVolumeSource{}}, {Name: "data", External: &ateletpb.ExternalVolumeSource{}}, }, Containers: []*ateletpb.Container{ @@ -975,7 +975,7 @@ func TestRemoveActorDirsKeepsPopulatedVolume(t *testing.T) { useTempNodeDirs(t) const actorUID = "actor-uid-1" - stillMounted := filepath.Join(ateletpath.VolumeHostPath(actorUID, "data"), "payload") + stillMounted := filepath.Join(ateletpath.ExternalVolumeHostPath(actorUID, "data"), "payload") if err := os.MkdirAll(filepath.Dir(stillMounted), 0o755); err != nil { t.Fatalf("creating volume dir: %v", err) } @@ -1184,7 +1184,7 @@ func TestBuildWorkerWorkloadSpec_ImageVolumeMounts(t *testing.T) { spec := &ateletpb.WorkloadSpec{ Volumes: []*ateletpb.Volume{ {Name: "agent", Image: &ateletpb.ImageVolumeSource{}}, - {Name: "data", DurableDir: &ateletpb.DurableDirVolume{}}, + {Name: "data", DurableDir: &ateletpb.DurableDirVolumeSource{}}, {Name: "ext", External: &ateletpb.ExternalVolumeSource{}}, }, Containers: []*ateletpb.Container{{ @@ -1209,10 +1209,10 @@ func TestBuildWorkerWorkloadSpec_ImageVolumeMounts(t *testing.T) { if len(ctr.GetImageVolumeMounts()) != 1 { t.Fatalf("image volume mounts = %v, want 1", ctr.GetImageVolumeMounts()) } - if name, path := ctr.GetImageVolumeMounts()[0].GetVolumeName(), ctr.GetImageVolumeMounts()[0].GetMountPath(); name != "agent" || path != "/ate" { + if name, path := ctr.GetImageVolumeMounts()[0].GetName(), ctr.GetImageVolumeMounts()[0].GetMountPath(); name != "agent" || path != "/ate" { t.Errorf("image volume mount = (%q, %q), want (agent, /ate)", name, path) } - if len(ctr.GetDurableDirVolumeMounts()) != 1 || ctr.GetDurableDirVolumeMounts()[0].GetVolumeName() != "data" { + if len(ctr.GetDurableDirVolumeMounts()) != 1 || ctr.GetDurableDirVolumeMounts()[0].GetName() != "data" { t.Errorf("durable mounts = %v, want just data", ctr.GetDurableDirVolumeMounts()) } } @@ -1533,7 +1533,7 @@ func TestShouldHaveSnapshots(t *testing.T) { Snapshot: &ateletpb.Snapshot{Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES}, Spec: &ateletpb.WorkloadSpec{ Volumes: []*ateletpb.Volume{ - {Name: "durable", DurableDir: &ateletpb.DurableDirVolume{}}, + {Name: "durable", DurableDir: &ateletpb.DurableDirVolumeSource{}}, }, }, }, @@ -1557,7 +1557,7 @@ func TestShouldHaveSnapshots(t *testing.T) { Snapshot: &ateletpb.Snapshot{Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES}, Spec: &ateletpb.WorkloadSpec{ Volumes: []*ateletpb.Volume{ - {Name: "durable", DurableDir: &ateletpb.DurableDirVolume{}}, + {Name: "durable", DurableDir: &ateletpb.DurableDirVolumeSource{}}, {Name: "csi", External: &ateletpb.ExternalVolumeSource{}}, }, }, diff --git a/cmd/atelet/systeminfovolume.go b/cmd/atelet/systeminfovolume.go index 76d6964220..30685b7602 100644 --- a/cmd/atelet/systeminfovolume.go +++ b/cmd/atelet/systeminfovolume.go @@ -166,7 +166,7 @@ func (r *systemInfoVolumeRefresher) DeregisterOwned(owner *registeredActor) { } // collectData builds the volume's contents keyed by volume-relative path. -func (r *systemInfoVolumeRefresher) collectData(ref resources.ActorRef, actorUID string, si *ateletpb.SystemInfoVolume) (payload map[string][]byte, err error) { +func (r *systemInfoVolumeRefresher) collectData(ref resources.ActorRef, actorUID string, si *ateletpb.SystemInfoVolumeSource) (payload map[string][]byte, err error) { payload = map[string][]byte{} for _, dataSource := range si.GetDataSources() { switch { @@ -213,7 +213,7 @@ func (r *systemInfoVolumeRefresher) write(ref resources.ActorRef, actorUID strin type systemInfoVolume struct { Name string Root string - Spec *ateletpb.SystemInfoVolume + Spec *ateletpb.SystemInfoVolumeSource } // apply writes payload into the volume and reports whether any file changed. @@ -382,7 +382,7 @@ func (r *systemInfoVolumeRefresher) refreshBundle(ctx context.Context, bundleNam } // projectsBundle reports whether the volume spec projects the named bundle. -func projectsBundle(si *ateletpb.SystemInfoVolume, bundleName string) bool { +func projectsBundle(si *ateletpb.SystemInfoVolumeSource, bundleName string) bool { for _, ds := range si.GetDataSources() { if tb := ds.GetTrustBundle(); tb != nil && slices.Contains(tb.GetNames(), bundleName) { return true diff --git a/cmd/atelet/systeminfovolume_test.go b/cmd/atelet/systeminfovolume_test.go index 2a56ab9368..0a6652a8de 100644 --- a/cmd/atelet/systeminfovolume_test.go +++ b/cmd/atelet/systeminfovolume_test.go @@ -108,16 +108,16 @@ func (s *ctbStore) remove(t *testing.T) { } } -func trustVolumeSpec(relPath string) *ateletpb.SystemInfoVolume { - return &ateletpb.SystemInfoVolume{ +func trustVolumeSpec(relPath string) *ateletpb.SystemInfoVolumeSource { + return &ateletpb.SystemInfoVolumeSource{ DataSources: []*ateletpb.SystemInfoDataSource{ {TrustBundle: &ateletpb.TrustBundleDataSource{Names: []string{trustbundle.EgressName}, Path: relPath}}, }, } } -func metadataVolumeSpec() *ateletpb.SystemInfoVolume { - return &ateletpb.SystemInfoVolume{ +func metadataVolumeSpec() *ateletpb.SystemInfoVolumeSource { + return &ateletpb.SystemInfoVolumeSource{ DataSources: []*ateletpb.SystemInfoDataSource{ {ActorMetadata: &ateletpb.ActorMetadataDataSource{ Items: []*ateletpb.ActorMetadataItem{ @@ -395,7 +395,7 @@ func TestSystemInfoVolumeRefresher_RotationLeavesUnchangedFilesAlone(t *testing. store.set(t, certA) r := newSystemInfoVolumeRefresher(trustbundle.NewSource(store.lister.Get, nil), nil) root := filepath.Join(t.TempDir(), "system-info", "vol1") - spec := &ateletpb.SystemInfoVolume{ + spec := &ateletpb.SystemInfoVolumeSource{ DataSources: append(metadataVolumeSpec().GetDataSources(), trustVolumeSpec("trust/ca.pem").GetDataSources()...), } if _, err := r.Register("uid-1", resources.ActorRef{Atespace: "team-a", Name: "actor-1"}, []*systemInfoVolume{{Name: "vol1", Root: root, Spec: spec}}); err != nil { @@ -683,7 +683,7 @@ func TestSystemInfoVolumeRefresher_FailedReplacementDropsRegistration(t *testing func TestSystemInfoVolumesFor(t *testing.T) { spec := &ateletpb.WorkloadSpec{Volumes: []*ateletpb.Volume{ - {Name: "data", DurableDir: &ateletpb.DurableDirVolume{}}, + {Name: "data", DurableDir: &ateletpb.DurableDirVolumeSource{}}, {Name: "trust", SystemInfo: trustVolumeSpec("ca.pem")}, {Name: "meta", SystemInfo: metadataVolumeSpec()}, }} diff --git a/cmd/atelet/volumes.go b/cmd/atelet/volumes.go index b9c573c939..d1d04bc4bb 100644 --- a/cmd/atelet/volumes.go +++ b/cmd/atelet/volumes.go @@ -40,7 +40,7 @@ func (s *Atelet) mountExternalVolumes(ctx context.Context, actorUID string, volu if ext == nil { continue } - hostPath := ateletpath.VolumeHostPath(actorUID, vol.GetName()) + hostPath := ateletpath.ExternalVolumeHostPath(actorUID, vol.GetName()) if err := os.MkdirAll(hostPath, 0o750); err != nil { return fmt.Errorf("failed to create mount point %q: %w", hostPath, err) } @@ -73,7 +73,7 @@ func (s *Atelet) unmountExternalVolumes(ctx context.Context, actorUID string, vo if ext == nil { continue } - hostPath := ateletpath.VolumeHostPath(actorUID, vol.GetName()) + hostPath := ateletpath.ExternalVolumeHostPath(actorUID, vol.GetName()) slog.InfoContext(ctx, "Unmounting volume", slog.String("volume_id", ext.GetStorageVolumeId()), slog.String("host_path", hostPath), slog.String("volume_type", ext.GetVolumeType())) // TODO: Standardize volume plugin lookup and error handling across control plane // and worker plane (e.g. via a shared helper). diff --git a/cmd/atelet/volumes_test.go b/cmd/atelet/volumes_test.go index 46dc30e80d..4679f9d716 100644 --- a/cmd/atelet/volumes_test.go +++ b/cmd/atelet/volumes_test.go @@ -105,7 +105,7 @@ func TestUnmountExternalVolumes(t *testing.T) { } durableVol := &ateletpb.Volume{ Name: "durable-1", - DurableDir: &ateletpb.DurableDirVolume{}, + DurableDir: &ateletpb.DurableDirVolumeSource{}, } t.Run("success", func(t *testing.T) { @@ -187,7 +187,7 @@ func TestMountExternalVolumes(t *testing.T) { } durableVol := &ateletpb.Volume{ Name: "durable-1", - DurableDir: &ateletpb.DurableDirVolume{}, + DurableDir: &ateletpb.DurableDirVolumeSource{}, } t.Run("success mounts external volumes and creates mount directory", func(t *testing.T) { @@ -209,8 +209,8 @@ func TestMountExternalVolumes(t *testing.T) { t.Fatalf("mountCalls count = %d, want 2", len(fake.mountCalls)) } - expectedPath1 := ateletpath.VolumeHostPath(actorUID, "vol-1") - expectedPath2 := ateletpath.VolumeHostPath(actorUID, "vol-2") + expectedPath1 := ateletpath.ExternalVolumeHostPath(actorUID, "vol-1") + expectedPath2 := ateletpath.ExternalVolumeHostPath(actorUID, "vol-2") for _, path := range []string{expectedPath1, expectedPath2} { info, err := os.Stat(path) @@ -237,7 +237,7 @@ func TestMountExternalVolumes(t *testing.T) { t.Run("target directory already exists is handled cleanly", func(t *testing.T) { withTempActorsDir(t) - expectedPath := ateletpath.VolumeHostPath(actorUID, "vol-1") + expectedPath := ateletpath.ExternalVolumeHostPath(actorUID, "vol-1") if err := os.MkdirAll(expectedPath, 0o750); err != nil { t.Fatalf("pre-creating mount point: %v", err) } @@ -327,8 +327,8 @@ func TestMountExternalVolumes(t *testing.T) { t.Run("multi-volume partial failure does not roll back already mounted volume", func(t *testing.T) { withTempActorsDir(t) - expectedPath1 := ateletpath.VolumeHostPath(actorUID, "vol-1") - expectedPath2 := ateletpath.VolumeHostPath(actorUID, "vol-2") + expectedPath1 := ateletpath.ExternalVolumeHostPath(actorUID, "vol-1") + expectedPath2 := ateletpath.ExternalVolumeHostPath(actorUID, "vol-2") fake := &fakeWorkerPlugin{ mountErrs: map[string]error{ @@ -407,8 +407,8 @@ func TestVolumeHostDirectoryCleanup(t *testing.T) { t.Fatalf("mountExternalVolumes failed: %v", err) } - path1 := ateletpath.VolumeHostPath(actorUID, "vol-1") - path2 := ateletpath.VolumeHostPath(actorUID, "vol-2") + path1 := ateletpath.ExternalVolumeHostPath(actorUID, "vol-1") + path2 := ateletpath.ExternalVolumeHostPath(actorUID, "vol-2") for _, p := range []string{path1, path2} { if _, err := os.Stat(p); err != nil { t.Fatalf("expected mount path %q to exist before unmount: %v", p, err) @@ -445,8 +445,8 @@ func TestVolumeHostDirectoryCleanup(t *testing.T) { t.Fatalf("unmountExternalVolumes failed: %v", err) } - path1 := ateletpath.VolumeHostPath(actorUID, "vol-1") - path2 := ateletpath.VolumeHostPath(actorUID, "vol-2") + path1 := ateletpath.ExternalVolumeHostPath(actorUID, "vol-1") + path2 := ateletpath.ExternalVolumeHostPath(actorUID, "vol-2") for _, p := range []string{path1, path2} { if _, err := os.Stat(p); err != nil { t.Fatalf("expected mount path %q to exist before reset: %v", p, err) @@ -465,7 +465,7 @@ func TestVolumeHostDirectoryCleanup(t *testing.T) { } // The parent volumes directory is preserved and empty. - volumesDir := ateletpath.VolumesDir(actorUID) + volumesDir := ateletpath.ExternalVolumeMountsDir(actorUID) entries, err := os.ReadDir(volumesDir) if err != nil { t.Fatalf("ReadDir(%q) failed: %v", volumesDir, err) @@ -490,7 +490,7 @@ func TestVolumeHostDirectoryCleanup(t *testing.T) { } // Simulate lingering content in the volume mount point (e.g. unmount failed or data present). - path1 := ateletpath.VolumeHostPath(actorUID, "vol-1") + path1 := ateletpath.ExternalVolumeHostPath(actorUID, "vol-1") filePath := filepath.Join(path1, "lingering.txt") if err := os.WriteFile(filePath, []byte("volume data"), 0o600); err != nil { t.Fatalf("writing test file: %v", err) @@ -517,7 +517,7 @@ func TestVolumeHostDirectoryCleanup(t *testing.T) { t.Fatalf("resetActorDirs failed when volumes dir does not exist: %v", err) } - volumesDir := ateletpath.VolumesDir(actorUID) + volumesDir := ateletpath.ExternalVolumeMountsDir(actorUID) if info, err := os.Stat(volumesDir); err != nil || !info.IsDir() { t.Errorf("expected volumes dir %q to be created, stat err = %v", volumesDir, err) } diff --git a/cmd/ateworker-gvisor/durable.go b/cmd/ateworker-gvisor/durable.go index 0dcb8b4e93..0fa0fe6c8e 100644 --- a/cmd/ateworker-gvisor/durable.go +++ b/cmd/ateworker-gvisor/durable.go @@ -81,8 +81,8 @@ func fsRestoreContainers(containers []*ateworkerpb.Container) []string { for _, c := range containers { ownsMount := false for _, m := range c.GetDurableDirVolumeMounts() { - if !seen[m.GetVolumeName()] { - seen[m.GetVolumeName()] = true + if !seen[m.GetName()] { + seen[m.GetName()] = true ownsMount = true } } diff --git a/cmd/ateworker-gvisor/durable_test.go b/cmd/ateworker-gvisor/durable_test.go index ee92c52184..09caee38a4 100644 --- a/cmd/ateworker-gvisor/durable_test.go +++ b/cmd/ateworker-gvisor/durable_test.go @@ -29,15 +29,15 @@ func TestDurableFSCheckpointPaths(t *testing.T) { containers := []*ateworkerpb.Container{ { Name: "app", - DurableDirVolumeMounts: []*ateworkerpb.DurableDirVolumeMount{ - {VolumeName: "data", MountPath: "/var/data"}, - {VolumeName: "cache", MountPath: "/var/cache"}, + DurableDirVolumeMounts: []*ateworkerpb.VolumeMount{ + {Name: "data", MountPath: "/var/data"}, + {Name: "cache", MountPath: "/var/cache"}, }, }, { Name: "sidecar", - DurableDirVolumeMounts: []*ateworkerpb.DurableDirVolumeMount{ - {VolumeName: "logs", MountPath: "/var/log/app"}, + DurableDirVolumeMounts: []*ateworkerpb.VolumeMount{ + {Name: "logs", MountPath: "/var/log/app"}, }, }, } @@ -156,21 +156,21 @@ func TestFSRestoreContainers(t *testing.T) { }, { Name: "app", - DurableDirVolumeMounts: []*ateworkerpb.DurableDirVolumeMount{ - {VolumeName: "data", MountPath: "/var/data"}, + DurableDirVolumeMounts: []*ateworkerpb.VolumeMount{ + {Name: "data", MountPath: "/var/data"}, }, }, { Name: "sidecar-shared-only", - DurableDirVolumeMounts: []*ateworkerpb.DurableDirVolumeMount{ - {VolumeName: "data", MountPath: "/mnt/data"}, + DurableDirVolumeMounts: []*ateworkerpb.VolumeMount{ + {Name: "data", MountPath: "/mnt/data"}, }, }, { Name: "sidecar-with-own-vol", - DurableDirVolumeMounts: []*ateworkerpb.DurableDirVolumeMount{ - {VolumeName: "data", MountPath: "/mnt/data"}, - {VolumeName: "logs", MountPath: "/var/log"}, + DurableDirVolumeMounts: []*ateworkerpb.VolumeMount{ + {Name: "data", MountPath: "/mnt/data"}, + {Name: "logs", MountPath: "/var/log"}, }, }, } diff --git a/cmd/ateworker-gvisor/fidelity_test.go b/cmd/ateworker-gvisor/fidelity_test.go index d9c6fdca4b..486a9caef5 100644 --- a/cmd/ateworker-gvisor/fidelity_test.go +++ b/cmd/ateworker-gvisor/fidelity_test.go @@ -72,7 +72,7 @@ func TestRPCsRejectRootfsBeforeTouchingTheSandbox(t *testing.T) { RestoreDir: "/node/actors/actor-a/restore-state", DurableDirVolumeMountsDir: "/node/actors/actor-a/durable-dirs", SystemInfoVolumeRootsDir: "/node/actors/actor-a/system-info", - VolumesDir: "/node/actors/actor-a/volumes", + ExternalVolumeMountsDir: "/node/actors/actor-a/volumes", SandboxAssetsDir: "/node/actors/actor-a/sandbox-assets", } rootfs := ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS diff --git a/cmd/ateworker-gvisor/main_test.go b/cmd/ateworker-gvisor/main_test.go index d7c598797a..b2f5fcc539 100644 --- a/cmd/ateworker-gvisor/main_test.go +++ b/cmd/ateworker-gvisor/main_test.go @@ -77,7 +77,7 @@ func TestRPCsRejectUntrustedRunscPath(t *testing.T) { RestoreDir: "/node/actors/actor-a/restore", DurableDirVolumeMountsDir: "/node/actors/actor-a/durable-dirs", SystemInfoVolumeRootsDir: "/node/actors/actor-a/system-info", - VolumesDir: "/node/actors/actor-a/volumes", + ExternalVolumeMountsDir: "/node/actors/actor-a/volumes", SandboxAssetsDir: "/node/actors/actor-a/sandbox-assets", } for name, call := range map[string]func() error{ @@ -118,7 +118,7 @@ func TestRestoreWorkloadRejectsUnsupportedFidelity(t *testing.T) { RestoreDir: "/node/actors/actor-a/restore", DurableDirVolumeMountsDir: "/node/actors/actor-a/durable-dirs", SystemInfoVolumeRootsDir: "/node/actors/actor-a/system-info", - VolumesDir: "/node/actors/actor-a/volumes", + ExternalVolumeMountsDir: "/node/actors/actor-a/volumes", SandboxAssetsDir: sandboxAssetsDir, } // A zero service cannot acquire locks, configure networking, or open log pipes. diff --git a/cmd/ateworker-gvisor/runsc.go b/cmd/ateworker-gvisor/runsc.go index 7d53c37082..a9ab7675de 100644 --- a/cmd/ateworker-gvisor/runsc.go +++ b/cmd/ateworker-gvisor/runsc.go @@ -77,7 +77,7 @@ func durableVolumeNames(spec *ateworkerpb.WorkloadSpec) []string { var names []string for _, c := range spec.GetContainers() { for _, m := range c.GetDurableDirVolumeMounts() { - names = append(names, m.GetVolumeName()) + names = append(names, m.GetName()) } } slices.Sort(names) diff --git a/cmd/ateworker-gvisor/runsc_test.go b/cmd/ateworker-gvisor/runsc_test.go index a263fbab1a..8f1489851c 100644 --- a/cmd/ateworker-gvisor/runsc_test.go +++ b/cmd/ateworker-gvisor/runsc_test.go @@ -309,7 +309,7 @@ func TestRestoreWorkloadRejectsEmptyRestoreDir(t *testing.T) { CheckpointDir: "/node/actors/actor-a/checkpoint-state", DurableDirVolumeMountsDir: "/node/actors/actor-a/durable-dirs", SystemInfoVolumeRootsDir: "/node/actors/actor-a/system-info", - VolumesDir: "/node/actors/actor-a/volumes", + ExternalVolumeMountsDir: "/node/actors/actor-a/volumes", SandboxAssetsDir: "/node/actors/actor-a/sandbox-assets", }, }, diff --git a/cmd/ateworker-microvm/csi.go b/cmd/ateworker-microvm/csi.go index 6417ce0ad2..76970503d1 100644 --- a/cmd/ateworker-microvm/csi.go +++ b/cmd/ateworker-microvm/csi.go @@ -29,7 +29,7 @@ import ( // hasCsiVolumes reports whether any container mounts a CSI volume. func hasCsiVolumes(containers []*ateworkerpb.Container) bool { for _, c := range containers { - if len(c.GetCsiVolumeMounts()) > 0 { + if len(c.GetExternalVolumeMounts()) > 0 { return true } } diff --git a/cmd/ateworker-microvm/csi_test.go b/cmd/ateworker-microvm/csi_test.go index f03cff8d5d..d9ba55a295 100644 --- a/cmd/ateworker-microvm/csi_test.go +++ b/cmd/ateworker-microvm/csi_test.go @@ -38,8 +38,8 @@ func TestHasCsiVolumes(t *testing.T) { containers: []*ateworkerpb.Container{ { Name: "c1", - DurableDirVolumeMounts: []*ateworkerpb.DurableDirVolumeMount{ - {VolumeName: "data", MountPath: "/data"}, + DurableDirVolumeMounts: []*ateworkerpb.VolumeMount{ + {Name: "data", MountPath: "/data"}, }, }, }, @@ -50,8 +50,8 @@ func TestHasCsiVolumes(t *testing.T) { containers: []*ateworkerpb.Container{ { Name: "c1", - CsiVolumeMounts: []*ateworkerpb.VolumeMount{ - {VolumeName: "csi-vol", MountPath: "/csi"}, + ExternalVolumeMounts: []*ateworkerpb.VolumeMount{ + {Name: "csi-vol", MountPath: "/csi"}, }, }, }, @@ -65,8 +65,8 @@ func TestHasCsiVolumes(t *testing.T) { }, { Name: "c2", - CsiVolumeMounts: []*ateworkerpb.VolumeMount{ - {VolumeName: "csi-vol", MountPath: "/csi"}, + ExternalVolumeMounts: []*ateworkerpb.VolumeMount{ + {Name: "csi-vol", MountPath: "/csi"}, }, }, }, diff --git a/cmd/ateworker-microvm/durable.go b/cmd/ateworker-microvm/durable.go index 915ccc669d..ea327b66cd 100644 --- a/cmd/ateworker-microvm/durable.go +++ b/cmd/ateworker-microvm/durable.go @@ -81,7 +81,7 @@ func durableVolumeNames(containers []*ateworkerpb.Container) []string { var names []string for _, c := range containers { for _, m := range c.GetDurableDirVolumeMounts() { - names = append(names, m.GetVolumeName()) + names = append(names, m.GetName()) } } slices.Sort(names) diff --git a/cmd/ateworker-microvm/durable_test.go b/cmd/ateworker-microvm/durable_test.go index ca848f9267..3eae29afc3 100644 --- a/cmd/ateworker-microvm/durable_test.go +++ b/cmd/ateworker-microvm/durable_test.go @@ -41,8 +41,8 @@ func TestHasDurableVolumes(t *testing.T) { name: "one of several containers has a durable volume", containers: []*ateworkerpb.Container{ {Name: "sidecar"}, - {Name: "app", DurableDirVolumeMounts: []*ateworkerpb.DurableDirVolumeMount{ - {VolumeName: "data", MountPath: "/home/counter"}, + {Name: "app", DurableDirVolumeMounts: []*ateworkerpb.VolumeMount{ + {Name: "data", MountPath: "/home/counter"}, }}, }, want: true, @@ -59,12 +59,12 @@ func TestHasDurableVolumes(t *testing.T) { func TestDurableVolumeNames(t *testing.T) { containers := []*ateworkerpb.Container{ - {Name: "app", DurableDirVolumeMounts: []*ateworkerpb.DurableDirVolumeMount{ - {VolumeName: "data", MountPath: "/data"}, - {VolumeName: "cache", MountPath: "/cache"}, + {Name: "app", DurableDirVolumeMounts: []*ateworkerpb.VolumeMount{ + {Name: "data", MountPath: "/data"}, + {Name: "cache", MountPath: "/cache"}, }}, - {Name: "sidecar", DurableDirVolumeMounts: []*ateworkerpb.DurableDirVolumeMount{ - {VolumeName: "data", MountPath: "/shared"}, + {Name: "sidecar", DurableDirVolumeMounts: []*ateworkerpb.VolumeMount{ + {Name: "data", MountPath: "/shared"}, }}, } got := durableVolumeNames(containers) diff --git a/cmd/ateworker-microvm/restore_test.go b/cmd/ateworker-microvm/restore_test.go index 8d6fd10ca3..32bbb2674a 100644 --- a/cmd/ateworker-microvm/restore_test.go +++ b/cmd/ateworker-microvm/restore_test.go @@ -340,7 +340,7 @@ func TestRestoreWorkloadRejectsEmptyRestoreDir(t *testing.T) { CheckpointDir: "/node/actors/actor-a/checkpoint-state", DurableDirVolumeMountsDir: "/node/actors/actor-a/durable-dirs", SystemInfoVolumeRootsDir: "/node/actors/actor-a/system-info", - VolumesDir: "/node/actors/actor-a/volumes", + ExternalVolumeMountsDir: "/node/actors/actor-a/volumes", SandboxAssetsDir: "/node/actors/actor-a/sandbox-assets", }, }, @@ -512,7 +512,7 @@ func TestRPCsRejectUntrustedRuntimeAssetPaths(t *testing.T) { RestoreDir: "/node/actors/actor-a/restore", DurableDirVolumeMountsDir: "/node/actors/actor-a/durable-dirs", SystemInfoVolumeRootsDir: "/node/actors/actor-a/system-info", - VolumesDir: "/node/actors/actor-a/volumes", + ExternalVolumeMountsDir: "/node/actors/actor-a/volumes", SandboxAssetsDir: "/node/actors/actor-a/sandbox-assets", } assets := map[string]string{assetCH: "/bin/sh"} diff --git a/cmd/ateworker-microvm/run.go b/cmd/ateworker-microvm/run.go index 8d2d8d0460..0b60f7376d 100644 --- a/cmd/ateworker-microvm/run.go +++ b/cmd/ateworker-microvm/run.go @@ -192,7 +192,7 @@ type actorContainer struct { // spec is the container's OCI spec shaped for micro-VM execution. spec *specs.Spec // imageMounts are the image volumes this container mounts, and where. - imageMounts []*ateworkerpb.ImageVolumeMount + imageMounts []*ateworkerpb.VolumeMount } // resolvedRuntime holds the concrete binary paths for a request, taken from fetched @@ -698,9 +698,9 @@ func (s *AteWorkerService) stageMergedRootfs(ctx context.Context, rr resolvedRun return nil, fmt.Errorf("while staging merged rootfs for %q: %w", c.name, err) } for _, vm := range c.imageMounts { - src := imagecache.ImageVolumeMountPath(c.bundle, vm.GetVolumeName()) - if err := kata.StageImageVolume(ctx, src, id, c.name, vm.GetVolumeName()); err != nil { - return nil, fmt.Errorf("while staging image volume %q for %q: %w", vm.GetVolumeName(), c.name, err) + src := imagecache.ImageVolumeMountPath(c.bundle, vm.GetName()) + if err := kata.StageImageVolume(ctx, src, id, c.name, vm.GetName()); err != nil { + return nil, fmt.Errorf("while staging image volume %q for %q: %w", vm.GetName(), c.name, err) } } } @@ -710,7 +710,7 @@ func (s *AteWorkerService) stageMergedRootfs(ctx context.Context, rr resolvedRun } } if hasCsiVolumes(containers) { - if err := s.stageCsiVolumes(ctx, id, actorDirs.GetVolumesDir()); err != nil { + if err := s.stageCsiVolumes(ctx, id, actorDirs.GetExternalVolumeMountsDir()); err != nil { return nil, fmt.Errorf("while staging CSI volumes: %w", err) } } diff --git a/internal/ateinterceptors/ateinterceptors_test.go b/internal/ateinterceptors/ateinterceptors_test.go index c62f2cd470..34ebc9847f 100644 --- a/internal/ateinterceptors/ateinterceptors_test.go +++ b/internal/ateinterceptors/ateinterceptors_test.go @@ -359,7 +359,7 @@ func TestServerUnaryInterceptorRequestLogMasksEnvValues(t *testing.T) { Containers: []*ateletpb.Container{ { Name: "main", - Env: []*ateletpb.EnvEntry{ + Env: []*ateletpb.EnvVar{ {Name: "API_KEY", Value: "sk-secret"}, {Name: "PLAIN", Value: "not-a-secret"}, }, diff --git a/internal/ocispec/microvm.go b/internal/ocispec/microvm.go index 5fcdb231c1..fc11c2047e 100644 --- a/internal/ocispec/microvm.go +++ b/internal/ocispec/microvm.go @@ -115,7 +115,7 @@ func mergeKataResources(from *specs.LinuxResources) *specs.LinuxResources { func guestVolumeSource(hostPath string, actorDirs *ateworkerpb.ActorDirs, containerID string) (string, error) { for _, staged := range []struct{ host, guest string }{ {actorDirs.GetDurableDirVolumeMountsDir(), path.Join(GuestSharedDir, ShareDurable)}, - {actorDirs.GetVolumesDir(), path.Join(GuestSharedDir, ShareCSI)}, + {actorDirs.GetExternalVolumeMountsDir(), path.Join(GuestSharedDir, ShareCSI)}, {actorDirs.GetSystemInfoVolumeRootsDir(), path.Join(GuestSharedDir, ShareSystemInfo)}, {imagecache.ImageVolumeMountPath(filepath.Join(actorDirs.GetOciBundleDir(), containerID), ""), path.Join(GuestSharedDir, containerID, ShareVolumes)}, } { diff --git a/internal/ocispec/ocispec.go b/internal/ocispec/ocispec.go index b6a7c18914..5a2d491e10 100644 --- a/internal/ocispec/ocispec.go +++ b/internal/ocispec/ocispec.go @@ -171,17 +171,17 @@ func Build(o Options) *specs.Spec { } actorDirs := o.ActorDirs for _, m := range container.GetDurableDirVolumeMounts() { - bind(m.GetMountPath(), filepath.Join(actorDirs.GetDurableDirVolumeMountsDir(), m.GetVolumeName()), "rw") + bind(m.GetMountPath(), filepath.Join(actorDirs.GetDurableDirVolumeMountsDir(), m.GetName()), "rw") } - for _, m := range container.GetCsiVolumeMounts() { - bind(m.GetMountPath(), filepath.Join(actorDirs.GetVolumesDir(), m.GetVolumeName()), "rw") + for _, m := range container.GetExternalVolumeMounts() { + bind(m.GetMountPath(), filepath.Join(actorDirs.GetExternalVolumeMountsDir(), m.GetName()), "rw") } for _, m := range container.GetSystemInfoVolumeMounts() { - bind(m.GetMountPath(), filepath.Join(actorDirs.GetSystemInfoVolumeRootsDir(), m.GetVolumeName()), "ro") + bind(m.GetMountPath(), filepath.Join(actorDirs.GetSystemInfoVolumeRootsDir(), m.GetName()), "ro") } bundlePath := filepath.Join(actorDirs.GetOciBundleDir(), container.GetName()) for _, m := range container.GetImageVolumeMounts() { - bind(m.GetMountPath(), imagecache.ImageVolumeMountPath(bundlePath, m.GetVolumeName()), "ro") + bind(m.GetMountPath(), imagecache.ImageVolumeMountPath(bundlePath, m.GetName()), "ro") } return spec diff --git a/internal/ocispec/ocispec_test.go b/internal/ocispec/ocispec_test.go index 8b945c4476..a4edb2b4a8 100644 --- a/internal/ocispec/ocispec_test.go +++ b/internal/ocispec/ocispec_test.go @@ -51,19 +51,19 @@ func TestBuild_VolumeMounts(t *testing.T) { ActorDirs: &ateworkerpb.ActorDirs{ OciBundleDir: "/node/actors/a/bundles", DurableDirVolumeMountsDir: durableDir, - VolumesDir: volumesDir, + ExternalVolumeMountsDir: volumesDir, SystemInfoVolumeRootsDir: sysInfoDir, }, Container: &ateworkerpb.Container{ Name: "app", ContainerSpec: &ateworkerpb.ContainerSpec{Args: []string{"/app"}}, - DurableDirVolumeMounts: []*ateworkerpb.DurableDirVolumeMount{ - {VolumeName: "data", MountPath: "/var/data"}, - {VolumeName: "data", MountPath: "/home/counter"}, + DurableDirVolumeMounts: []*ateworkerpb.VolumeMount{ + {Name: "data", MountPath: "/var/data"}, + {Name: "data", MountPath: "/home/counter"}, }, - SystemInfoVolumeMounts: []*ateworkerpb.SystemInfoVolumeMount{{VolumeName: "sysinfo", MountPath: "/run/ate"}}, - CsiVolumeMounts: []*ateworkerpb.VolumeMount{{VolumeName: "csi", MountPath: "/mnt/csi"}}, - ImageVolumeMounts: []*ateworkerpb.ImageVolumeMount{{VolumeName: "agent", MountPath: "/ate"}}, + SystemInfoVolumeMounts: []*ateworkerpb.VolumeMount{{Name: "sysinfo", MountPath: "/run/ate"}}, + ExternalVolumeMounts: []*ateworkerpb.VolumeMount{{Name: "csi", MountPath: "/mnt/csi"}}, + ImageVolumeMounts: []*ateworkerpb.VolumeMount{{Name: "agent", MountPath: "/ate"}}, }, }) diff --git a/internal/ocispec/parity_test.go b/internal/ocispec/parity_test.go index e448625b07..5e6d5d30f6 100644 --- a/internal/ocispec/parity_test.go +++ b/internal/ocispec/parity_test.go @@ -32,7 +32,7 @@ var parityActorDirs = &ateworkerpb.ActorDirs{ OciBundleDir: "/node/actors/a/bundles", DurableDirVolumeMountsDir: "/node/actors/a/durable-dir", SystemInfoVolumeRootsDir: "/node/actors/a/system-info", - VolumesDir: "/node/actors/a/volumes", + ExternalVolumeMountsDir: "/node/actors/a/volumes", } // parityOptions mounts one volume of every kind. @@ -42,10 +42,10 @@ var parityOptions = Options{ Container: &ateworkerpb.Container{ Name: "app", ContainerSpec: &ateworkerpb.ContainerSpec{Args: []string{"/app"}}, - DurableDirVolumeMounts: []*ateworkerpb.DurableDirVolumeMount{{VolumeName: "data", MountPath: "/var/data"}}, - SystemInfoVolumeMounts: []*ateworkerpb.SystemInfoVolumeMount{{VolumeName: "sysinfo", MountPath: "/run/ate"}}, - CsiVolumeMounts: []*ateworkerpb.VolumeMount{{VolumeName: "csi", MountPath: "/mnt/csi"}}, - ImageVolumeMounts: []*ateworkerpb.ImageVolumeMount{{VolumeName: "agent", MountPath: "/ate"}}, + DurableDirVolumeMounts: []*ateworkerpb.VolumeMount{{Name: "data", MountPath: "/var/data"}}, + SystemInfoVolumeMounts: []*ateworkerpb.VolumeMount{{Name: "sysinfo", MountPath: "/run/ate"}}, + ExternalVolumeMounts: []*ateworkerpb.VolumeMount{{Name: "csi", MountPath: "/mnt/csi"}}, + ImageVolumeMounts: []*ateworkerpb.VolumeMount{{Name: "agent", MountPath: "/ate"}}, }, } diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index 22a6e53386..91554e0af9 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -1926,26 +1926,26 @@ func (x *WorkloadSpec) GetVolumes() []*Volume { return nil } -type DurableDirVolume struct { +type DurableDirVolumeSource struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } -func (x *DurableDirVolume) Reset() { - *x = DurableDirVolume{} +func (x *DurableDirVolumeSource) Reset() { + *x = DurableDirVolumeSource{} mi := &file_atelet_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *DurableDirVolume) String() string { +func (x *DurableDirVolumeSource) String() string { return protoimpl.X.MessageStringOf(x) } -func (*DurableDirVolume) ProtoMessage() {} +func (*DurableDirVolumeSource) ProtoMessage() {} -func (x *DurableDirVolume) ProtoReflect() protoreflect.Message { +func (x *DurableDirVolumeSource) ProtoReflect() protoreflect.Message { mi := &file_atelet_proto_msgTypes[27] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -1957,8 +1957,8 @@ func (x *DurableDirVolume) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use DurableDirVolume.ProtoReflect.Descriptor instead. -func (*DurableDirVolume) Descriptor() ([]byte, []int) { +// Deprecated: Use DurableDirVolumeSource.ProtoReflect.Descriptor instead. +func (*DurableDirVolumeSource) Descriptor() ([]byte, []int) { return file_atelet_proto_rawDescGZIP(), []int{27} } @@ -2342,10 +2342,10 @@ func (x *SystemInfoDataSource) GetTrustBundle() *TrustBundleDataSource { return nil } -// SystemInfoVolume is a read-only volume whose files are generated by atelet -// on every Run/Restore, so they carry the values of the actor actually being -// started, whatever checkpointed state it boots from. -type SystemInfoVolume struct { +// SystemInfoVolumeSource is a read-only volume whose files are generated by +// atelet on every Run/Restore, so they carry the values of the actor actually +// being started, whatever checkpointed state it boots from. +type SystemInfoVolumeSource struct { state protoimpl.MessageState `protogen:"open.v1"` // +k8s:optional // +k8s:maxItems=8 # matches the template's data_sources bound @@ -2356,20 +2356,20 @@ type SystemInfoVolume struct { sizeCache protoimpl.SizeCache } -func (x *SystemInfoVolume) Reset() { - *x = SystemInfoVolume{} +func (x *SystemInfoVolumeSource) Reset() { + *x = SystemInfoVolumeSource{} mi := &file_atelet_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *SystemInfoVolume) String() string { +func (x *SystemInfoVolumeSource) String() string { return protoimpl.X.MessageStringOf(x) } -func (*SystemInfoVolume) ProtoMessage() {} +func (*SystemInfoVolumeSource) ProtoMessage() {} -func (x *SystemInfoVolume) ProtoReflect() protoreflect.Message { +func (x *SystemInfoVolumeSource) ProtoReflect() protoreflect.Message { mi := &file_atelet_proto_msgTypes[34] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -2381,12 +2381,12 @@ func (x *SystemInfoVolume) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use SystemInfoVolume.ProtoReflect.Descriptor instead. -func (*SystemInfoVolume) Descriptor() ([]byte, []int) { +// Deprecated: Use SystemInfoVolumeSource.ProtoReflect.Descriptor instead. +func (*SystemInfoVolumeSource) Descriptor() ([]byte, []int) { return file_atelet_proto_rawDescGZIP(), []int{34} } -func (x *SystemInfoVolume) GetDataSources() []*SystemInfoDataSource { +func (x *SystemInfoVolumeSource) GetDataSources() []*SystemInfoDataSource { if x != nil { return x.DataSources } @@ -2403,13 +2403,13 @@ type Volume struct { // // +k8s:optional // +k8s:unionMember - DurableDir *DurableDirVolume `protobuf:"bytes,2,opt,name=durable_dir,json=durableDir,proto3" json:"durable_dir,omitempty"` + DurableDir *DurableDirVolumeSource `protobuf:"bytes,2,opt,name=durable_dir,json=durableDir,proto3" json:"durable_dir,omitempty"` // +k8s:optional // +k8s:unionMember External *ExternalVolumeSource `protobuf:"bytes,3,opt,name=external,proto3" json:"external,omitempty"` // +k8s:optional // +k8s:unionMember - SystemInfo *SystemInfoVolume `protobuf:"bytes,4,opt,name=system_info,json=systemInfo,proto3" json:"system_info,omitempty"` + SystemInfo *SystemInfoVolumeSource `protobuf:"bytes,4,opt,name=system_info,json=systemInfo,proto3" json:"system_info,omitempty"` // +k8s:optional // +k8s:unionMember Image *ImageVolumeSource `protobuf:"bytes,5,opt,name=image,proto3" json:"image,omitempty"` @@ -2454,7 +2454,7 @@ func (x *Volume) GetName() string { return "" } -func (x *Volume) GetDurableDir() *DurableDirVolume { +func (x *Volume) GetDurableDir() *DurableDirVolumeSource { if x != nil { return x.DurableDir } @@ -2468,7 +2468,7 @@ func (x *Volume) GetExternal() *ExternalVolumeSource { return nil } -func (x *Volume) GetSystemInfo() *SystemInfoVolume { +func (x *Volume) GetSystemInfo() *SystemInfoVolumeSource { if x != nil { return x.SystemInfo } @@ -2566,7 +2566,7 @@ type Container struct { // +k8s:maxItems=32 # matches the template's env bound // +k8s:listType=map # each variable is set at most once // +k8s:listMapKey=name - Env []*EnvEntry `protobuf:"bytes,4,rep,name=env,proto3" json:"env,omitempty"` + Env []*EnvVar `protobuf:"bytes,4,rep,name=env,proto3" json:"env,omitempty"` // +k8s:optional WakeupProbe *WakeupProbe `protobuf:"bytes,5,opt,name=wakeup_probe,json=wakeupProbe,proto3" json:"wakeup_probe,omitempty"` // Keyed by mount_path: each path hosts exactly one mount, while a volume @@ -2647,7 +2647,7 @@ func (x *Container) GetArgs() []string { return nil } -func (x *Container) GetEnv() []*EnvEntry { +func (x *Container) GetEnv() []*EnvVar { if x != nil { return x.Env } @@ -2853,7 +2853,7 @@ func (x *ResourceLimits) GetCpuMillis() int64 { return 0 } -type EnvEntry struct { +type EnvVar struct { state protoimpl.MessageState `protogen:"open.v1"` // name may be any printable ASCII character except '='. // @@ -2868,20 +2868,20 @@ type EnvEntry struct { sizeCache protoimpl.SizeCache } -func (x *EnvEntry) Reset() { - *x = EnvEntry{} +func (x *EnvVar) Reset() { + *x = EnvVar{} mi := &file_atelet_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *EnvEntry) String() string { +func (x *EnvVar) String() string { return protoimpl.X.MessageStringOf(x) } -func (*EnvEntry) ProtoMessage() {} +func (*EnvVar) ProtoMessage() {} -func (x *EnvEntry) ProtoReflect() protoreflect.Message { +func (x *EnvVar) ProtoReflect() protoreflect.Message { mi := &file_atelet_proto_msgTypes[41] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -2893,19 +2893,19 @@ func (x *EnvEntry) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use EnvEntry.ProtoReflect.Descriptor instead. -func (*EnvEntry) Descriptor() ([]byte, []int) { +// Deprecated: Use EnvVar.ProtoReflect.Descriptor instead. +func (*EnvVar) Descriptor() ([]byte, []int) { return file_atelet_proto_rawDescGZIP(), []int{41} } -func (x *EnvEntry) GetName() string { +func (x *EnvVar) GetName() string { if x != nil { return x.Name } return "" } -func (x *EnvEntry) GetValue() string { +func (x *EnvVar) GetValue() string { if x != nil { return x.Value } @@ -4209,8 +4209,8 @@ const file_atelet_proto_rawDesc = "" + "\n" + "containers\x18\x01 \x03(\v2\x11.atelet.ContainerR\n" + "containers\x12(\n" + - "\avolumes\x18\x02 \x03(\v2\x0e.atelet.VolumeR\avolumes\"\x12\n" + - "\x10DurableDirVolume\"\x9b\x03\n" + + "\avolumes\x18\x02 \x03(\v2\x0e.atelet.VolumeR\avolumes\"\x18\n" + + "\x16DurableDirVolumeSource\"\x9b\x03\n" + "\x14ExternalVolumeSource\x12*\n" + "\x11storage_volume_id\x18\x01 \x01(\tR\x0fstorageVolumeId\x12\x1f\n" + "\vvolume_type\x18\x02 \x01(\tR\n" + @@ -4235,27 +4235,27 @@ const file_atelet_proto_rawDesc = "" + "\x05names\x18\x03 \x03(\tR\x05namesJ\x04\b\x02\x10\x03R\x04name\"\xa0\x01\n" + "\x14SystemInfoDataSource\x12F\n" + "\x0eactor_metadata\x18\x01 \x01(\v2\x1f.atelet.ActorMetadataDataSourceR\ractorMetadata\x12@\n" + - "\ftrust_bundle\x18\x02 \x01(\v2\x1d.atelet.TrustBundleDataSourceR\vtrustBundle\"S\n" + - "\x10SystemInfoVolume\x12?\n" + - "\fdata_sources\x18\x01 \x03(\v2\x1c.atelet.SystemInfoDataSourceR\vdataSources\"\xfd\x01\n" + + "\ftrust_bundle\x18\x02 \x01(\v2\x1d.atelet.TrustBundleDataSourceR\vtrustBundle\"Y\n" + + "\x16SystemInfoVolumeSource\x12?\n" + + "\fdata_sources\x18\x01 \x03(\v2\x1c.atelet.SystemInfoDataSourceR\vdataSources\"\x89\x02\n" + "\x06Volume\x12\x12\n" + - "\x04name\x18\x01 \x01(\tR\x04name\x129\n" + - "\vdurable_dir\x18\x02 \x01(\v2\x18.atelet.DurableDirVolumeR\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12?\n" + + "\vdurable_dir\x18\x02 \x01(\v2\x1e.atelet.DurableDirVolumeSourceR\n" + "durableDir\x128\n" + - "\bexternal\x18\x03 \x01(\v2\x1c.atelet.ExternalVolumeSourceR\bexternal\x129\n" + - "\vsystem_info\x18\x04 \x01(\v2\x18.atelet.SystemInfoVolumeR\n" + + "\bexternal\x18\x03 \x01(\v2\x1c.atelet.ExternalVolumeSourceR\bexternal\x12?\n" + + "\vsystem_info\x18\x04 \x01(\v2\x1e.atelet.SystemInfoVolumeSourceR\n" + "systemInfo\x12/\n" + "\x05image\x18\x05 \x01(\v2\x19.atelet.ImageVolumeSourceR\x05image\"@\n" + "\vVolumeMount\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x1d\n" + "\n" + - "mount_path\x18\x02 \x01(\tR\tmountPath\"\xf3\x02\n" + + "mount_path\x18\x02 \x01(\tR\tmountPath\"\xf1\x02\n" + "\tContainer\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n" + "\x05image\x18\x02 \x01(\tR\x05image\x12\x18\n" + "\acommand\x18\x03 \x03(\tR\acommand\x12\x12\n" + - "\x04args\x18\a \x03(\tR\x04args\x12\"\n" + - "\x03env\x18\x04 \x03(\v2\x10.atelet.EnvEntryR\x03env\x126\n" + + "\x04args\x18\a \x03(\tR\x04args\x12 \n" + + "\x03env\x18\x04 \x03(\v2\x0e.atelet.EnvVarR\x03env\x126\n" + "\fwakeup_probe\x18\x05 \x01(\v2\x13.atelet.WakeupProbeR\vwakeupProbe\x128\n" + "\rvolume_mounts\x18\x06 \x03(\v2\x13.atelet.VolumeMountR\fvolumeMounts\x12B\n" + "\x10security_context\x18\b \x01(\v2\x17.atelet.SecurityContextR\x0fsecurityContext\x124\n" + @@ -4268,8 +4268,8 @@ const file_atelet_proto_rawDesc = "" + "\x0eResourceLimits\x12!\n" + "\fmemory_bytes\x18\x01 \x01(\x03R\vmemoryBytes\x12\x1d\n" + "\n" + - "cpu_millis\x18\x02 \x01(\x03R\tcpuMillis\"9\n" + - "\bEnvEntry\x12\x12\n" + + "cpu_millis\x18\x02 \x01(\x03R\tcpuMillis\"7\n" + + "\x06EnvVar\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x19\n" + "\x05value\x18\x02 \x01(\tB\x03\x80\x01\x01R\x05value\"h\n" + "\vWakeupProbe\x120\n" + @@ -4419,21 +4419,21 @@ var file_atelet_proto_goTypes = []any{ (*ArchAssets)(nil), // 27: atelet.ArchAssets (*SandboxAssets)(nil), // 28: atelet.SandboxAssets (*WorkloadSpec)(nil), // 29: atelet.WorkloadSpec - (*DurableDirVolume)(nil), // 30: atelet.DurableDirVolume + (*DurableDirVolumeSource)(nil), // 30: atelet.DurableDirVolumeSource (*ExternalVolumeSource)(nil), // 31: atelet.ExternalVolumeSource (*ImageVolumeSource)(nil), // 32: atelet.ImageVolumeSource (*ActorMetadataItem)(nil), // 33: atelet.ActorMetadataItem (*ActorMetadataDataSource)(nil), // 34: atelet.ActorMetadataDataSource (*TrustBundleDataSource)(nil), // 35: atelet.TrustBundleDataSource (*SystemInfoDataSource)(nil), // 36: atelet.SystemInfoDataSource - (*SystemInfoVolume)(nil), // 37: atelet.SystemInfoVolume + (*SystemInfoVolumeSource)(nil), // 37: atelet.SystemInfoVolumeSource (*Volume)(nil), // 38: atelet.Volume (*VolumeMount)(nil), // 39: atelet.VolumeMount (*Container)(nil), // 40: atelet.Container (*SecurityContext)(nil), // 41: atelet.SecurityContext (*Capabilities)(nil), // 42: atelet.Capabilities (*ResourceLimits)(nil), // 43: atelet.ResourceLimits - (*EnvEntry)(nil), // 44: atelet.EnvEntry + (*EnvVar)(nil), // 44: atelet.EnvVar (*WakeupProbe)(nil), // 45: atelet.WakeupProbe (*HttpGetAction)(nil), // 46: atelet.HttpGetAction (*RunResponse)(nil), // 47: atelet.RunResponse @@ -4485,12 +4485,12 @@ var file_atelet_proto_depIdxs = []int32{ 33, // 25: atelet.ActorMetadataDataSource.items:type_name -> atelet.ActorMetadataItem 34, // 26: atelet.SystemInfoDataSource.actor_metadata:type_name -> atelet.ActorMetadataDataSource 35, // 27: atelet.SystemInfoDataSource.trust_bundle:type_name -> atelet.TrustBundleDataSource - 36, // 28: atelet.SystemInfoVolume.data_sources:type_name -> atelet.SystemInfoDataSource - 30, // 29: atelet.Volume.durable_dir:type_name -> atelet.DurableDirVolume + 36, // 28: atelet.SystemInfoVolumeSource.data_sources:type_name -> atelet.SystemInfoDataSource + 30, // 29: atelet.Volume.durable_dir:type_name -> atelet.DurableDirVolumeSource 31, // 30: atelet.Volume.external:type_name -> atelet.ExternalVolumeSource - 37, // 31: atelet.Volume.system_info:type_name -> atelet.SystemInfoVolume + 37, // 31: atelet.Volume.system_info:type_name -> atelet.SystemInfoVolumeSource 32, // 32: atelet.Volume.image:type_name -> atelet.ImageVolumeSource - 44, // 33: atelet.Container.env:type_name -> atelet.EnvEntry + 44, // 33: atelet.Container.env:type_name -> atelet.EnvVar 45, // 34: atelet.Container.wakeup_probe:type_name -> atelet.WakeupProbe 39, // 35: atelet.Container.volume_mounts:type_name -> atelet.VolumeMount 41, // 36: atelet.Container.security_context:type_name -> atelet.SecurityContext diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index fb6b8123c9..dca175468d 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -544,7 +544,7 @@ message WorkloadSpec { repeated Volume volumes = 2; } -message DurableDirVolume { +message DurableDirVolumeSource { } message ExternalVolumeSource { @@ -661,10 +661,10 @@ message SystemInfoDataSource { TrustBundleDataSource trust_bundle = 2; } -// SystemInfoVolume is a read-only volume whose files are generated by atelet -// on every Run/Restore, so they carry the values of the actor actually being -// started, whatever checkpointed state it boots from. -message SystemInfoVolume { +// SystemInfoVolumeSource is a read-only volume whose files are generated by +// atelet on every Run/Restore, so they carry the values of the actor actually +// being started, whatever checkpointed state it boots from. +message SystemInfoVolumeSource { // +k8s:optional // +k8s:maxItems=8 # matches the template's data_sources bound // +k8s:listType=atomic @@ -682,7 +682,7 @@ message Volume { // // +k8s:optional // +k8s:unionMember - DurableDirVolume durable_dir = 2; + DurableDirVolumeSource durable_dir = 2; // +k8s:optional // +k8s:unionMember @@ -690,7 +690,7 @@ message Volume { // +k8s:optional // +k8s:unionMember - SystemInfoVolume system_info = 4; + SystemInfoVolumeSource system_info = 4; // +k8s:optional // +k8s:unionMember @@ -738,7 +738,7 @@ message Container { // +k8s:maxItems=32 # matches the template's env bound // +k8s:listType=map # each variable is set at most once // +k8s:listMapKey=name - repeated EnvEntry env = 4; + repeated EnvVar env = 4; // +k8s:optional WakeupProbe wakeup_probe = 5; @@ -802,7 +802,7 @@ message ResourceLimits { int64 cpu_millis = 2; } -message EnvEntry { +message EnvVar { // name may be any printable ASCII character except '='. // // +k8s:required diff --git a/internal/proto/ateworkerpb/ateworker.pb.go b/internal/proto/ateworkerpb/ateworker.pb.go index 3cf28e003c..2c12a7afed 100644 --- a/internal/proto/ateworkerpb/ateworker.pb.go +++ b/internal/proto/ateworkerpb/ateworker.pb.go @@ -175,8 +175,8 @@ type ActorDirs struct { DurableDirVolumeMountsDir string `protobuf:"bytes,5,opt,name=durable_dir_volume_mounts_dir,json=durableDirVolumeMountsDir,proto3" json:"durable_dir_volume_mounts_dir,omitempty"` // Holds one subdirectory per system-info volume. SystemInfoVolumeRootsDir string `protobuf:"bytes,6,opt,name=system_info_volume_roots_dir,json=systemInfoVolumeRootsDir,proto3" json:"system_info_volume_roots_dir,omitempty"` - // Holds one mount point per CSI volume. - VolumesDir string `protobuf:"bytes,7,opt,name=volumes_dir,json=volumesDir,proto3" json:"volumes_dir,omitempty"` + // Holds one mount point per external volume. + ExternalVolumeMountsDir string `protobuf:"bytes,7,opt,name=external_volume_mounts_dir,json=externalVolumeMountsDir,proto3" json:"external_volume_mounts_dir,omitempty"` // Holds the sandbox runtime assets staged for this actor (runsc, or the // micro-VM kernel, VMM and guest image), hard-linked from atelet's asset // cache. The only location a sandbox runtime may run a binary from; the @@ -259,9 +259,9 @@ func (x *ActorDirs) GetSystemInfoVolumeRootsDir() string { return "" } -func (x *ActorDirs) GetVolumesDir() string { +func (x *ActorDirs) GetExternalVolumeMountsDir() string { if x != nil { - return x.VolumesDir + return x.ExternalVolumeMountsDir } return "" } @@ -648,15 +648,17 @@ type Container struct { WakeupProbe *WakeupProbe `protobuf:"bytes,2,opt,name=wakeup_probe,json=wakeupProbe,proto3" json:"wakeup_probe,omitempty"` // durable_dir_volume_mounts are the durable-dir volumes this container // mounts, if any. - DurableDirVolumeMounts []*DurableDirVolumeMount `protobuf:"bytes,3,rep,name=durable_dir_volume_mounts,json=durableDirVolumeMounts,proto3" json:"durable_dir_volume_mounts,omitempty"` - // csi_volume_mounts are the CSI volumes this container mounts, if any. - CsiVolumeMounts []*VolumeMount `protobuf:"bytes,4,rep,name=csi_volume_mounts,json=csiVolumeMounts,proto3" json:"csi_volume_mounts,omitempty"` + DurableDirVolumeMounts []*VolumeMount `protobuf:"bytes,3,rep,name=durable_dir_volume_mounts,json=durableDirVolumeMounts,proto3" json:"durable_dir_volume_mounts,omitempty"` + // external_volume_mounts are the external volumes this container mounts, if + // any. + ExternalVolumeMounts []*VolumeMount `protobuf:"bytes,4,rep,name=external_volume_mounts,json=externalVolumeMounts,proto3" json:"external_volume_mounts,omitempty"` // system_info_volume_mounts are the system-info volumes this container - // mounts, if any. Contents are generated by atelet on the host; the - // container sees them read-only. - SystemInfoVolumeMounts []*SystemInfoVolumeMount `protobuf:"bytes,5,rep,name=system_info_volume_mounts,json=systemInfoVolumeMounts,proto3" json:"system_info_volume_mounts,omitempty"` + // mounts, if any. Contents are generated by atelet on the host on every + // Run/Restore and never captured into snapshots. The container sees them + // read-only. + SystemInfoVolumeMounts []*VolumeMount `protobuf:"bytes,5,rep,name=system_info_volume_mounts,json=systemInfoVolumeMounts,proto3" json:"system_info_volume_mounts,omitempty"` // image_volume_mounts are the image volumes this container mounts, if any. - ImageVolumeMounts []*ImageVolumeMount `protobuf:"bytes,6,rep,name=image_volume_mounts,json=imageVolumeMounts,proto3" json:"image_volume_mounts,omitempty"` + ImageVolumeMounts []*VolumeMount `protobuf:"bytes,6,rep,name=image_volume_mounts,json=imageVolumeMounts,proto3" json:"image_volume_mounts,omitempty"` // container_spec is what the worker builds the container's OCI spec from, // besides its volume mounts. Unset on RPCs that start no process. ContainerSpec *ContainerSpec `protobuf:"bytes,7,opt,name=container_spec,json=containerSpec,proto3" json:"container_spec,omitempty"` @@ -708,28 +710,28 @@ func (x *Container) GetWakeupProbe() *WakeupProbe { return nil } -func (x *Container) GetDurableDirVolumeMounts() []*DurableDirVolumeMount { +func (x *Container) GetDurableDirVolumeMounts() []*VolumeMount { if x != nil { return x.DurableDirVolumeMounts } return nil } -func (x *Container) GetCsiVolumeMounts() []*VolumeMount { +func (x *Container) GetExternalVolumeMounts() []*VolumeMount { if x != nil { - return x.CsiVolumeMounts + return x.ExternalVolumeMounts } return nil } -func (x *Container) GetSystemInfoVolumeMounts() []*SystemInfoVolumeMount { +func (x *Container) GetSystemInfoVolumeMounts() []*VolumeMount { if x != nil { return x.SystemInfoVolumeMounts } return nil } -func (x *Container) GetImageVolumeMounts() []*ImageVolumeMount { +func (x *Container) GetImageVolumeMounts() []*VolumeMount { if x != nil { return x.ImageVolumeMounts } @@ -875,9 +877,13 @@ func (x *ResourceLimits) GetCpuMillis() int64 { // VolumeMount is one volume mounted into a container. type VolumeMount struct { - state protoimpl.MessageState `protogen:"open.v1"` - VolumeName string `protobuf:"bytes,1,opt,name=volume_name,json=volumeName,proto3" json:"volume_name,omitempty"` - MountPath string `protobuf:"bytes,2,opt,name=mount_path,json=mountPath,proto3" json:"mount_path,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + // name is the name the ActorTemplate gave the volume. For durable-dir and + // system-info volumes it selects the per-volume directory atelet prepared + // for the actor on the host. + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // mount_path is where the container sees the volume. + MountPath string `protobuf:"bytes,2,opt,name=mount_path,json=mountPath,proto3" json:"mount_path,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -912,9 +918,9 @@ func (*VolumeMount) Descriptor() ([]byte, []int) { return file_ateworker_proto_rawDescGZIP(), []int{9} } -func (x *VolumeMount) GetVolumeName() string { +func (x *VolumeMount) GetName() string { if x != nil { - return x.VolumeName + return x.Name } return "" } @@ -926,177 +932,6 @@ func (x *VolumeMount) GetMountPath() string { return "" } -// DurableDirVolumeMount is one durable-dir volume mounted into a container. -type DurableDirVolumeMount struct { - state protoimpl.MessageState `protogen:"open.v1"` - // volume_name is the name the ActorTemplate gave the volume. It selects the - // per-volume directory atelet prepared for the actor on the host. - VolumeName string `protobuf:"bytes,1,opt,name=volume_name,json=volumeName,proto3" json:"volume_name,omitempty"` - // mount_path is where the container sees the volume. - MountPath string `protobuf:"bytes,2,opt,name=mount_path,json=mountPath,proto3" json:"mount_path,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *DurableDirVolumeMount) Reset() { - *x = DurableDirVolumeMount{} - mi := &file_ateworker_proto_msgTypes[10] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *DurableDirVolumeMount) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DurableDirVolumeMount) ProtoMessage() {} - -func (x *DurableDirVolumeMount) ProtoReflect() protoreflect.Message { - mi := &file_ateworker_proto_msgTypes[10] - if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DurableDirVolumeMount.ProtoReflect.Descriptor instead. -func (*DurableDirVolumeMount) Descriptor() ([]byte, []int) { - return file_ateworker_proto_rawDescGZIP(), []int{10} -} - -func (x *DurableDirVolumeMount) GetVolumeName() string { - if x != nil { - return x.VolumeName - } - return "" -} - -func (x *DurableDirVolumeMount) GetMountPath() string { - if x != nil { - return x.MountPath - } - return "" -} - -// SystemInfoVolumeMount is one system-info volume mounted (read-only) into a -// container. Unlike durable-dir volumes, system-info contents are generated -// by atelet on every Run/Restore and are never captured into snapshots. -type SystemInfoVolumeMount struct { - state protoimpl.MessageState `protogen:"open.v1"` - // volume_name is the name the ActorTemplate gave the volume. It selects the - // per-volume directory atelet prepared for the actor on the host. - VolumeName string `protobuf:"bytes,1,opt,name=volume_name,json=volumeName,proto3" json:"volume_name,omitempty"` - // mount_path is where the container sees the volume. - MountPath string `protobuf:"bytes,2,opt,name=mount_path,json=mountPath,proto3" json:"mount_path,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *SystemInfoVolumeMount) Reset() { - *x = SystemInfoVolumeMount{} - mi := &file_ateworker_proto_msgTypes[11] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *SystemInfoVolumeMount) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SystemInfoVolumeMount) ProtoMessage() {} - -func (x *SystemInfoVolumeMount) ProtoReflect() protoreflect.Message { - mi := &file_ateworker_proto_msgTypes[11] - if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SystemInfoVolumeMount.ProtoReflect.Descriptor instead. -func (*SystemInfoVolumeMount) Descriptor() ([]byte, []int) { - return file_ateworker_proto_rawDescGZIP(), []int{11} -} - -func (x *SystemInfoVolumeMount) GetVolumeName() string { - if x != nil { - return x.VolumeName - } - return "" -} - -func (x *SystemInfoVolumeMount) GetMountPath() string { - if x != nil { - return x.MountPath - } - return "" -} - -// ImageVolumeMount is one image volume mounted into a container. The worker uses -// these to construct the container's volume mounts — each names the volume -// and its destination path. -type ImageVolumeMount struct { - state protoimpl.MessageState `protogen:"open.v1"` - // volume_name is the name the ActorTemplate gave the volume. - VolumeName string `protobuf:"bytes,1,opt,name=volume_name,json=volumeName,proto3" json:"volume_name,omitempty"` - // mount_path is where the container sees the volume. - MountPath string `protobuf:"bytes,2,opt,name=mount_path,json=mountPath,proto3" json:"mount_path,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *ImageVolumeMount) Reset() { - *x = ImageVolumeMount{} - mi := &file_ateworker_proto_msgTypes[12] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *ImageVolumeMount) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ImageVolumeMount) ProtoMessage() {} - -func (x *ImageVolumeMount) ProtoReflect() protoreflect.Message { - mi := &file_ateworker_proto_msgTypes[12] - if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ImageVolumeMount.ProtoReflect.Descriptor instead. -func (*ImageVolumeMount) Descriptor() ([]byte, []int) { - return file_ateworker_proto_rawDescGZIP(), []int{12} -} - -func (x *ImageVolumeMount) GetVolumeName() string { - if x != nil { - return x.VolumeName - } - return "" -} - -func (x *ImageVolumeMount) GetMountPath() string { - if x != nil { - return x.MountPath - } - return "" -} - // WakeupProbe describes how to check that a container is ready to serve. // Only HTTP is supported today. type WakeupProbe struct { @@ -1110,7 +945,7 @@ type WakeupProbe struct { func (x *WakeupProbe) Reset() { *x = WakeupProbe{} - mi := &file_ateworker_proto_msgTypes[13] + mi := &file_ateworker_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1122,7 +957,7 @@ func (x *WakeupProbe) String() string { func (*WakeupProbe) ProtoMessage() {} func (x *WakeupProbe) ProtoReflect() protoreflect.Message { - mi := &file_ateworker_proto_msgTypes[13] + mi := &file_ateworker_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1135,7 +970,7 @@ func (x *WakeupProbe) ProtoReflect() protoreflect.Message { // Deprecated: Use WakeupProbe.ProtoReflect.Descriptor instead. func (*WakeupProbe) Descriptor() ([]byte, []int) { - return file_ateworker_proto_rawDescGZIP(), []int{13} + return file_ateworker_proto_rawDescGZIP(), []int{10} } func (x *WakeupProbe) GetHttpGet() *HttpGetAction { @@ -1165,7 +1000,7 @@ type HttpGetAction struct { func (x *HttpGetAction) Reset() { *x = HttpGetAction{} - mi := &file_ateworker_proto_msgTypes[14] + mi := &file_ateworker_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1177,7 +1012,7 @@ func (x *HttpGetAction) String() string { func (*HttpGetAction) ProtoMessage() {} func (x *HttpGetAction) ProtoReflect() protoreflect.Message { - mi := &file_ateworker_proto_msgTypes[14] + mi := &file_ateworker_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1190,7 +1025,7 @@ func (x *HttpGetAction) ProtoReflect() protoreflect.Message { // Deprecated: Use HttpGetAction.ProtoReflect.Descriptor instead. func (*HttpGetAction) Descriptor() ([]byte, []int) { - return file_ateworker_proto_rawDescGZIP(), []int{14} + return file_ateworker_proto_rawDescGZIP(), []int{11} } func (x *HttpGetAction) GetPath() string { @@ -1215,7 +1050,7 @@ type RunWorkloadResponse struct { func (x *RunWorkloadResponse) Reset() { *x = RunWorkloadResponse{} - mi := &file_ateworker_proto_msgTypes[15] + mi := &file_ateworker_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1227,7 +1062,7 @@ func (x *RunWorkloadResponse) String() string { func (*RunWorkloadResponse) ProtoMessage() {} func (x *RunWorkloadResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateworker_proto_msgTypes[15] + mi := &file_ateworker_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1240,7 +1075,7 @@ func (x *RunWorkloadResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RunWorkloadResponse.ProtoReflect.Descriptor instead. func (*RunWorkloadResponse) Descriptor() ([]byte, []int) { - return file_ateworker_proto_rawDescGZIP(), []int{15} + return file_ateworker_proto_rawDescGZIP(), []int{12} } type CheckpointWorkloadRequest struct { @@ -1275,7 +1110,7 @@ type CheckpointWorkloadRequest struct { func (x *CheckpointWorkloadRequest) Reset() { *x = CheckpointWorkloadRequest{} - mi := &file_ateworker_proto_msgTypes[16] + mi := &file_ateworker_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1287,7 +1122,7 @@ func (x *CheckpointWorkloadRequest) String() string { func (*CheckpointWorkloadRequest) ProtoMessage() {} func (x *CheckpointWorkloadRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateworker_proto_msgTypes[16] + mi := &file_ateworker_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1300,7 +1135,7 @@ func (x *CheckpointWorkloadRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CheckpointWorkloadRequest.ProtoReflect.Descriptor instead. func (*CheckpointWorkloadRequest) Descriptor() ([]byte, []int) { - return file_ateworker_proto_rawDescGZIP(), []int{16} + return file_ateworker_proto_rawDescGZIP(), []int{13} } func (x *CheckpointWorkloadRequest) GetAtespace() string { @@ -1395,7 +1230,7 @@ type CheckpointWorkloadResponse struct { func (x *CheckpointWorkloadResponse) Reset() { *x = CheckpointWorkloadResponse{} - mi := &file_ateworker_proto_msgTypes[17] + mi := &file_ateworker_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1407,7 +1242,7 @@ func (x *CheckpointWorkloadResponse) String() string { func (*CheckpointWorkloadResponse) ProtoMessage() {} func (x *CheckpointWorkloadResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateworker_proto_msgTypes[17] + mi := &file_ateworker_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1420,7 +1255,7 @@ func (x *CheckpointWorkloadResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CheckpointWorkloadResponse.ProtoReflect.Descriptor instead. func (*CheckpointWorkloadResponse) Descriptor() ([]byte, []int) { - return file_ateworker_proto_rawDescGZIP(), []int{17} + return file_ateworker_proto_rawDescGZIP(), []int{14} } func (x *CheckpointWorkloadResponse) GetSnapshotFiles() []string { @@ -1472,7 +1307,7 @@ type RestoreWorkloadRequest struct { func (x *RestoreWorkloadRequest) Reset() { *x = RestoreWorkloadRequest{} - mi := &file_ateworker_proto_msgTypes[18] + mi := &file_ateworker_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1484,7 +1319,7 @@ func (x *RestoreWorkloadRequest) String() string { func (*RestoreWorkloadRequest) ProtoMessage() {} func (x *RestoreWorkloadRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateworker_proto_msgTypes[18] + mi := &file_ateworker_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1497,7 +1332,7 @@ func (x *RestoreWorkloadRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreWorkloadRequest.ProtoReflect.Descriptor instead. func (*RestoreWorkloadRequest) Descriptor() ([]byte, []int) { - return file_ateworker_proto_rawDescGZIP(), []int{18} + return file_ateworker_proto_rawDescGZIP(), []int{15} } func (x *RestoreWorkloadRequest) GetAtespace() string { @@ -1613,7 +1448,7 @@ type RestoreWorkloadResponse struct { func (x *RestoreWorkloadResponse) Reset() { *x = RestoreWorkloadResponse{} - mi := &file_ateworker_proto_msgTypes[19] + mi := &file_ateworker_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1625,7 +1460,7 @@ func (x *RestoreWorkloadResponse) String() string { func (*RestoreWorkloadResponse) ProtoMessage() {} func (x *RestoreWorkloadResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateworker_proto_msgTypes[19] + mi := &file_ateworker_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1638,7 +1473,7 @@ func (x *RestoreWorkloadResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreWorkloadResponse.ProtoReflect.Descriptor instead. func (*RestoreWorkloadResponse) Descriptor() ([]byte, []int) { - return file_ateworker_proto_rawDescGZIP(), []int{19} + return file_ateworker_proto_rawDescGZIP(), []int{16} } type GetWorkloadStatsRequest struct { @@ -1654,7 +1489,7 @@ type GetWorkloadStatsRequest struct { func (x *GetWorkloadStatsRequest) Reset() { *x = GetWorkloadStatsRequest{} - mi := &file_ateworker_proto_msgTypes[20] + mi := &file_ateworker_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1666,7 +1501,7 @@ func (x *GetWorkloadStatsRequest) String() string { func (*GetWorkloadStatsRequest) ProtoMessage() {} func (x *GetWorkloadStatsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateworker_proto_msgTypes[20] + mi := &file_ateworker_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1679,7 +1514,7 @@ func (x *GetWorkloadStatsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetWorkloadStatsRequest.ProtoReflect.Descriptor instead. func (*GetWorkloadStatsRequest) Descriptor() ([]byte, []int) { - return file_ateworker_proto_rawDescGZIP(), []int{20} + return file_ateworker_proto_rawDescGZIP(), []int{17} } func (x *GetWorkloadStatsRequest) GetActorUid() string { @@ -1744,7 +1579,7 @@ type WorkloadStatsSample struct { func (x *WorkloadStatsSample) Reset() { *x = WorkloadStatsSample{} - mi := &file_ateworker_proto_msgTypes[21] + mi := &file_ateworker_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1756,7 +1591,7 @@ func (x *WorkloadStatsSample) String() string { func (*WorkloadStatsSample) ProtoMessage() {} func (x *WorkloadStatsSample) ProtoReflect() protoreflect.Message { - mi := &file_ateworker_proto_msgTypes[21] + mi := &file_ateworker_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1769,7 +1604,7 @@ func (x *WorkloadStatsSample) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkloadStatsSample.ProtoReflect.Descriptor instead. func (*WorkloadStatsSample) Descriptor() ([]byte, []int) { - return file_ateworker_proto_rawDescGZIP(), []int{21} + return file_ateworker_proto_rawDescGZIP(), []int{18} } func (x *WorkloadStatsSample) GetAtespace() string { @@ -1872,7 +1707,7 @@ type GetWorkloadStatsResponse struct { func (x *GetWorkloadStatsResponse) Reset() { *x = GetWorkloadStatsResponse{} - mi := &file_ateworker_proto_msgTypes[22] + mi := &file_ateworker_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1884,7 +1719,7 @@ func (x *GetWorkloadStatsResponse) String() string { func (*GetWorkloadStatsResponse) ProtoMessage() {} func (x *GetWorkloadStatsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateworker_proto_msgTypes[22] + mi := &file_ateworker_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1897,7 +1732,7 @@ func (x *GetWorkloadStatsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetWorkloadStatsResponse.ProtoReflect.Descriptor instead. func (*GetWorkloadStatsResponse) Descriptor() ([]byte, []int) { - return file_ateworker_proto_rawDescGZIP(), []int{22} + return file_ateworker_proto_rawDescGZIP(), []int{19} } func (x *GetWorkloadStatsResponse) GetSample() *WorkloadStatsSample { @@ -1915,7 +1750,7 @@ type GetActiveWorkloadStatsRequest struct { func (x *GetActiveWorkloadStatsRequest) Reset() { *x = GetActiveWorkloadStatsRequest{} - mi := &file_ateworker_proto_msgTypes[23] + mi := &file_ateworker_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1927,7 +1762,7 @@ func (x *GetActiveWorkloadStatsRequest) String() string { func (*GetActiveWorkloadStatsRequest) ProtoMessage() {} func (x *GetActiveWorkloadStatsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateworker_proto_msgTypes[23] + mi := &file_ateworker_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1940,7 +1775,7 @@ func (x *GetActiveWorkloadStatsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetActiveWorkloadStatsRequest.ProtoReflect.Descriptor instead. func (*GetActiveWorkloadStatsRequest) Descriptor() ([]byte, []int) { - return file_ateworker_proto_rawDescGZIP(), []int{23} + return file_ateworker_proto_rawDescGZIP(), []int{20} } type GetActiveWorkloadStatsResponse struct { @@ -1960,7 +1795,7 @@ type GetActiveWorkloadStatsResponse struct { func (x *GetActiveWorkloadStatsResponse) Reset() { *x = GetActiveWorkloadStatsResponse{} - mi := &file_ateworker_proto_msgTypes[24] + mi := &file_ateworker_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1972,7 +1807,7 @@ func (x *GetActiveWorkloadStatsResponse) String() string { func (*GetActiveWorkloadStatsResponse) ProtoMessage() {} func (x *GetActiveWorkloadStatsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateworker_proto_msgTypes[24] + mi := &file_ateworker_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1985,7 +1820,7 @@ func (x *GetActiveWorkloadStatsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetActiveWorkloadStatsResponse.ProtoReflect.Descriptor instead. func (*GetActiveWorkloadStatsResponse) Descriptor() ([]byte, []int) { - return file_ateworker_proto_rawDescGZIP(), []int{24} + return file_ateworker_proto_rawDescGZIP(), []int{21} } func (x *GetActiveWorkloadStatsResponse) GetSamples() []*WorkloadStatsSample { @@ -1999,7 +1834,7 @@ var File_ateworker_proto protoreflect.FileDescriptor const file_ateworker_proto_rawDesc = "" + "\n" + - "\x0fateworker.proto\x12\tateworker\"\xe5\x02\n" + + "\x0fateworker.proto\x12\tateworker\"\x81\x03\n" + "\tActorDirs\x12\x19\n" + "\broot_dir\x18\x01 \x01(\tR\arootDir\x12$\n" + "\x0eoci_bundle_dir\x18\x02 \x01(\tR\fociBundleDir\x12%\n" + @@ -2007,9 +1842,8 @@ const file_ateworker_proto_rawDesc = "" + "\vrestore_dir\x18\x04 \x01(\tR\n" + "restoreDir\x12@\n" + "\x1ddurable_dir_volume_mounts_dir\x18\x05 \x01(\tR\x19durableDirVolumeMountsDir\x12>\n" + - "\x1csystem_info_volume_roots_dir\x18\x06 \x01(\tR\x18systemInfoVolumeRootsDir\x12\x1f\n" + - "\vvolumes_dir\x18\a \x01(\tR\n" + - "volumesDir\x12,\n" + + "\x1csystem_info_volume_roots_dir\x18\x06 \x01(\tR\x18systemInfoVolumeRootsDir\x12;\n" + + "\x1aexternal_volume_mounts_dir\x18\a \x01(\tR\x17externalVolumeMountsDir\x12,\n" + "\x12sandbox_assets_dir\x18\b \x01(\tR\x10sandboxAssetsDir\"\xdb\x02\n" + "\x18TerminateWorkloadRequest\x12\x1a\n" + "\batespace\x18\x01 \x01(\tR\batespace\x12\x1d\n" + @@ -2050,14 +1884,14 @@ const file_ateworker_proto_rawDesc = "" + "\fWorkloadSpec\x124\n" + "\n" + "containers\x18\x01 \x03(\v2\x14.ateworker.ContainerR\n" + - "containers\"\xe6\x03\n" + + "containers\"\xd7\x03\n" + "\tContainer\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x129\n" + - "\fwakeup_probe\x18\x02 \x01(\v2\x16.ateworker.WakeupProbeR\vwakeupProbe\x12[\n" + - "\x19durable_dir_volume_mounts\x18\x03 \x03(\v2 .ateworker.DurableDirVolumeMountR\x16durableDirVolumeMounts\x12B\n" + - "\x11csi_volume_mounts\x18\x04 \x03(\v2\x16.ateworker.VolumeMountR\x0fcsiVolumeMounts\x12[\n" + - "\x19system_info_volume_mounts\x18\x05 \x03(\v2 .ateworker.SystemInfoVolumeMountR\x16systemInfoVolumeMounts\x12K\n" + - "\x13image_volume_mounts\x18\x06 \x03(\v2\x1b.ateworker.ImageVolumeMountR\x11imageVolumeMounts\x12?\n" + + "\fwakeup_probe\x18\x02 \x01(\v2\x16.ateworker.WakeupProbeR\vwakeupProbe\x12Q\n" + + "\x19durable_dir_volume_mounts\x18\x03 \x03(\v2\x16.ateworker.VolumeMountR\x16durableDirVolumeMounts\x12L\n" + + "\x16external_volume_mounts\x18\x04 \x03(\v2\x16.ateworker.VolumeMountR\x14externalVolumeMounts\x12Q\n" + + "\x19system_info_volume_mounts\x18\x05 \x03(\v2\x16.ateworker.VolumeMountR\x16systemInfoVolumeMounts\x12F\n" + + "\x13image_volume_mounts\x18\x06 \x03(\v2\x16.ateworker.VolumeMountR\x11imageVolumeMounts\x12?\n" + "\x0econtainer_spec\x18\a \x01(\v2\x18.ateworker.ContainerSpecR\rcontainerSpec\"\x97\x01\n" + "\rContainerSpec\x12\x12\n" + "\x04args\x18\x01 \x03(\tR\x04args\x12\x15\n" + @@ -2067,25 +1901,9 @@ const file_ateworker_proto_rawDesc = "" + "\x0eResourceLimits\x12!\n" + "\fmemory_bytes\x18\x01 \x01(\x03R\vmemoryBytes\x12\x1d\n" + "\n" + - "cpu_millis\x18\x02 \x01(\x03R\tcpuMillis\"M\n" + - "\vVolumeMount\x12\x1f\n" + - "\vvolume_name\x18\x01 \x01(\tR\n" + - "volumeName\x12\x1d\n" + - "\n" + - "mount_path\x18\x02 \x01(\tR\tmountPath\"W\n" + - "\x15DurableDirVolumeMount\x12\x1f\n" + - "\vvolume_name\x18\x01 \x01(\tR\n" + - "volumeName\x12\x1d\n" + - "\n" + - "mount_path\x18\x02 \x01(\tR\tmountPath\"W\n" + - "\x15SystemInfoVolumeMount\x12\x1f\n" + - "\vvolume_name\x18\x01 \x01(\tR\n" + - "volumeName\x12\x1d\n" + - "\n" + - "mount_path\x18\x02 \x01(\tR\tmountPath\"R\n" + - "\x10ImageVolumeMount\x12\x1f\n" + - "\vvolume_name\x18\x01 \x01(\tR\n" + - "volumeName\x12\x1d\n" + + "cpu_millis\x18\x02 \x01(\x03R\tcpuMillis\"@\n" + + "\vVolumeMount\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x1d\n" + "\n" + "mount_path\x18\x02 \x01(\tR\tmountPath\"k\n" + "\vWakeupProbe\x123\n" + @@ -2195,7 +2013,7 @@ func file_ateworker_proto_rawDescGZIP() []byte { } var file_ateworker_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_ateworker_proto_msgTypes = make([]protoimpl.MessageInfo, 28) +var file_ateworker_proto_msgTypes = make([]protoimpl.MessageInfo, 25) var file_ateworker_proto_goTypes = []any{ (SnapshotFidelity)(0), // 0: ateworker.SnapshotFidelity (StatsSource)(0), // 1: ateworker.StatsSource @@ -2209,64 +2027,61 @@ var file_ateworker_proto_goTypes = []any{ (*ContainerSpec)(nil), // 9: ateworker.ContainerSpec (*ResourceLimits)(nil), // 10: ateworker.ResourceLimits (*VolumeMount)(nil), // 11: ateworker.VolumeMount - (*DurableDirVolumeMount)(nil), // 12: ateworker.DurableDirVolumeMount - (*SystemInfoVolumeMount)(nil), // 13: ateworker.SystemInfoVolumeMount - (*ImageVolumeMount)(nil), // 14: ateworker.ImageVolumeMount - (*WakeupProbe)(nil), // 15: ateworker.WakeupProbe - (*HttpGetAction)(nil), // 16: ateworker.HttpGetAction - (*RunWorkloadResponse)(nil), // 17: ateworker.RunWorkloadResponse - (*CheckpointWorkloadRequest)(nil), // 18: ateworker.CheckpointWorkloadRequest - (*CheckpointWorkloadResponse)(nil), // 19: ateworker.CheckpointWorkloadResponse - (*RestoreWorkloadRequest)(nil), // 20: ateworker.RestoreWorkloadRequest - (*RestoreWorkloadResponse)(nil), // 21: ateworker.RestoreWorkloadResponse - (*GetWorkloadStatsRequest)(nil), // 22: ateworker.GetWorkloadStatsRequest - (*WorkloadStatsSample)(nil), // 23: ateworker.WorkloadStatsSample - (*GetWorkloadStatsResponse)(nil), // 24: ateworker.GetWorkloadStatsResponse - (*GetActiveWorkloadStatsRequest)(nil), // 25: ateworker.GetActiveWorkloadStatsRequest - (*GetActiveWorkloadStatsResponse)(nil), // 26: ateworker.GetActiveWorkloadStatsResponse - nil, // 27: ateworker.RunWorkloadRequest.RuntimeAssetPathsEntry - nil, // 28: ateworker.CheckpointWorkloadRequest.RuntimeAssetPathsEntry - nil, // 29: ateworker.RestoreWorkloadRequest.RuntimeAssetPathsEntry + (*WakeupProbe)(nil), // 12: ateworker.WakeupProbe + (*HttpGetAction)(nil), // 13: ateworker.HttpGetAction + (*RunWorkloadResponse)(nil), // 14: ateworker.RunWorkloadResponse + (*CheckpointWorkloadRequest)(nil), // 15: ateworker.CheckpointWorkloadRequest + (*CheckpointWorkloadResponse)(nil), // 16: ateworker.CheckpointWorkloadResponse + (*RestoreWorkloadRequest)(nil), // 17: ateworker.RestoreWorkloadRequest + (*RestoreWorkloadResponse)(nil), // 18: ateworker.RestoreWorkloadResponse + (*GetWorkloadStatsRequest)(nil), // 19: ateworker.GetWorkloadStatsRequest + (*WorkloadStatsSample)(nil), // 20: ateworker.WorkloadStatsSample + (*GetWorkloadStatsResponse)(nil), // 21: ateworker.GetWorkloadStatsResponse + (*GetActiveWorkloadStatsRequest)(nil), // 22: ateworker.GetActiveWorkloadStatsRequest + (*GetActiveWorkloadStatsResponse)(nil), // 23: ateworker.GetActiveWorkloadStatsResponse + nil, // 24: ateworker.RunWorkloadRequest.RuntimeAssetPathsEntry + nil, // 25: ateworker.CheckpointWorkloadRequest.RuntimeAssetPathsEntry + nil, // 26: ateworker.RestoreWorkloadRequest.RuntimeAssetPathsEntry } var file_ateworker_proto_depIdxs = []int32{ 7, // 0: ateworker.TerminateWorkloadRequest.spec:type_name -> ateworker.WorkloadSpec 2, // 1: ateworker.TerminateWorkloadRequest.actor_dirs:type_name -> ateworker.ActorDirs 7, // 2: ateworker.RunWorkloadRequest.spec:type_name -> ateworker.WorkloadSpec - 27, // 3: ateworker.RunWorkloadRequest.runtime_asset_paths:type_name -> ateworker.RunWorkloadRequest.RuntimeAssetPathsEntry + 24, // 3: ateworker.RunWorkloadRequest.runtime_asset_paths:type_name -> ateworker.RunWorkloadRequest.RuntimeAssetPathsEntry 6, // 4: ateworker.RunWorkloadRequest.egress_gateway:type_name -> ateworker.EgressGateway 2, // 5: ateworker.RunWorkloadRequest.actor_dirs:type_name -> ateworker.ActorDirs 8, // 6: ateworker.WorkloadSpec.containers:type_name -> ateworker.Container - 15, // 7: ateworker.Container.wakeup_probe:type_name -> ateworker.WakeupProbe - 12, // 8: ateworker.Container.durable_dir_volume_mounts:type_name -> ateworker.DurableDirVolumeMount - 11, // 9: ateworker.Container.csi_volume_mounts:type_name -> ateworker.VolumeMount - 13, // 10: ateworker.Container.system_info_volume_mounts:type_name -> ateworker.SystemInfoVolumeMount - 14, // 11: ateworker.Container.image_volume_mounts:type_name -> ateworker.ImageVolumeMount + 12, // 7: ateworker.Container.wakeup_probe:type_name -> ateworker.WakeupProbe + 11, // 8: ateworker.Container.durable_dir_volume_mounts:type_name -> ateworker.VolumeMount + 11, // 9: ateworker.Container.external_volume_mounts:type_name -> ateworker.VolumeMount + 11, // 10: ateworker.Container.system_info_volume_mounts:type_name -> ateworker.VolumeMount + 11, // 11: ateworker.Container.image_volume_mounts:type_name -> ateworker.VolumeMount 9, // 12: ateworker.Container.container_spec:type_name -> ateworker.ContainerSpec 10, // 13: ateworker.ContainerSpec.resources:type_name -> ateworker.ResourceLimits - 16, // 14: ateworker.WakeupProbe.http_get:type_name -> ateworker.HttpGetAction + 13, // 14: ateworker.WakeupProbe.http_get:type_name -> ateworker.HttpGetAction 7, // 15: ateworker.CheckpointWorkloadRequest.spec:type_name -> ateworker.WorkloadSpec - 28, // 16: ateworker.CheckpointWorkloadRequest.runtime_asset_paths:type_name -> ateworker.CheckpointWorkloadRequest.RuntimeAssetPathsEntry + 25, // 16: ateworker.CheckpointWorkloadRequest.runtime_asset_paths:type_name -> ateworker.CheckpointWorkloadRequest.RuntimeAssetPathsEntry 0, // 17: ateworker.CheckpointWorkloadRequest.fidelity:type_name -> ateworker.SnapshotFidelity 2, // 18: ateworker.CheckpointWorkloadRequest.actor_dirs:type_name -> ateworker.ActorDirs 7, // 19: ateworker.RestoreWorkloadRequest.spec:type_name -> ateworker.WorkloadSpec - 29, // 20: ateworker.RestoreWorkloadRequest.runtime_asset_paths:type_name -> ateworker.RestoreWorkloadRequest.RuntimeAssetPathsEntry + 26, // 20: ateworker.RestoreWorkloadRequest.runtime_asset_paths:type_name -> ateworker.RestoreWorkloadRequest.RuntimeAssetPathsEntry 0, // 21: ateworker.RestoreWorkloadRequest.fidelity:type_name -> ateworker.SnapshotFidelity 6, // 22: ateworker.RestoreWorkloadRequest.egress_gateway:type_name -> ateworker.EgressGateway 2, // 23: ateworker.RestoreWorkloadRequest.actor_dirs:type_name -> ateworker.ActorDirs 1, // 24: ateworker.WorkloadStatsSample.source:type_name -> ateworker.StatsSource - 23, // 25: ateworker.GetWorkloadStatsResponse.sample:type_name -> ateworker.WorkloadStatsSample - 23, // 26: ateworker.GetActiveWorkloadStatsResponse.samples:type_name -> ateworker.WorkloadStatsSample + 20, // 25: ateworker.GetWorkloadStatsResponse.sample:type_name -> ateworker.WorkloadStatsSample + 20, // 26: ateworker.GetActiveWorkloadStatsResponse.samples:type_name -> ateworker.WorkloadStatsSample 5, // 27: ateworker.Worker.RunWorkload:input_type -> ateworker.RunWorkloadRequest - 18, // 28: ateworker.Worker.CheckpointWorkload:input_type -> ateworker.CheckpointWorkloadRequest - 20, // 29: ateworker.Worker.RestoreWorkload:input_type -> ateworker.RestoreWorkloadRequest - 22, // 30: ateworker.Worker.GetWorkloadStats:input_type -> ateworker.GetWorkloadStatsRequest - 25, // 31: ateworker.Worker.GetActiveWorkloadStats:input_type -> ateworker.GetActiveWorkloadStatsRequest + 15, // 28: ateworker.Worker.CheckpointWorkload:input_type -> ateworker.CheckpointWorkloadRequest + 17, // 29: ateworker.Worker.RestoreWorkload:input_type -> ateworker.RestoreWorkloadRequest + 19, // 30: ateworker.Worker.GetWorkloadStats:input_type -> ateworker.GetWorkloadStatsRequest + 22, // 31: ateworker.Worker.GetActiveWorkloadStats:input_type -> ateworker.GetActiveWorkloadStatsRequest 3, // 32: ateworker.Worker.TerminateWorkload:input_type -> ateworker.TerminateWorkloadRequest - 17, // 33: ateworker.Worker.RunWorkload:output_type -> ateworker.RunWorkloadResponse - 19, // 34: ateworker.Worker.CheckpointWorkload:output_type -> ateworker.CheckpointWorkloadResponse - 21, // 35: ateworker.Worker.RestoreWorkload:output_type -> ateworker.RestoreWorkloadResponse - 24, // 36: ateworker.Worker.GetWorkloadStats:output_type -> ateworker.GetWorkloadStatsResponse - 26, // 37: ateworker.Worker.GetActiveWorkloadStats:output_type -> ateworker.GetActiveWorkloadStatsResponse + 14, // 33: ateworker.Worker.RunWorkload:output_type -> ateworker.RunWorkloadResponse + 16, // 34: ateworker.Worker.CheckpointWorkload:output_type -> ateworker.CheckpointWorkloadResponse + 18, // 35: ateworker.Worker.RestoreWorkload:output_type -> ateworker.RestoreWorkloadResponse + 21, // 36: ateworker.Worker.GetWorkloadStats:output_type -> ateworker.GetWorkloadStatsResponse + 23, // 37: ateworker.Worker.GetActiveWorkloadStats:output_type -> ateworker.GetActiveWorkloadStatsResponse 4, // 38: ateworker.Worker.TerminateWorkload:output_type -> ateworker.TerminateWorkloadResponse 33, // [33:39] is the sub-list for method output_type 27, // [27:33] is the sub-list for method input_type @@ -2281,14 +2096,14 @@ func file_ateworker_proto_init() { return } file_ateworker_proto_msgTypes[3].OneofWrappers = []any{} - file_ateworker_proto_msgTypes[18].OneofWrappers = []any{} + file_ateworker_proto_msgTypes[15].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_ateworker_proto_rawDesc), len(file_ateworker_proto_rawDesc)), NumEnums: 2, - NumMessages: 28, + NumMessages: 25, NumExtensions: 0, NumServices: 1, }, diff --git a/internal/proto/ateworkerpb/ateworker.proto b/internal/proto/ateworkerpb/ateworker.proto index 8fb4825449..3a51bc76d0 100644 --- a/internal/proto/ateworkerpb/ateworker.proto +++ b/internal/proto/ateworkerpb/ateworker.proto @@ -122,8 +122,8 @@ message ActorDirs { string durable_dir_volume_mounts_dir = 5; // Holds one subdirectory per system-info volume. string system_info_volume_roots_dir = 6; - // Holds one mount point per CSI volume. - string volumes_dir = 7; + // Holds one mount point per external volume. + string external_volume_mounts_dir = 7; // Holds the sandbox runtime assets staged for this actor (runsc, or the // micro-VM kernel, VMM and guest image), hard-linked from atelet's asset // cache. The only location a sandbox runtime may run a binary from; the @@ -193,18 +193,20 @@ message Container { // durable_dir_volume_mounts are the durable-dir volumes this container // mounts, if any. - repeated DurableDirVolumeMount durable_dir_volume_mounts = 3; + repeated VolumeMount durable_dir_volume_mounts = 3; - // csi_volume_mounts are the CSI volumes this container mounts, if any. - repeated VolumeMount csi_volume_mounts = 4; + // external_volume_mounts are the external volumes this container mounts, if + // any. + repeated VolumeMount external_volume_mounts = 4; // system_info_volume_mounts are the system-info volumes this container - // mounts, if any. Contents are generated by atelet on the host; the - // container sees them read-only. - repeated SystemInfoVolumeMount system_info_volume_mounts = 5; + // mounts, if any. Contents are generated by atelet on the host on every + // Run/Restore and never captured into snapshots. The container sees them + // read-only. + repeated VolumeMount system_info_volume_mounts = 5; // image_volume_mounts are the image volumes this container mounts, if any. - repeated ImageVolumeMount image_volume_mounts = 6; + repeated VolumeMount image_volume_mounts = 6; // container_spec is what the worker builds the container's OCI spec from, // besides its volume mounts. Unset on RPCs that start no process. @@ -238,36 +240,10 @@ message ResourceLimits { // VolumeMount is one volume mounted into a container. message VolumeMount { - string volume_name = 1; - string mount_path = 2; -} - -// DurableDirVolumeMount is one durable-dir volume mounted into a container. -message DurableDirVolumeMount { - // volume_name is the name the ActorTemplate gave the volume. It selects the - // per-volume directory atelet prepared for the actor on the host. - string volume_name = 1; - // mount_path is where the container sees the volume. - string mount_path = 2; -} - -// SystemInfoVolumeMount is one system-info volume mounted (read-only) into a -// container. Unlike durable-dir volumes, system-info contents are generated -// by atelet on every Run/Restore and are never captured into snapshots. -message SystemInfoVolumeMount { - // volume_name is the name the ActorTemplate gave the volume. It selects the - // per-volume directory atelet prepared for the actor on the host. - string volume_name = 1; - // mount_path is where the container sees the volume. - string mount_path = 2; -} - -// ImageVolumeMount is one image volume mounted into a container. The worker uses -// these to construct the container's volume mounts — each names the volume -// and its destination path. -message ImageVolumeMount { - // volume_name is the name the ActorTemplate gave the volume. - string volume_name = 1; + // name is the name the ActorTemplate gave the volume. For durable-dir and + // system-info volumes it selects the per-volume directory atelet prepared + // for the actor on the host. + string name = 1; // mount_path is where the container sees the volume. string mount_path = 2; } diff --git a/internal/protoredact/protoredact_test.go b/internal/protoredact/protoredact_test.go index 28c9ef9d92..ce8057df31 100644 --- a/internal/protoredact/protoredact_test.go +++ b/internal/protoredact/protoredact_test.go @@ -55,7 +55,7 @@ func TestRedactedRecursesIntoRealMapFields(t *testing.T) { }, Spec: &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{ Name: "main", - Env: []*ateletpb.EnvEntry{{Name: "API_KEY", Value: "sk-secret"}}, + Env: []*ateletpb.EnvVar{{Name: "API_KEY", Value: "sk-secret"}}, }}}, } redacted, _ := protoredact.Redacted(run) @@ -304,7 +304,7 @@ func TestRedactedReturnsMessagesWithNothingToMaskWithoutCopying(t *testing.T) { &ateapipb.ActorTemplate{Containers: []*ateapipb.Container{{Name: "c", Image: "img"}}}, &ateapipb.ListActorTemplatesResponse{ActorTemplates: []*ateapipb.ActorTemplate{{Containers: []*ateapipb.Container{{Name: "c"}}}}}, &ateapipb.MintActorJwtResponse{}, - &ateletpb.RunRequest{Spec: &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Env: []*ateletpb.EnvEntry{{Name: "K"}}}}}}, + &ateletpb.RunRequest{Spec: &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Env: []*ateletpb.EnvVar{{Name: "K"}}}}}}, } { if protoredact.NeedsRedaction(m) { t.Errorf("%T: NeedsRedaction should be false", m) @@ -318,7 +318,7 @@ func TestRedactedReturnsMessagesWithNothingToMaskWithoutCopying(t *testing.T) { &ateapipb.MintActorJwtResponse{ActorJwt: "t"}, &ateapipb.ActorTemplate{Containers: []*ateapipb.Container{{Env: []*ateapipb.EnvVar{{Name: "K", Value: "v"}}}}}, &ateapipb.ListActorTemplatesResponse{ActorTemplates: []*ateapipb.ActorTemplate{{}, {Containers: []*ateapipb.Container{{}, {Env: []*ateapipb.EnvVar{{Name: "K", Value: "v"}}}}}}}, - &ateletpb.RunRequest{Spec: &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Env: []*ateletpb.EnvEntry{{Name: "K", Value: "v"}}}}}}, + &ateletpb.RunRequest{Spec: &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Env: []*ateletpb.EnvVar{{Name: "K", Value: "v"}}}}}}, } { if !protoredact.NeedsRedaction(m) { t.Errorf("%T: NeedsRedaction should be true", m) @@ -356,7 +356,7 @@ func TestNeedsRedactionIsSafeUnderConcurrentFirstUse(t *testing.T) { &ateapipb.ListActorsResponse{}, &ateapipb.ActorTemplate{Containers: []*ateapipb.Container{{Env: []*ateapipb.EnvVar{{Name: "K", Value: "v"}}}}}, &ateapipb.Worker{}, - &ateletpb.RunRequest{Spec: &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Env: []*ateletpb.EnvEntry{{Name: "K", Value: "v"}}}}}}, + &ateletpb.RunRequest{Spec: &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Env: []*ateletpb.EnvVar{{Name: "K", Value: "v"}}}}}}, } want := []bool{true, false, true, false, true} var wg sync.WaitGroup @@ -421,7 +421,7 @@ func TestDebugRedactFieldsArePinned(t *testing.T) { "ateapi.MintActorJwtResponse.actor_jwt": true, "ateapi.MintWorkerActorJwtResponse.actor_jwt": true, "atelet.AssetFile.url": true, - "atelet.EnvEntry.value": true, + "atelet.EnvVar.value": true, "ateworker.ContainerSpec.env": true, "credprovider.FetchSecretResponse.opaque_bytes": true, "objectstore.v1.FetchAssetRequest.asset_uri": true, diff --git a/internal/resources/validate.go b/internal/resources/validate.go index aeefd1215c..98c1e04fb8 100644 --- a/internal/resources/validate.go +++ b/internal/resources/validate.go @@ -121,7 +121,7 @@ func ValidateActorDirs(actorDirs *ateworkerpb.ActorDirs, fldPath *field.Path) fi {"restore_dir", actorDirs.GetRestoreDir()}, {"durable_dir_volume_mounts_dir", actorDirs.GetDurableDirVolumeMountsDir()}, {"system_info_volume_roots_dir", actorDirs.GetSystemInfoVolumeRootsDir()}, - {"volumes_dir", actorDirs.GetVolumesDir()}, + {"external_volume_mounts_dir", actorDirs.GetExternalVolumeMountsDir()}, {"sandbox_assets_dir", actorDirs.GetSandboxAssetsDir()}, } { errs = append(errs, validateAbsDir(actorDir.path, fldPath.Child(actorDir.name))...) diff --git a/internal/resources/validate_test.go b/internal/resources/validate_test.go index 172cce0fea..41b8029a98 100644 --- a/internal/resources/validate_test.go +++ b/internal/resources/validate_test.go @@ -322,7 +322,7 @@ func testActorDirs() *ateworkerpb.ActorDirs { RestoreDir: "/node/actors/a/restore-state", DurableDirVolumeMountsDir: "/node/actors/a/durable-dir", SystemInfoVolumeRootsDir: "/node/actors/a/system-info", - VolumesDir: "/node/actors/a/volumes", + ExternalVolumeMountsDir: "/node/actors/a/volumes", SandboxAssetsDir: "/node/actors/a/sandbox-assets", } } From 08779062cd761d1a9a6a6526de33dbb86ca228ff Mon Sep 17 00:00:00 2001 From: Max Thompson Date: Fri, 16 Oct 2026 17:19:39 +0000 Subject: [PATCH 5/6] Add ate-idp-server to serve the discovery document and key set for actor JWTs (#2099) Part of #1756. - Adds `cmd/ate-idp-server`, which serves `/.well-known/openid-configuration` and `/openid/v1/jwks` over HTTPS with a servicedns credential bundle, so relying parties can verify actor JWTs. - Reads the actor JWT authority pool from `--actor-id-jwt-pool` with `localjwtauthority.RefreshingPool`, the reader ate-api-server signs with, and builds each document from it per request. - Builds both documents at startup and exits if it can't. After that, a request whose document can't be built gets a 500 rather than an empty or stale set. - Requires `--issuer` rather than deriving a default, so it can't disagree with ateapi, and accepts only `https://`, so the routes are fixed. The check is the new `actoridjwt.ValidateIssuer`; the discovery document moves out of `internal/oidcdiscovery` into the server, its only user. - Makes the EC case in `localjwtauthority.JWKS` explicit: P-256 with fixed 32-byte coordinates and an error for any other curve, rather than reading the coordinate size off the encoded point. - Records in `docs/metrics/substrate.yaml` that it emits no metrics. For reviewers: the pool holds the signing keys, so ate-idp-server will mount the same Secret ate-api-server does, in place of the controller and ConfigMap in #2134. ate-api-server and ate-setup still accept an issuer with a path until the installer deploys ate-idp-server. It serves `/healthz` but no `/readyz`. Nothing deploys it yet. Testing: unit tests for routing, the host-only issuer check, the discovery document, startup failures, key changes, 500s on a broken set, the P-256 encoding, and required flags, plus a local run over TLS against a generated pool. E2E can't cover it until it's deployed. - [ ] Tests pass - [ ] Appropriate changes to documentation are included in the PR --- cmd/ate-idp-server/internal/server/server.go | 116 +++++++++ .../internal/server/server_test.go | 232 ++++++++++++++++++ cmd/ate-idp-server/main.go | 123 ++++++++++ cmd/ate-idp-server/main_test.go | 36 +++ docs/metrics/substrate.yaml | 6 + internal/actoridjwt/issuer.go | 34 +++ internal/actoridjwt/issuer_test.go | 56 +++++ internal/localjwtauthority/jwks.go | 26 +- internal/localjwtauthority/jwks_test.go | 10 + internal/oidcdiscovery/discovery.go | 63 ----- internal/oidcdiscovery/discovery_test.go | 113 --------- 11 files changed, 628 insertions(+), 187 deletions(-) create mode 100644 cmd/ate-idp-server/internal/server/server.go create mode 100644 cmd/ate-idp-server/internal/server/server_test.go create mode 100644 cmd/ate-idp-server/main.go create mode 100644 cmd/ate-idp-server/main_test.go create mode 100644 internal/actoridjwt/issuer.go create mode 100644 internal/actoridjwt/issuer_test.go delete mode 100644 internal/oidcdiscovery/discovery.go delete mode 100644 internal/oidcdiscovery/discovery_test.go diff --git a/cmd/ate-idp-server/internal/server/server.go b/cmd/ate-idp-server/internal/server/server.go new file mode 100644 index 0000000000..9d72354da7 --- /dev/null +++ b/cmd/ate-idp-server/internal/server/server.go @@ -0,0 +1,116 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package server serves the actor JWT issuer's OpenID discovery document and +// JWK set. +package server + +import ( + "encoding/json" + "fmt" + "log/slog" + "net/http" + "slices" + + "go.ate.dev/substrate/internal/actoridjwt" + "go.ate.dev/substrate/internal/localjwtauthority" +) + +// KeySource supplies the verification keys to publish. +// *localjwtauthority.RefreshingPool is one. +type KeySource interface { + VerificationKeys() ([]*localjwtauthority.VerificationKey, error) +} + +// Server serves the discovery document at /.well-known/openid-configuration +// and the JWK set at /openid/v1/jwks, plus /healthz. It builds both documents +// from its KeySource on every request and answers 500 when that fails. +type Server struct { + issuer string + keys KeySource +} + +// New returns a Server that publishes keys for issuer, which must be +// https://. It fails if the keys cannot be published. +func New(issuer string, keys KeySource) (*Server, error) { + if err := actoridjwt.ValidateIssuer(issuer); err != nil { + return nil, err + } + s := &Server{issuer: issuer, keys: keys} + if _, _, err := s.build(); err != nil { + return nil, fmt.Errorf("cannot publish the keys: %w", err) + } + return s, nil +} + +// Register adds the server's routes to mux. +func (s *Server) Register(mux *http.ServeMux) { + mux.HandleFunc("GET /.well-known/openid-configuration", func(w http.ResponseWriter, r *http.Request) { + discovery, _, err := s.build() + serveDocument(w, r, discovery, err) + }) + mux.HandleFunc("GET /openid/v1/jwks", func(w http.ResponseWriter, r *http.Request) { + _, jwks, err := s.build() + serveDocument(w, r, jwks, err) + }) + mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + }) +} + +type discoveryDocument struct { + Issuer string `json:"issuer"` + JWKSURI string `json:"jwks_uri"` + ResponseTypesSupported []string `json:"response_types_supported"` + SubjectTypesSupported []string `json:"subject_types_supported"` + IDTokenSigningAlgValuesSupported []string `json:"id_token_signing_alg_values_supported"` + ClaimsSupported []string `json:"claims_supported"` +} + +func (s *Server) build() (discovery, jwks []byte, err error) { + verificationKeys, err := s.keys.VerificationKeys() + if err != nil { + return nil, nil, fmt.Errorf("reading verification keys: %w", err) + } + if jwks, err = localjwtauthority.JWKS(verificationKeys); err != nil { + return nil, nil, err + } + algorithms := make([]string, 0, len(verificationKeys)) + for _, vk := range verificationKeys { + algorithms = append(algorithms, vk.Algorithm) + } + slices.Sort(algorithms) + discovery, err = json.Marshal(discoveryDocument{ + Issuer: s.issuer, + JWKSURI: s.issuer + "/openid/v1/jwks", + ResponseTypesSupported: []string{"id_token"}, + SubjectTypesSupported: []string{"public"}, + IDTokenSigningAlgValuesSupported: slices.Compact(algorithms), + ClaimsSupported: []string{"iss", "sub", "aud", "exp", "nbf", "iat", "jti", "actor.ate.dev", "worker-for-actor.ate.dev"}, + }) + if err != nil { + return nil, nil, err + } + return discovery, jwks, nil +} + +func serveDocument(w http.ResponseWriter, r *http.Request, doc []byte, err error) { + if err != nil { + slog.ErrorContext(r.Context(), "Cannot publish the actor JWT keys", slog.Any("err", err)) + http.Error(w, "cannot publish the key set", http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json; charset=utf-8") + _, _ = w.Write(doc) +} diff --git a/cmd/ate-idp-server/internal/server/server_test.go b/cmd/ate-idp-server/internal/server/server_test.go new file mode 100644 index 0000000000..c6ab6d17f9 --- /dev/null +++ b/cmd/ate-idp-server/internal/server/server_test.go @@ -0,0 +1,232 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/google/go-cmp/cmp" + + "go.ate.dev/substrate/internal/localjwtauthority" +) + +// fakeKeys is a KeySource whose keys and error a test can change. +type fakeKeys struct { + mu sync.Mutex + keys []*localjwtauthority.VerificationKey + err error +} + +func (f *fakeKeys) VerificationKeys() ([]*localjwtauthority.VerificationKey, error) { + f.mu.Lock() + defer f.mu.Unlock() + return f.keys, f.err +} + +func (f *fakeKeys) set(keys []*localjwtauthority.VerificationKey, err error) { + f.mu.Lock() + defer f.mu.Unlock() + f.keys, f.err = keys, err +} + +// verificationKeys generates one key per algorithm. +func verificationKeys(t *testing.T, algorithms ...string) []*localjwtauthority.VerificationKey { + t.Helper() + pool := &localjwtauthority.ConcretePool{} + for _, alg := range algorithms { + authority, err := localjwtauthority.GenerateAuthority(alg, "") + if err != nil { + t.Fatal(err) + } + pool.Authorities = append(pool.Authorities, authority) + } + keys, err := pool.VerificationKeys() + if err != nil { + t.Fatal(err) + } + return keys +} + +func keySet(t *testing.T, keys []*localjwtauthority.VerificationKey) string { + t.Helper() + jwks, err := localjwtauthority.JWKS(keys) + if err != nil { + t.Fatal(err) + } + return string(jwks) +} + +func newServer(t *testing.T, issuer string, keys KeySource) http.Handler { + t.Helper() + s, err := New(issuer, keys) + if err != nil { + t.Fatal(err) + } + mux := http.NewServeMux() + s.Register(mux) + return mux +} + +func get(h http.Handler, method, target string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(method, target, nil)) + return w +} + +func TestServesDocuments(t *testing.T) { + keys := verificationKeys(t, "ES256") + s := newServer(t, "https://idp.ate-system.svc", &fakeKeys{keys: keys}) + tests := []struct { + target string + want string + }{ + { + target: "/.well-known/openid-configuration", + want: `{"issuer":"https://idp.ate-system.svc",` + + `"jwks_uri":"https://idp.ate-system.svc/openid/v1/jwks",` + + `"response_types_supported":["id_token"],` + + `"subject_types_supported":["public"],` + + `"id_token_signing_alg_values_supported":["ES256"],` + + `"claims_supported":["iss","sub","aud","exp","nbf","iat","jti","actor.ate.dev","worker-for-actor.ate.dev"]}`, + }, + {target: "/openid/v1/jwks", want: keySet(t, keys)}, + } + for _, tt := range tests { + w := get(s, http.MethodGet, tt.target) + if w.Code != http.StatusOK { + t.Fatalf("GET %s = %d, want 200", tt.target, w.Code) + } + if diff := cmp.Diff(tt.want, w.Body.String()); diff != "" { + t.Errorf("GET %s body (-want +got):\n%s", tt.target, diff) + } + if got := w.Header().Get("Content-Type"); got != "application/json; charset=utf-8" { + t.Errorf("GET %s: Content-Type = %q, want application/json; charset=utf-8", tt.target, got) + } + } +} + +func TestNewFails(t *testing.T) { + keys := verificationKeys(t, "ES256") + tests := map[string]struct { + issuer string + keys KeySource + }{ + "issuer not https": {issuer: "http://idp.ate-system.svc", keys: &fakeKeys{keys: keys}}, + "issuer with path": {issuer: "https://idp.example.com/prod", keys: &fakeKeys{keys: keys}}, + "keys unreadable": {issuer: "https://idp.ate-system.svc", keys: &fakeKeys{err: os.ErrNotExist}}, + "no keys": {issuer: "https://idp.ate-system.svc", keys: &fakeKeys{}}, + } + for name, tt := range tests { + if _, err := New(tt.issuer, tt.keys); err == nil { + t.Errorf("%s: New returned nil error", name) + } + } +} + +func TestFollowsKeyChanges(t *testing.T) { + first := verificationKeys(t, "ES256") + keys := &fakeKeys{keys: first} + s := newServer(t, "https://idp.ate-system.svc", keys) + if got, want := get(s, http.MethodGet, "/openid/v1/jwks").Body.String(), keySet(t, first); got != want { + t.Fatalf("served key set = %q, want %q", got, want) + } + + rotated := append(verificationKeys(t, "RS256"), first...) + keys.set(rotated, nil) + if got, want := get(s, http.MethodGet, "/openid/v1/jwks").Body.String(), keySet(t, rotated); got != want { + t.Errorf("after rotation: served key set = %q, want %q", got, want) + } + + for name, change := range map[string]func(){ + "read error": func() { keys.set(nil, os.ErrNotExist) }, + "empty pool": func() { keys.set(nil, nil) }, + } { + change() + for _, target := range []string{"/.well-known/openid-configuration", "/openid/v1/jwks"} { + if got := get(s, http.MethodGet, target).Code; got != http.StatusInternalServerError { + t.Errorf("after %s: GET %s = %d, want 500", name, target, got) + } + } + } +} + +func TestDiscoveryAdvertisesPoolAlgorithms(t *testing.T) { + s := newServer(t, "https://idp.ate-system.svc", &fakeKeys{keys: verificationKeys(t, "RS256", "ES256", "ES256")}) + var doc struct { + Algorithms []string `json:"id_token_signing_alg_values_supported"` + } + if err := json.Unmarshal(get(s, http.MethodGet, "/.well-known/openid-configuration").Body.Bytes(), &doc); err != nil { + t.Fatal(err) + } + if diff := cmp.Diff([]string{"ES256", "RS256"}, doc.Algorithms); diff != "" { + t.Errorf("algorithms (-want +got):\n%s", diff) + } +} + +func TestMethodsAndUnknownPaths(t *testing.T) { + s := newServer(t, "https://idp.ate-system.svc", &fakeKeys{keys: verificationKeys(t, "ES256")}) + + if got := get(s, http.MethodGet, "/healthz").Code; got != http.StatusOK { + t.Errorf("GET /healthz = %d, want 200", got) + } + + if got := get(s, http.MethodHead, "/openid/v1/jwks").Code; got != http.StatusOK { + t.Errorf("HEAD = %d, want 200", got) + } + + w := get(s, http.MethodPost, "/openid/v1/jwks") + if w.Code != http.StatusMethodNotAllowed || w.Header().Get("Allow") != "GET, HEAD" { + t.Errorf("POST = %d, Allow %q; want 405, Allow GET, HEAD", w.Code, w.Header().Get("Allow")) + } + + if got := get(s, http.MethodGet, "/openid/v1/other").Code; got != http.StatusNotFound { + t.Errorf("GET unknown path = %d, want 404", got) + } +} + +func TestPublishesRefreshingPool(t *testing.T) { + wire, keyID, err := localjwtauthority.GeneratePool("ES256", "") + if err != nil { + t.Fatal(err) + } + file := filepath.Join(t.TempDir(), "pool.json") + if err := os.WriteFile(file, wire, 0o600); err != nil { + t.Fatal(err) + } + pool, err := localjwtauthority.NewRefreshingPool(file) + if err != nil { + t.Fatal(err) + } + s := newServer(t, "https://idp.ate-system.svc", pool) + + var set struct { + Keys []map[string]string `json:"keys"` + } + if err := json.Unmarshal(get(s, http.MethodGet, "/openid/v1/jwks").Body.Bytes(), &set); err != nil { + t.Fatal(err) + } + if len(set.Keys) != 1 { + t.Fatalf("served %d keys, want 1", len(set.Keys)) + } + if k := set.Keys[0]; k["kid"] != keyID || k["alg"] != "ES256" || k["d"] != "" { + t.Errorf("served key %v; want kid %q, alg ES256, and no private key", k, keyID) + } +} diff --git a/cmd/ate-idp-server/main.go b/cmd/ate-idp-server/main.go new file mode 100644 index 0000000000..8ab03fe758 --- /dev/null +++ b/cmd/ate-idp-server/main.go @@ -0,0 +1,123 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Command ate-idp-server serves the OpenID Connect discovery document and JWK +// set that relying parties use to verify actor JWTs. It publishes the public +// keys of the authority pool in --actor-id-jwt-pool, which it rereads at most +// once a minute. +package main + +import ( + "context" + "crypto/tls" + "fmt" + "log/slog" + "net/http" + "os/signal" + "slices" + "strings" + "syscall" + "time" + + "github.com/spf13/pflag" + + "go.ate.dev/substrate/cmd/ate-idp-server/internal/server" + "go.ate.dev/substrate/internal/credbundle" + "go.ate.dev/substrate/internal/localjwtauthority" + "go.ate.dev/substrate/internal/serverboot" + "go.ate.dev/substrate/internal/version" +) + +var ( + issuer = pflag.String("issuer", "", "Actor JWT issuer URL, https://; must match ate-api-server's --actor-jwt-issuer (required).") + poolFile = pflag.String("actor-id-jwt-pool", "", "File holding the actor JWT authority pool that ate-api-server signs with (required).") + listenAddr = pflag.String("listen-address", ":8443", "HTTPS listen address.") + serverBundle = pflag.String("server-cred-bundle", "", "Credential bundle (PEM key and chain) presented for serving TLS (required).") + logLevel = pflag.String("log-level", "info", "One of debug, info, warn, error.") + drainGrace = pflag.Duration("drain-grace", 5*time.Second, "How long to wait for in-flight requests on shutdown.") +) + +func main() { + pflag.Parse() + + ctx := context.Background() + serverboot.InitLogger() + if err := serverboot.SetLogLevel(*logLevel); err != nil { + serverboot.Fatal(ctx, "invalid --log-level", err) + } + slog.InfoContext(ctx, "starting ate-idp-server", slog.String("version", version.String())) + + if err := run(ctx); err != nil { + serverboot.Fatal(ctx, "ate-idp-server exited with error", err) + } +} + +func run(ctx context.Context) error { + if missing := missingFlags(map[string]string{ + "--issuer": *issuer, + "--actor-id-jwt-pool": *poolFile, + "--server-cred-bundle": *serverBundle, + }); len(missing) > 0 { + return fmt.Errorf("required flags not set: %s", strings.Join(missing, ", ")) + } + pool, err := localjwtauthority.NewRefreshingPool(*poolFile) + if err != nil { + return fmt.Errorf("loading --actor-id-jwt-pool: %w", err) + } + srv, err := server.New(*issuer, pool) + if err != nil { + return err + } + mux := http.NewServeMux() + srv.Register(mux) + slog.InfoContext(ctx, "serving issuer", slog.String("issuer", *issuer)) + + ctx, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) + defer stop() + + httpSrv := &http.Server{ + Addr: *listenAddr, + Handler: mux, + ReadHeaderTimeout: 10 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + TLSConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + GetCertificate: credbundle.Loader(*serverBundle), + }, + } + serveErr := make(chan error, 1) + go func() { serveErr <- httpSrv.ListenAndServeTLS("", "") }() + + select { + case err := <-serveErr: + return fmt.Errorf("serving on %s: %w", *listenAddr, err) + case <-ctx.Done(): + } + shutdownCtx, cancel := context.WithTimeout(context.Background(), *drainGrace) + defer cancel() + return httpSrv.Shutdown(shutdownCtx) +} + +// missingFlags returns the sorted names of the flags whose values are empty. +func missingFlags(flags map[string]string) []string { + var missing []string + for name, value := range flags { + if value == "" { + missing = append(missing, name) + } + } + slices.Sort(missing) + return missing +} diff --git a/cmd/ate-idp-server/main_test.go b/cmd/ate-idp-server/main_test.go new file mode 100644 index 0000000000..5465a2416a --- /dev/null +++ b/cmd/ate-idp-server/main_test.go @@ -0,0 +1,36 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestMissingFlags(t *testing.T) { + tests := []struct { + flags map[string]string + want []string + }{ + {flags: map[string]string{"--issuer": "https://idp.ate-system.svc", "--actor-id-jwt-pool": "/run/actor-id-jwt-pool/pool.json"}, want: nil}, + {flags: map[string]string{"--actor-id-jwt-pool": "", "--issuer": "", "--server-cred-bundle": "/run/bundle.pem"}, want: []string{"--actor-id-jwt-pool", "--issuer"}}, + } + for _, tt := range tests { + if diff := cmp.Diff(tt.want, missingFlags(tt.flags)); diff != "" { + t.Errorf("missingFlags(%v) (-want +got):\n%s", tt.flags, diff) + } + } +} diff --git a/docs/metrics/substrate.yaml b/docs/metrics/substrate.yaml index a7e229ddf4..f5e1aba961 100644 --- a/docs/metrics/substrate.yaml +++ b/docs/metrics/substrate.yaml @@ -242,6 +242,12 @@ blind_spots: brief: > This plugin has no metrics. It does not install the otelgrpc server handler. Thus rpc.server.call.duration has no credprovider series for it. + - area: ate-idp-server + code: cmd/ate-idp-server + brief: > + This component has no metrics. A relying party that cannot fetch the + discovery document or key set sees only failed token verification; check + the server's "Cannot publish the actor JWT keys" logs. - area: golden snapshot pipeline code: cmd/ateapi/internal/controlapi/template_reconciler.go brief: The template reconciler in ate-api has no instruments. Thus nothing measures a build of a golden image. diff --git a/internal/actoridjwt/issuer.go b/internal/actoridjwt/issuer.go new file mode 100644 index 0000000000..8ca8b4f630 --- /dev/null +++ b/internal/actoridjwt/issuer.go @@ -0,0 +1,34 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package actoridjwt + +import ( + "fmt" + "net/url" +) + +// ValidateIssuer returns an error unless issuer is https:// followed by a host +// and optional port, with nothing after it. ate-idp-server serves the +// discovery document and key set at fixed paths under such an issuer. +func ValidateIssuer(issuer string) error { + u, err := url.Parse(issuer) + if err != nil { + return fmt.Errorf("invalid issuer: %w", err) + } + if u.Hostname() == "" || issuer != "https://"+u.Host { + return fmt.Errorf("issuer %q must be https://, with no path, query, or fragment", issuer) + } + return nil +} diff --git a/internal/actoridjwt/issuer_test.go b/internal/actoridjwt/issuer_test.go new file mode 100644 index 0000000000..48aec17d9e --- /dev/null +++ b/internal/actoridjwt/issuer_test.go @@ -0,0 +1,56 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package actoridjwt + +import "testing" + +func TestValidateIssuer(t *testing.T) { + tests := []struct { + name string + issuer string + wantErr bool + }{ + {name: "in-cluster default", issuer: "https://idp.ate-system.svc"}, + {name: "public host", issuer: "https://idp.example.com"}, + {name: "port", issuer: "https://idp.example.com:8443"}, + + {name: "empty", issuer: "", wantErr: true}, + {name: "http", issuer: "http://idp.example.com", wantErr: true}, + {name: "uppercase scheme", issuer: "HTTPS://idp.example.com", wantErr: true}, + {name: "no scheme", issuer: "idp.example.com", wantErr: true}, + {name: "no host", issuer: "https:///prod", wantErr: true}, + {name: "port without host", issuer: "https://:443", wantErr: true}, + {name: "opaque", issuer: "https:idp.example.com", wantErr: true}, + {name: "user info", issuer: "https://user:pass@idp.example.com", wantErr: true}, + {name: "trailing slash", issuer: "https://idp.example.com/", wantErr: true}, + {name: "path", issuer: "https://idp.example.com/prod", wantErr: true}, + {name: "query", issuer: "https://idp.example.com?cluster=prod", wantErr: true}, + {name: "empty query", issuer: "https://idp.example.com?", wantErr: true}, + {name: "fragment", issuer: "https://idp.example.com#prod", wantErr: true}, + {name: "empty fragment", issuer: "https://idp.example.com#", wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateIssuer(tt.issuer) + if tt.wantErr && err == nil { + t.Errorf("ValidateIssuer(%q) returned nil, want error", tt.issuer) + } + if !tt.wantErr && err != nil { + t.Errorf("ValidateIssuer(%q) returned error: %v", tt.issuer, err) + } + }) + } +} diff --git a/internal/localjwtauthority/jwks.go b/internal/localjwtauthority/jwks.go index 8f857ba972..f9fc6d258d 100644 --- a/internal/localjwtauthority/jwks.go +++ b/internal/localjwtauthority/jwks.go @@ -97,23 +97,27 @@ func JWKS(keys []*VerificationKey) ([]byte, error) { // toJWK encodes a key that algorithmFits has accepted. func toJWK(key *VerificationKey) (jwk, error) { - b64 := base64.RawURLEncoding.EncodeToString j := jwk{KeyID: key.KeyID, Use: "sig", Algorithm: key.Algorithm} switch k := key.PublicKey.(type) { case *rsa.PublicKey: j.KeyType = "RSA" - j.N = b64(k.N.Bytes()) - j.E = b64(big.NewInt(int64(k.E)).Bytes()) + j.N = base64.RawURLEncoding.EncodeToString(k.N.Bytes()) + j.E = base64.RawURLEncoding.EncodeToString(big.NewInt(int64(k.E)).Bytes()) case *ecdsa.PublicKey: - // The uncompressed point is 0x04 || x || y, each coordinate padded to - // the curve size as RFC 7518 requires. - point, err := k.Bytes() - if err != nil { - return jwk{}, err + switch k.Curve { + case elliptic.P256(): + // Bytes returns 0x04 || x || y, with each coordinate padded to 32 + // bytes, the fixed size of a P-256 coordinate. + point, err := k.Bytes() + if err != nil { + return jwk{}, err + } + j.KeyType, j.Curve = "EC", "P-256" + j.X = base64.RawURLEncoding.EncodeToString(point[1:33]) + j.Y = base64.RawURLEncoding.EncodeToString(point[33:65]) + default: + return jwk{}, errors.New("unsupported curve") } - size := (len(point) - 1) / 2 - j.KeyType, j.Curve = "EC", "P-256" - j.X, j.Y = b64(point[1:1+size]), b64(point[1+size:]) } return j, nil } diff --git a/internal/localjwtauthority/jwks_test.go b/internal/localjwtauthority/jwks_test.go index cd85ec8f75..8717bf6b54 100644 --- a/internal/localjwtauthority/jwks_test.go +++ b/internal/localjwtauthority/jwks_test.go @@ -144,6 +144,16 @@ func TestJWKSRejects(t *testing.T) { } } +func TestToJWKRejectsOtherCurves(t *testing.T) { + p384, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader) + if err != nil { + t.Fatal(err) + } + if j, err := toJWK(&VerificationKey{KeyID: "k", Algorithm: "ES384", PublicKey: &p384.PublicKey}); err == nil { + t.Errorf("toJWK(P-384 key) = %+v, want error", j) + } +} + // RFC 7518 section 6.2.1.2 requires each EC coordinate at the full curve size, // even when it starts with a zero byte. func TestJWKSPadsECCoordinates(t *testing.T) { diff --git a/internal/oidcdiscovery/discovery.go b/internal/oidcdiscovery/discovery.go deleted file mode 100644 index 4b0898afb8..0000000000 --- a/internal/oidcdiscovery/discovery.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package oidcdiscovery - -import ( - "encoding/json" - "errors" - "slices" - "strings" -) - -// JWKSPath is the path of the key set, relative to the issuer. -const JWKSPath = "/openid/v1/jwks" - -// JWKSURI returns the URL of issuer's key set. -func JWKSURI(issuer string) string { - return strings.TrimSuffix(issuer, "/") + JWKSPath -} - -type discoveryDocument struct { - Issuer string `json:"issuer"` - JWKSURI string `json:"jwks_uri"` - ResponseTypesSupported []string `json:"response_types_supported"` - SubjectTypesSupported []string `json:"subject_types_supported"` - IDTokenSigningAlgValuesSupported []string `json:"id_token_signing_alg_values_supported"` - ClaimsSupported []string `json:"claims_supported"` -} - -// DiscoveryDocument returns the JSON served at -// /.well-known/openid-configuration. The issuer is copied verbatim and -// the document advertises each distinct algorithm in algorithms. -func DiscoveryDocument(issuer string, algorithms []string) ([]byte, error) { - if err := ValidateIssuer(issuer); err != nil { - return nil, err - } - if len(algorithms) == 0 { - return nil, errors.New("at least one signing algorithm is required") - } - algs := slices.Clone(algorithms) - slices.Sort(algs) - algs = slices.Compact(algs) - - return json.Marshal(discoveryDocument{ - Issuer: issuer, - JWKSURI: JWKSURI(issuer), - ResponseTypesSupported: []string{"id_token"}, - SubjectTypesSupported: []string{"public"}, - IDTokenSigningAlgValuesSupported: algs, - ClaimsSupported: []string{"iss", "sub", "aud", "exp", "nbf", "iat", "jti", "actor.ate.dev", "worker-for-actor.ate.dev"}, - }) -} diff --git a/internal/oidcdiscovery/discovery_test.go b/internal/oidcdiscovery/discovery_test.go deleted file mode 100644 index f308c5038f..0000000000 --- a/internal/oidcdiscovery/discovery_test.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package oidcdiscovery - -import ( - "encoding/json" - "slices" - "testing" - - "github.com/google/go-cmp/cmp" -) - -func TestDiscoveryDocument(t *testing.T) { - got, err := DiscoveryDocument("https://idp.ate-system.svc", []string{"RS256"}) - if err != nil { - t.Fatal(err) - } - want := `{"issuer":"https://idp.ate-system.svc",` + - `"jwks_uri":"https://idp.ate-system.svc/openid/v1/jwks",` + - `"response_types_supported":["id_token"],` + - `"subject_types_supported":["public"],` + - `"id_token_signing_alg_values_supported":["RS256"],` + - `"claims_supported":["iss","sub","aud","exp","nbf","iat","jti","actor.ate.dev","worker-for-actor.ate.dev"]}` - if diff := cmp.Diff(want, string(got)); diff != "" { - t.Errorf("DiscoveryDocument mismatch (-want +got):\n%s", diff) - } -} - -func TestDiscoveryDocumentIssuerWithPath(t *testing.T) { - got, err := DiscoveryDocument("https://idp.example.com/prod/", []string{"ES256"}) - if err != nil { - t.Fatal(err) - } - var doc struct { - Issuer string `json:"issuer"` - JWKSURI string `json:"jwks_uri"` - } - if err := json.Unmarshal(got, &doc); err != nil { - t.Fatal(err) - } - if doc.Issuer != "https://idp.example.com/prod/" { - t.Errorf("issuer = %q, want it unchanged", doc.Issuer) - } - if doc.JWKSURI != "https://idp.example.com/prod/openid/v1/jwks" { - t.Errorf("jwks_uri = %q, want https://idp.example.com/prod/openid/v1/jwks", doc.JWKSURI) - } -} - -func TestDiscoveryDocumentAlgorithms(t *testing.T) { - algorithms := []string{"RS256", "ES256", "RS256"} - got, err := DiscoveryDocument("https://idp.example.com", algorithms) - if err != nil { - t.Fatal(err) - } - var doc struct { - Algorithms []string `json:"id_token_signing_alg_values_supported"` - } - if err := json.Unmarshal(got, &doc); err != nil { - t.Fatal(err) - } - if diff := cmp.Diff([]string{"ES256", "RS256"}, doc.Algorithms); diff != "" { - t.Errorf("algorithms mismatch (-want +got):\n%s", diff) - } - if !slices.Equal(algorithms, []string{"RS256", "ES256", "RS256"}) { - t.Errorf("DiscoveryDocument modified its input: %v", algorithms) - } -} - -func TestDiscoveryDocumentErrors(t *testing.T) { - tests := []struct { - name string - issuer string - algorithms []string - }{ - {name: "invalid issuer", issuer: "http://idp.example.com", algorithms: []string{"RS256"}}, - {name: "no algorithms", issuer: "https://idp.example.com"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got, err := DiscoveryDocument(tt.issuer, tt.algorithms); err == nil { - t.Errorf("DiscoveryDocument(%q, %v) = %s, want error", tt.issuer, tt.algorithms, got) - } - }) - } -} - -func TestJWKSURI(t *testing.T) { - tests := []struct { - issuer string - want string - }{ - {issuer: "https://idp.ate-system.svc", want: "https://idp.ate-system.svc/openid/v1/jwks"}, - {issuer: "https://idp.example.com/", want: "https://idp.example.com/openid/v1/jwks"}, - {issuer: "https://idp.example.com/prod", want: "https://idp.example.com/prod/openid/v1/jwks"}, - } - for _, tt := range tests { - if got := JWKSURI(tt.issuer); got != tt.want { - t.Errorf("JWKSURI(%q) = %q, want %q", tt.issuer, got, tt.want) - } - } -} From d22a4bb5150d80b523deff7bb962bb4a1123bcd9 Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Fri, 16 Oct 2026 19:08:26 +0000 Subject: [PATCH 6/6] kube-api-linter: fix API type docs and the sandboxClass default (#2461) Part of #207 Drops the temporary exclusions and fixes what they hid. Field docs start with the JSON name. `sandboxClass` is optional, the `gvisor` default is unchanged. Written with AI assistance. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- .golangci-kal.yaml | 17 +----- docs/api-guide.md | 2 +- .../generated/ate.dev_csidriverconfigs.yaml | 16 ++--- .../generated/ate.dev_sandboxconfigs.yaml | 11 ++-- .../generated/ate.dev_workerpools.yaml | 50 ++++++++-------- pkg/api/v1alpha1/csidriverconfig_types.go | 22 ++++--- pkg/api/v1alpha1/sandboxconfig_types.go | 14 ++--- .../v1alpha1/sandboxconfig_validation_test.go | 15 +++++ pkg/api/v1alpha1/workerpool_types.go | 58 +++++++++---------- pkg/api/v1alpha1/zz_generated.deepcopy.go | 14 ++--- 10 files changed, 112 insertions(+), 107 deletions(-) diff --git a/.golangci-kal.yaml b/.golangci-kal.yaml index 53ab0456ff..8ae78cb43e 100644 --- a/.golangci-kal.yaml +++ b/.golangci-kal.yaml @@ -73,21 +73,6 @@ linters: exclusions: rules: - # TEMPORARY: every current commentstart, conditions, defaultorrequired, - # optionalorrequired and defaults finding, excluded file-wide so this - # PR only introduces the tool with no content changes, per review on - # #416. A follow-up PR removes these and fixes the findings by class. - - path: 'pkg/api/v1alpha1/.*_types\.go' - text: '^commentstart:' - - path: 'pkg/api/v1alpha1/.*_types\.go' - text: '^conditions:' - - path: 'pkg/api/v1alpha1/.*_types\.go' - text: '^defaultorrequired:' - - path: 'pkg/api/v1alpha1/.*_types\.go' - text: '^optionalorrequired:' - - path: 'pkg/api/v1alpha1/.*_types\.go' - text: '^defaults:' - # Pre-existing findings from rules that require Go API changes # (pointer fields, omitempty/omitzero tags, map-to-list refactors) # on the existing v1alpha1 types. Each rule names the fields it @@ -110,7 +95,7 @@ linters: - path: 'pkg/api/v1alpha1/csidriverconfig_types\.go' text: '^optionalfields: .*\bCSIDriverTLSConfig\.(ServerName|UsePodIdentity)\b' - path: 'pkg/api/v1alpha1/csidriverconfig_types\.go' - text: '^nonpointerstructs: .*\bCSIDriverConfig\.Spec\b' + text: '^(nonpointerstructs|requiredfields): .*\bCSIDriverConfig\.Spec\b' - path: 'pkg/api/v1alpha1/workerpool_types\.go' text: '^optionalfields: .*\bWorkerPoolPodTemplate\.PriorityClassName\b' - path: 'pkg/api/v1alpha1/workerpool_types\.go' diff --git a/docs/api-guide.md b/docs/api-guide.md index 075bfa045b..315f127537 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -451,7 +451,7 @@ This means a single, cluster-managed config pins the sandbox runtime version for | Field | Type | Description | | :--- | :--- | :--- | -| `sandboxClass` | `string` | **Required.** Runtime family this config applies to: `gvisor` (default) or `microvm`. An `ActorTemplate` only uses `SandboxConfig`s whose `sandboxClass` matches its own. | +| `sandboxClass` | `string` | Optional. Runtime family this config applies to: `gvisor` (default) or `microvm`. An `ActorTemplate` only uses `SandboxConfig`s whose `sandboxClass` matches its own. | | `defaultVersion` | `string` | **Required.** Name of the entry in `versions` that actors boot with. It must name an existing version that is not `Disabled`. | | `versions` | `[]SandboxVersionConfig` | **Required.** 1–16 sandbox versions, unique by `name`. | diff --git a/manifests/ate-install/generated/ate.dev_csidriverconfigs.yaml b/manifests/ate-install/generated/ate.dev_csidriverconfigs.yaml index edf4c95431..492ba5c6ec 100644 --- a/manifests/ate-install/generated/ate.dev_csidriverconfigs.yaml +++ b/manifests/ate-install/generated/ate.dev_csidriverconfigs.yaml @@ -60,17 +60,17 @@ spec: metadata: type: object spec: - description: CSIDriverConfigSpec defines the desired state of CSIDriverConfig + description: spec defines the desired state of CSIDriverConfig properties: controllerEndpoint: description: |- - ControllerEndpoint is the gRPC endpoint for the CSI Controller service. + controllerEndpoint is the gRPC endpoint for the CSI Controller service. Must be a valid network URI (e.g. dns:///csi-service:9000 or tcp://127.0.0.1:9000). pattern: ^(tcp|dns)://.+$ type: string driverName: description: |- - DriverName is the standard CSI driver name (e.g. "hostpath.csi.k8s.io"). + driverName is the standard CSI driver name (e.g. "hostpath.csi.k8s.io"). Matches the StorageClass referenced in ActorTemplate volume definitions. maxLength: 63 minLength: 1 @@ -78,23 +78,23 @@ spec: type: string nodeSocketOverride: description: |- - NodeSocketOverride is an optional override for the CSI Node service socket + nodeSocketOverride is an optional override for the CSI Node service socket on the worker nodes. If empty, ATE defaults to unix:///var/lib/kubelet/plugins/[DriverName]/csi.sock. pattern: ^unix://.+$ type: string tls: - description: TLS configures TLS/mTLS for the connection to the ControllerEndpoint. + description: tls configures TLS/mTLS for the connection to the ControllerEndpoint. properties: enabled: - description: Enabled controls whether TLS is used. + description: enabled controls whether TLS is used. type: boolean serverName: description: |- - ServerName specifies the expected server name (SAN) for TLS certificate verification. + serverName specifies the expected server name (SAN) for TLS certificate verification. Required when TLS is enabled. type: string usePodIdentity: - description: UsePodIdentity indicates whether to reuse Substrate's + description: usePodIdentity indicates whether to reuse Substrate's Pod Identity (SPIFFE) certificates. type: boolean required: diff --git a/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml b/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml index 5260bac99b..74caf8b3c7 100644 --- a/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml +++ b/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml @@ -79,7 +79,7 @@ spec: sandboxClass: default: gvisor description: |- - SandboxClass is the sandbox runtime family this config applies to. An + sandboxClass is the sandbox runtime family this config applies to. An ActorTemplate only uses SandboxConfigs whose SandboxClass matches its sandbox_config.sandbox_class. enum: @@ -103,13 +103,13 @@ spec: properties: sha256: description: |- - SHA256 is the lower-case hex SHA256 of the asset. It both names the cached + sha256 is the lower-case hex SHA256 of the asset. It both names the cached file (preventing collisions) and verifies the download's integrity. pattern: ^[a-f0-9]{64}$ type: string url: description: |- - URL is where to download the asset from (e.g. a gs:// URL). atelet + url is where to download the asset from (e.g. a gs:// URL). atelet fetches every URL through the node's object-store plugin, with the plugin's credentials. The plugin refuses URLs under a snapshot location. The in-tree plugin serves only its storage backend's scheme: gs:// on @@ -125,7 +125,7 @@ spec: type: object type: object description: |- - Assets is the set of files atelet fetches for this runtime, keyed first by + assets is the set of files atelet fetches for this runtime, keyed first by architecture (GOARCH, e.g. "amd64", "arm64") and then by asset name. The asset names are interpreted by the sandbox backend: gVisor expects a "gvisor" asset (the release's gvisor.tar.zstd, which atelet extracts so @@ -145,7 +145,7 @@ spec: type: string pauseImage: description: |- - PauseImage is the container image used as the root sandbox container. + pauseImage is the container image used as the root sandbox container. It holds the sandbox's namespaces and runs no workload code, so it is an implementation detail of the sandbox rather than something actor authors choose. Required for gvisor; not allowed for microvm, which runs no pause @@ -183,7 +183,6 @@ spec: x-kubernetes-list-type: map required: - defaultVersion - - sandboxClass - versions type: object x-kubernetes-validations: diff --git a/manifests/ate-install/generated/ate.dev_workerpools.yaml b/manifests/ate-install/generated/ate.dev_workerpools.yaml index a29c928aab..a46df13c49 100644 --- a/manifests/ate-install/generated/ate.dev_workerpools.yaml +++ b/manifests/ate-install/generated/ate.dev_workerpools.yaml @@ -75,13 +75,13 @@ spec: description: spec defines the desired state of WorkerPool properties: replicas: - description: Replicas is the number of worker pods to run. + description: replicas is the number of worker pods to run. format: int32 minimum: 0 type: integer sandboxClasses: description: |- - SandboxClasses lists the sandbox runtime families this pool runs. Exactly + sandboxClasses lists the sandbox runtime families this pool runs. Exactly one entry is allowed today. The names cannot change after creation. items: description: WorkerPoolSandboxClass is one sandbox runtime family @@ -89,14 +89,14 @@ spec: properties: configRef: description: |- - ConfigRef names a cluster-scoped SandboxConfig for this sandbox class. + configRef names a cluster-scoped SandboxConfig for this sandbox class. The referenced config's SandboxClass must match Name. When omitted, the pool uses the newest SandboxConfig of this class annotated sandboxconfig.ate.dev/is-class-default: "true". The controller reports the resolved config in status.sandboxClasses. properties: name: - description: Name is the SandboxConfig's metadata.name. + description: name is the SandboxConfig's metadata.name. maxLength: 253 minLength: 1 type: string @@ -105,7 +105,7 @@ spec: type: object name: description: |- - Name selects the sandbox runtime family, which drives the worker pod + name selects the sandbox runtime family, which drives the worker pod shape (KVM/vhost device mounts and node placement). The concrete binary is still selected by WorkerImage. The sandbox binaries themselves come from the SandboxConfig each ActorTemplate names (required). @@ -128,14 +128,14 @@ spec: - message: sandboxClasses names are immutable rule: self.map(c, c.name) == oldSelf.map(c, c.name) template: - description: Template holds optional metadata, scheduling, and resource + description: template holds optional metadata, scheduling, and resource settings for worker workloads. properties: annotations: additionalProperties: type: string description: |- - Annotations are added to the generated Deployment and worker pods. Keys + annotations are added to the generated Deployment and worker pods. Keys in the ate.dev domain and its subdomains are reserved for controllers. maxProperties: 64 type: object @@ -154,7 +154,7 @@ spec: pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ type: string description: |- - Labels are added to the generated Deployment and worker pods. Keys in + labels are added to the generated Deployment and worker pods. Keys in the ate.dev domain and its subdomains are reserved for controllers. maxProperties: 64 type: object @@ -165,7 +165,7 @@ spec: rule: self.all(key, !format.qualifiedName().validate(key).hasValue()) nodeAffinity: description: |- - NodeAffinity scheduling rules for the worker pods. Mapped to + nodeAffinity scheduling rules for the worker pods. Mapped to spec.affinity.nodeAffinity on the pod. properties: preferredDuringSchedulingIgnoredDuringExecution: @@ -366,14 +366,14 @@ spec: nodeSelector: additionalProperties: type: string - description: NodeSelector is a selector which must be true for + description: nodeSelector is a selector which must be true for the pod to fit on a node. type: object priorityClassName: - description: PriorityClassName for the worker pods. + description: priorityClassName for the worker pods. type: string resources: - description: Resources are the compute resources allocated for + description: resources are the compute resources allocated for each worker pod. properties: claims: @@ -434,13 +434,13 @@ spec: type: object serviceAccountName: description: |- - ServiceAccountName for the worker pods. The ServiceAccount must exist + serviceAccountName for the worker pods. The ServiceAccount must exist in the WorkerPool's namespace. When omitted, the pods run as the "ateworker" ServiceAccount, which the controller creates. minLength: 1 type: string tolerations: - description: Tolerations for the worker pods. + description: tolerations for the worker pods. items: description: |- The pod this Toleration is attached to tolerates any taint that matches @@ -483,7 +483,7 @@ spec: x-kubernetes-list-type: atomic type: object workerImage: - description: WorkerImage is the worker container image to deploy as + description: workerImage is the worker container image to deploy as workers. minLength: 1 type: string @@ -496,18 +496,18 @@ spec: description: status is the observed state of WorkerPool properties: readyReplicas: - description: ReadyReplicas is the number of ready worker pods. + description: readyReplicas is the number of ready worker pods. format: int32 minimum: 0 type: integer replicas: - description: Replicas is the total number of worker pods. + description: replicas is the total number of worker pods. format: int32 minimum: 0 type: integer sandboxClasses: description: |- - SandboxClasses reports the SandboxConfig each sandbox class of the pool + sandboxClasses reports the SandboxConfig each sandbox class of the pool uses and whether the class's spec resolves. items: description: |- @@ -515,7 +515,7 @@ spec: of one sandbox class of a WorkerPool. properties: conditions: - description: Conditions describe the current state of this sandbox + description: conditions describe the current state of this sandbox class. items: description: Condition contains details for one aspect of @@ -578,13 +578,13 @@ spec: x-kubernetes-list-type: map configRef: description: |- - ConfigRef names the cluster-scoped SandboxConfig in use. That is the + configRef names the cluster-scoped SandboxConfig in use. That is the config spec.sandboxClasses[].configRef names, or the class default, or the last resolved config while the spec does not resolve. Omitted when the class has never resolved. properties: name: - description: Name is the SandboxConfig's metadata.name. + description: name is the SandboxConfig's metadata.name. maxLength: 253 minLength: 1 type: string @@ -592,20 +592,20 @@ spec: - name type: object name: - description: Name is the sandbox runtime family. + description: name is the sandbox runtime family. enum: - gvisor - microvm type: string pendingConfigRef: description: |- - PendingConfigRef names the SandboxConfig the class is switching to + pendingConfigRef names the SandboxConfig the class is switching to while the controller confirms it against the API server. It protects the config from deletion like configRef does. The same reconcile clears it, and configRef either moves to it or stays where it was. properties: name: - description: Name is the SandboxConfig's metadata.name. + description: name is the SandboxConfig's metadata.name. maxLength: 253 minLength: 1 type: string @@ -621,7 +621,7 @@ spec: - name x-kubernetes-list-type: map selector: - description: Selector is the label selector for the worker pods. + description: selector is the label selector for the worker pods. type: string type: object required: diff --git a/pkg/api/v1alpha1/csidriverconfig_types.go b/pkg/api/v1alpha1/csidriverconfig_types.go index e790237669..23a566699a 100644 --- a/pkg/api/v1alpha1/csidriverconfig_types.go +++ b/pkg/api/v1alpha1/csidriverconfig_types.go @@ -20,7 +20,7 @@ import ( // CSIDriverConfigSpec defines the desired state of CSIDriverConfig type CSIDriverConfigSpec struct { - // DriverName is the standard CSI driver name (e.g. "hostpath.csi.k8s.io"). + // driverName is the standard CSI driver name (e.g. "hostpath.csi.k8s.io"). // Matches the StorageClass referenced in ActorTemplate volume definitions. // // +required @@ -29,7 +29,7 @@ type CSIDriverConfigSpec struct { // +kubebuilder:validation:Pattern=`^(substrate\.io/)?([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)$` DriverName string `json:"driverName"` - // ControllerEndpoint is the gRPC endpoint for the CSI Controller service. + // controllerEndpoint is the gRPC endpoint for the CSI Controller service. // Must be a valid network URI (e.g. dns:///csi-service:9000 or tcp://127.0.0.1:9000). // TODO: Harden endpoint validation to prevent invalid or unsafe URI inputs. // @@ -37,14 +37,14 @@ type CSIDriverConfigSpec struct { // +kubebuilder:validation:Pattern=`^(tcp|dns)://.+$` ControllerEndpoint string `json:"controllerEndpoint"` - // NodeSocketOverride is an optional override for the CSI Node service socket + // nodeSocketOverride is an optional override for the CSI Node service socket // on the worker nodes. If empty, ATE defaults to unix:///var/lib/kubelet/plugins/[DriverName]/csi.sock. // // +optional // +kubebuilder:validation:Pattern=`^unix://.+$` NodeSocketOverride string `json:"nodeSocketOverride,omitempty"` - // TLS configures TLS/mTLS for the connection to the ControllerEndpoint. + // tls configures TLS/mTLS for the connection to the ControllerEndpoint. // +optional TLS *CSIDriverTLSConfig `json:"tls,omitempty"` } @@ -53,16 +53,17 @@ type CSIDriverConfigSpec struct { // +kubebuilder:validation:XValidation:rule="!self.enabled || (has(self.usePodIdentity) && self.usePodIdentity)",message="tls.usePodIdentity must be true when tls.enabled is true; manual certificates are not yet supported" // +kubebuilder:validation:XValidation:rule="!self.enabled || (has(self.serverName) && size(self.serverName) > 0)",message="tls.serverName is required when tls.enabled is true" type CSIDriverTLSConfig struct { - // Enabled controls whether TLS is used. + // enabled controls whether TLS is used. // +required Enabled bool `json:"enabled"` // TODO: Add alternative support for manual certs by adding SecretReference fields. - // UsePodIdentity indicates whether to reuse Substrate's Pod Identity (SPIFFE) certificates. + + // usePodIdentity indicates whether to reuse Substrate's Pod Identity (SPIFFE) certificates. // +optional UsePodIdentity bool `json:"usePodIdentity,omitempty"` - // ServerName specifies the expected server name (SAN) for TLS certificate verification. + // serverName specifies the expected server name (SAN) for TLS certificate verification. // Required when TLS is enabled. // +optional ServerName string `json:"serverName,omitempty"` @@ -78,9 +79,14 @@ type CSIDriverTLSConfig struct { // +kubebuilder:printcolumn:name="Driver",type=string,JSONPath=`.spec.driverName` // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` type CSIDriverConfig struct { - metav1.TypeMeta `json:",inline"` + metav1.TypeMeta `json:",inline"` + + // metadata is a standard object metadata + // +optional metav1.ObjectMeta `json:"metadata,omitempty"` + // spec defines the desired state of CSIDriverConfig + // +required Spec CSIDriverConfigSpec `json:"spec"` } diff --git a/pkg/api/v1alpha1/sandboxconfig_types.go b/pkg/api/v1alpha1/sandboxconfig_types.go index 8abbf96c8c..e344536eca 100644 --- a/pkg/api/v1alpha1/sandboxconfig_types.go +++ b/pkg/api/v1alpha1/sandboxconfig_types.go @@ -35,7 +35,7 @@ const ( // AssetFile is one content-addressed file that atelet fetches for a sandbox // runtime (e.g. the gVisor runsc binary, or a micro-VM kernel/firmware/config). type AssetFile struct { - // URL is where to download the asset from (e.g. a gs:// URL). atelet + // url is where to download the asset from (e.g. a gs:// URL). atelet // fetches every URL through the node's object-store plugin, with the // plugin's credentials. The plugin refuses URLs under a snapshot location. // The in-tree plugin serves only its storage backend's scheme: gs:// on @@ -48,7 +48,7 @@ type AssetFile struct { // +kubebuilder:validation:MaxLength=2048 URL string `json:"url"` - // SHA256 is the lower-case hex SHA256 of the asset. It both names the cached + // sha256 is the lower-case hex SHA256 of the asset. It both names the cached // file (preventing collisions) and verifies the download's integrity. // // +required @@ -118,7 +118,7 @@ type SandboxVersionConfig struct { // TODO: drop PauseImage once gVisor can run without a pause container: // https://github.com/google/gvisor/pull/13981 - // PauseImage is the container image used as the root sandbox container. + // pauseImage is the container image used as the root sandbox container. // It holds the sandbox's namespaces and runs no workload code, so it is an // implementation detail of the sandbox rather than something actor authors // choose. Required for gvisor; not allowed for microvm, which runs no pause @@ -137,7 +137,7 @@ type SandboxVersionConfig struct { // +kubebuilder:validation:XValidation:rule="self.contains('@')",message="All images must include a digest" PauseImage string `json:"pauseImage,omitempty"` - // Assets is the set of files atelet fetches for this runtime, keyed first by + // assets is the set of files atelet fetches for this runtime, keyed first by // architecture (GOARCH, e.g. "amd64", "arm64") and then by asset name. The // asset names are interpreted by the sandbox backend: gVisor expects a // "gvisor" asset (the release's gvisor.tar.zstd, which atelet extracts so @@ -157,14 +157,14 @@ type SandboxVersionConfig struct { // +kubebuilder:validation:XValidation:rule="!self.versions.exists(v, v.name == self.defaultVersion && has(v.state) && v.state == 'Disabled')",message="the version named by defaultVersion must not be Disabled" // +kubebuilder:validation:XValidation:rule="self.sandboxClass == 'gvisor' ? self.versions.all(v, has(v.pauseImage) && size(v.pauseImage) > 0) : self.versions.all(v, !has(v.pauseImage) || size(v.pauseImage) == 0)",message="pauseImage is required on every version for gvisor and not allowed for other sandbox classes" type SandboxConfigSpec struct { - // SandboxClass is the sandbox runtime family this config applies to. An + // sandboxClass is the sandbox runtime family this config applies to. An // ActorTemplate only uses SandboxConfigs whose SandboxClass matches its // sandbox_config.sandbox_class. // - // +required + // +optional // +kubebuilder:validation:Enum=gvisor;microvm // +kubebuilder:default=gvisor - SandboxClass SandboxClass `json:"sandboxClass"` + SandboxClass SandboxClass `json:"sandboxClass,omitempty"` // defaultVersion names the entry in versions that new sandboxes boot with. // It must name an existing, non-Disabled version. diff --git a/pkg/api/v1alpha1/sandboxconfig_validation_test.go b/pkg/api/v1alpha1/sandboxconfig_validation_test.go index fd6e4536b9..9678b65ac4 100644 --- a/pkg/api/v1alpha1/sandboxconfig_validation_test.go +++ b/pkg/api/v1alpha1/sandboxconfig_validation_test.go @@ -348,6 +348,21 @@ func TestSandboxConfigValidation(t *testing.T) { } } +// An omitted sandboxClass is defaulted to gvisor, not rejected. +func TestSandboxConfigSandboxClassDefault(t *testing.T) { + ctx := t.Context() + sc := sandboxConfig("default-class", SandboxClassGvisor, gvisorAssets()) + sc.Spec.SandboxClass = "" + if err := k8sClient.Create(ctx, sc); err != nil { + t.Fatalf("Create() unexpected error: %v", err) + } + t.Cleanup(func() { _ = k8sClient.Delete(ctx, sc) }) + + if sc.Spec.SandboxClass != SandboxClassGvisor { + t.Errorf("SandboxClass = %q, want %q", sc.Spec.SandboxClass, SandboxClassGvisor) + } +} + func TestSandboxConfigUpdateRemovingDefaultVersion(t *testing.T) { ctx := t.Context() sc := withVersions(sandboxConfig("update-remove-default", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v2", SandboxClassGvisor, gvisorAssets())) diff --git a/pkg/api/v1alpha1/workerpool_types.go b/pkg/api/v1alpha1/workerpool_types.go index 20782b0b72..411b68327e 100644 --- a/pkg/api/v1alpha1/workerpool_types.go +++ b/pkg/api/v1alpha1/workerpool_types.go @@ -30,7 +30,7 @@ type WorkerPoolLabelValue string // settings for worker workloads. NodeAffinity is mapped to // spec.affinity.nodeAffinity on the pod. type WorkerPoolPodTemplate struct { - // Labels are added to the generated Deployment and worker pods. Keys in + // labels are added to the generated Deployment and worker pods. Keys in // the ate.dev domain and its subdomains are reserved for controllers. // // +optional @@ -39,7 +39,7 @@ type WorkerPoolPodTemplate struct { // +kubebuilder:validation:XValidation:rule="self.all(key, !format.qualifiedName().validate(key).hasValue())",message="label keys must be valid Kubernetes qualified names" Labels map[string]WorkerPoolLabelValue `json:"labels,omitempty"` - // Annotations are added to the generated Deployment and worker pods. Keys + // annotations are added to the generated Deployment and worker pods. Keys // in the ate.dev domain and its subdomains are reserved for controllers. // // +optional @@ -48,35 +48,35 @@ type WorkerPoolPodTemplate struct { // +kubebuilder:validation:XValidation:rule="self.all(key, !format.qualifiedName().validate(key).hasValue())",message="annotation keys must be valid Kubernetes qualified names" Annotations map[string]string `json:"annotations,omitempty"` - // NodeSelector is a selector which must be true for the pod to fit on a node. + // nodeSelector is a selector which must be true for the pod to fit on a node. // // +optional NodeSelector map[string]string `json:"nodeSelector,omitempty"` - // Tolerations for the worker pods. + // tolerations for the worker pods. // // +optional // +kubebuilder:validation:MaxItems=16 // +listType=atomic Tolerations []corev1.Toleration `json:"tolerations,omitempty"` - // PriorityClassName for the worker pods. + // priorityClassName for the worker pods. // // +optional PriorityClassName string `json:"priorityClassName,omitempty"` - // NodeAffinity scheduling rules for the worker pods. Mapped to + // nodeAffinity scheduling rules for the worker pods. Mapped to // spec.affinity.nodeAffinity on the pod. // // +optional NodeAffinity *corev1.NodeAffinity `json:"nodeAffinity,omitempty"` - // Resources are the compute resources allocated for each worker pod. + // resources are the compute resources allocated for each worker pod. // // +optional Resources *corev1.ResourceRequirements `json:"resources,omitempty"` - // ServiceAccountName for the worker pods. The ServiceAccount must exist + // serviceAccountName for the worker pods. The ServiceAccount must exist // in the WorkerPool's namespace. When omitted, the pods run as the // "ateworker" ServiceAccount, which the controller creates. // @@ -86,22 +86,22 @@ type WorkerPoolPodTemplate struct { } type WorkerPoolSpec struct { - // Replicas is the number of worker pods to run. + // replicas is the number of worker pods to run. // +required // +kubebuilder:validation:Minimum=0 Replicas int32 `json:"replicas"` - // WorkerImage is the worker container image to deploy as workers. + // workerImage is the worker container image to deploy as workers. // +kubebuilder:validation:MinLength=1 // +required WorkerImage string `json:"workerImage"` - // Template holds optional metadata, scheduling, and resource settings for worker workloads. + // template holds optional metadata, scheduling, and resource settings for worker workloads. // // +optional Template *WorkerPoolPodTemplate `json:"template,omitempty"` - // SandboxClasses lists the sandbox runtime families this pool runs. Exactly + // sandboxClasses lists the sandbox runtime families this pool runs. Exactly // one entry is allowed today. The names cannot change after creation. // // +required @@ -115,7 +115,7 @@ type WorkerPoolSpec struct { // WorkerPoolSandboxClass is one sandbox runtime family a WorkerPool runs. type WorkerPoolSandboxClass struct { - // Name selects the sandbox runtime family, which drives the worker pod + // name selects the sandbox runtime family, which drives the worker pod // shape (KVM/vhost device mounts and node placement). The concrete binary // is still selected by WorkerImage. The sandbox binaries themselves come // from the SandboxConfig each ActorTemplate names (required). @@ -126,7 +126,7 @@ type WorkerPoolSandboxClass struct { // +kubebuilder:validation:Enum=gvisor;microvm Name SandboxClass `json:"name,omitempty"` - // ConfigRef names a cluster-scoped SandboxConfig for this sandbox class. + // configRef names a cluster-scoped SandboxConfig for this sandbox class. // The referenced config's SandboxClass must match Name. When omitted, the // pool uses the newest SandboxConfig of this class annotated // sandboxconfig.ate.dev/is-class-default: "true". The controller reports @@ -138,7 +138,7 @@ type WorkerPoolSandboxClass struct { // SandboxConfigReference names a cluster-scoped SandboxConfig. type SandboxConfigReference struct { - // Name is the SandboxConfig's metadata.name. + // name is the SandboxConfig's metadata.name. // // +required // +kubebuilder:validation:MinLength=1 @@ -157,21 +157,21 @@ func (s *WorkerPoolSpec) DefaultSandboxClass() SandboxClass { } type WorkerPoolStatus struct { - // Replicas is the total number of worker pods. + // replicas is the total number of worker pods. // +kubebuilder:validation:Minimum=0 // +optional Replicas int32 `json:"replicas"` - // ReadyReplicas is the number of ready worker pods. + // readyReplicas is the number of ready worker pods. // +kubebuilder:validation:Minimum=0 // +optional ReadyReplicas int32 `json:"readyReplicas,omitempty"` - // Selector is the label selector for the worker pods. + // selector is the label selector for the worker pods. // +optional Selector string `json:"selector,omitempty"` - // SandboxClasses reports the SandboxConfig each sandbox class of the pool + // sandboxClasses reports the SandboxConfig each sandbox class of the pool // uses and whether the class's spec resolves. // // +optional @@ -184,13 +184,20 @@ type WorkerPoolStatus struct { // WorkerPoolSandboxClassStatus reports the SandboxConfig and resolution state // of one sandbox class of a WorkerPool. type WorkerPoolSandboxClassStatus struct { - // Name is the sandbox runtime family. + // conditions describe the current state of this sandbox class. + // + // +optional + // +listType=map + // +listMapKey=type + Conditions []metav1.Condition `json:"conditions,omitempty"` + + // name is the sandbox runtime family. // // +required // +kubebuilder:validation:Enum=gvisor;microvm Name SandboxClass `json:"name,omitempty"` - // ConfigRef names the cluster-scoped SandboxConfig in use. That is the + // configRef names the cluster-scoped SandboxConfig in use. That is the // config spec.sandboxClasses[].configRef names, or the class default, or // the last resolved config while the spec does not resolve. Omitted when // the class has never resolved. @@ -198,20 +205,13 @@ type WorkerPoolSandboxClassStatus struct { // +optional ConfigRef *SandboxConfigReference `json:"configRef,omitempty"` - // PendingConfigRef names the SandboxConfig the class is switching to + // pendingConfigRef names the SandboxConfig the class is switching to // while the controller confirms it against the API server. It protects // the config from deletion like configRef does. The same reconcile // clears it, and configRef either moves to it or stays where it was. // // +optional PendingConfigRef *SandboxConfigReference `json:"pendingConfigRef,omitempty"` - - // Conditions describe the current state of this sandbox class. - // - // +optional - // +listType=map - // +listMapKey=type - Conditions []metav1.Condition `json:"conditions,omitempty"` } const ( diff --git a/pkg/api/v1alpha1/zz_generated.deepcopy.go b/pkg/api/v1alpha1/zz_generated.deepcopy.go index 394977bbf3..bee9dcfcad 100644 --- a/pkg/api/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/api/v1alpha1/zz_generated.deepcopy.go @@ -405,6 +405,13 @@ func (in *WorkerPoolSandboxClass) DeepCopy() *WorkerPoolSandboxClass { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WorkerPoolSandboxClassStatus) DeepCopyInto(out *WorkerPoolSandboxClassStatus) { *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.ConfigRef != nil { in, out := &in.ConfigRef, &out.ConfigRef *out = new(SandboxConfigReference) @@ -415,13 +422,6 @@ func (in *WorkerPoolSandboxClassStatus) DeepCopyInto(out *WorkerPoolSandboxClass *out = new(SandboxConfigReference) **out = **in } - if in.Conditions != nil { - in, out := &in.Conditions, &out.Conditions - *out = make([]metav1.Condition, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkerPoolSandboxClassStatus.