From 2fb2ac02abd74d0aa6f6af474318e162e6874e69 Mon Sep 17 00:00:00 2001 From: Celrenheit Date: Wed, 12 Aug 2026 20:47:24 +0000 Subject: [PATCH 1/3] test(oci): assert the cache hit by observing the artifact, not the clock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `require.LessOrEqual(t, d, time.Second)` failed on an emulated arm64 host at 1.19s while the cache was working perfectly. Wall-clock was the wrong instrument twice over. It is a proxy for "did not redo the work" whose budget depends entirely on the hardware the test lands on — and it was not even measuring the network, because resolveSource calls crane.Digest to turn the tag into a digest BEFORE the cache lookup, so every Build makes a registry round-trip whose latency the test cannot bound. A slow link fails it as surely as a slow CPU. The cache-hit path stats the done marker and returns; a miss re-pulls and rewrites disk.ext4. So an unchanged mtime, size and os.SameFile prove the artifact was reused — the property the timing bound stood in for, established deterministically. Verified sensitive by forcing a real rebuild: both signals flip. Unrelated to the release below: the test is unchanged since the initial release, so it stands on its own. --- machine/oci/integration_test.go | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/machine/oci/integration_test.go b/machine/oci/integration_test.go index 1dbfb36..c4b2004 100644 --- a/machine/oci/integration_test.go +++ b/machine/oci/integration_test.go @@ -48,8 +48,26 @@ func TestBuildAlpine_Integration(t *testing.T) { assertValidRootFS(t, res.DiskPath) + // A cache hit is asserted by observing that the disk was not rebuilt, not + // by timing the call. + // + // Wall-clock is the wrong instrument here for two reasons. It is a proxy + // for "didn't redo the work" that depends entirely on the hardware the + // test happens to run on — and it isn't even measuring the network, since + // resolveSource calls crane.Digest to turn the tag into a digest BEFORE + // the cache lookup, so every Build makes a registry round-trip whose + // latency the test cannot bound. A one-second budget therefore fails on a + // slow or emulated host, and on a slow link, while the cache is working + // perfectly. + // + // The cache-hit path stats the done marker and returns; a miss re-pulls + // and rewrites disk.ext4. So an unchanged mtime, size and identity prove + // the artifact was reused, which is the property the timing bound was + // standing in for — and prove it deterministically. t.Run("cache hit", func(t *testing.T) { - start := time.Now() + before, err := os.Stat(res.DiskPath) + require.NoError(t, err, "stat before cached Build") + res2, err := Build(ctx, Options{ Ref: ref, CacheDir: cache, @@ -57,8 +75,14 @@ func TestBuildAlpine_Integration(t *testing.T) { }) require.NoError(t, err, "cached Build") require.Equal(t, res.DiskPath, res2.DiskPath, "cache miss") - d := time.Since(start) - require.LessOrEqual(t, d, time.Second, "cache hit too slow: %s (expected <1s)", d) + + after, err := os.Stat(res2.DiskPath) + require.NoError(t, err, "stat after cached Build") + require.Equal(t, before.ModTime(), after.ModTime(), + "disk.ext4 was rewritten — the second Build rebuilt instead of hitting the cache") + require.Equal(t, before.Size(), after.Size(), "disk.ext4 changed size") + require.True(t, os.SameFile(before, after), + "disk.ext4 was replaced by a different file — the second Build rebuilt and renamed") }) t.Run("materialize produces independent disks", func(t *testing.T) { From 0ab9a58622c86a1bbd20c126997dab082af0c2a3 Mon Sep 17 00:00:00 2001 From: Celrenheit Date: Wed, 12 Aug 2026 20:47:50 +0000 Subject: [PATCH 2/3] feat: swap devices, serial forwarding, MCP declarations, the pi runner, host-wide defaults MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The v0.4.0 feature set. Five threads, each pulling on the same premise — that a sandbox should arrive configured rather than needing setup once it is up. **Every sandbox boots with swap** (2 GiB default, `vm ( swap )`). The reason is the balloon controller, not the guest's appetite: under host memory pressure clawk reclaims guest RAM against guest demand, and a guest with nowhere to put cold anonymous pages answers with direct-reclaim stalls and eventually its OOM killer. A multi-second stall in the agent is not merely slow — a process that stops draining its socket lets the connection go idle, and on a link whose NAT reaps idle mappings in under a minute that ends the streaming response. It rides its own sparse virtio-blk device rather than a swapfile, because swapon(2) rejects a file with holes and a swapfile would cost its full size in real host bytes per sandbox. clawk-init writes the header itself rather than shelling to mkswap, since the rootfs is an arbitrary OCI image. The controller learned about swap at the same time and had to: paging out raises MemAvailable and lowers PSI, so a squeezed guest reads as roomy on both signals the controller uses. swapTrend holds reclaim while swap is GROWING and lets the hold decay after two minutes of quiet, because occupancy latches — a slot is freed only when its page faults back in — and reading the level as pressure would retire reclaim for that sandbox permanently. **Serial devices.** `clawk serial add /dev/cu.usbmodem1101` puts a board plugged into the Mac inside the sandbox as `/dev/`, so arduino-cli, esptool and a monitor can run in there against real hardware. The USB device is not passed through, because nothing clawk runs on can do that; what crosses is the byte stream and the line settings, which is all any of those tools wanted. The guest side is a PTY, the host side the real tty, opened only while a process in the sandbox holds the device — which is also what makes auto-reset work, since opening a port asserts DTR. A glob is resolved at open time, not at configure time, so a board that re-enumerates into its bootloader under a neighbouring name stays reachable. vz-only: the guest is the end that dials. **`mcp ( … )`** declares MCP servers so a sandbox comes up with them configured, no interactive login inside the VM. Each http/sse host is folded into a derived `mcp` network layer ranking below anything you wrote, so a declared server is reachable without a matching `network allow` and cannot override your own `deny`. Credential values stay out of clawk: `clawk.mod` and the rendered guest config hold a `${VAR}` reference, the value travels to the runner's process environment at attach. A URL embedding credentials is refused, since it is the one spelling that would defeat that. **The pi runner, opencode wired up, per-runner state on the host.** `clawk run pi` joins claude and codex, and opencode goes from registry-only to installed and persisted. The bigger fix underneath: only `~/.claude` was host-mounted, so codex's sessions and login lived on a rootfs vz re-clones every boot and were discarded by a plain `down`/`up` — not just by `destroy`. Each runner's home is now its own per-sandbox mount, opencode's two XDG dirs included. **Host-wide defaults in `~/.config/clawk/clawk.mod`.** A kernel path, a token alias, a personal skill mount and a house rule are properties of the host, not of the repo they were written in. One anonymous `sandbox ( … )` block outside any repo now supplies settings for every sandbox on the machine, as the lowest layer of the chain. Scope is enforced, relative paths resolve against the file's own directory, and `--no-global` drops the layer for a reproducible run. A broken file is fatal on every create path rather than degrading to "no defaults" — that silently discarded the repo's own clawk.mod too. Also from the pre-release review: `env ( … )` values now reach the runner's own process (not just its login shells) and a declaration wins over clawk's same-named variable unconditionally, even when it fails to resolve — otherwise a sandbox pointed at a third-party gateway got clawk's Anthropic token as an Authorization header. And the serial hangup path flushes before letting go, so `echo cmd > /dev/ttyACM0` is delivered rather than discarded. --- ARCHITECTURE.md | 13 + DESIGN.md | 16 +- README.md | 33 +- docs/commands.md | 37 +- docs/configuration.md | 137 ++- docs/images.md | 6 +- docs/mcp.md | 127 +++ docs/networking.md | 11 + docs/serial.md | 168 ++++ images/clawk-dev/Dockerfile | 19 +- internal/agentembed/agent_compile_test.go | 74 ++ internal/agentembed/embed.go | 9 + internal/agentembed/go.mod.in | 1 + internal/agentembed/init_main.go.in | 138 ++- internal/agentembed/main.go.in | 808 +++++++++++++++++- internal/agentembed/revfwd_lockstep_test.go | 2 + internal/agentembed/serial_agent_test.go.in | 512 +++++++++++ .../agentembed/serialfwd_lockstep_test.go | 83 ++ internal/cli/agent_session.go | 82 +- internal/cli/agent_session_test.go | 153 +++- internal/cli/agents.go | 74 +- internal/cli/agents_test.go | 58 ++ internal/cli/apply.go | 9 + internal/cli/cli_test.go | 1 + internal/cli/complete.go | 2 +- internal/cli/compose_mcp.go | 173 ++++ internal/cli/compose_mcp_test.go | 226 +++++ internal/cli/compose_serial.go | 110 +++ internal/cli/compose_serial_test.go | 93 ++ internal/cli/daemon.go | 21 +- internal/cli/fcd.go | 2 +- internal/cli/global_defaults_test.go | 207 +++++ internal/cli/here.go | 67 +- internal/cli/main_test.go | 23 + internal/cli/namespace.go | 1 + internal/cli/resources.go | 25 + internal/cli/root.go | 8 + internal/cli/run.go | 85 +- internal/cli/serial.go | 368 ++++++++ internal/cli/serial_proxy.go | 425 +++++++++ internal/cli/serial_proxy_test.go | 379 ++++++++ internal/cli/serial_test.go | 283 ++++++ internal/cli/sessions.go | 2 +- internal/cli/setup.go | 4 + internal/cli/shares.go | 19 +- internal/cli/shares_test.go | 77 ++ internal/cli/shell.go | 2 +- internal/cli/status.go | 20 + internal/cli/swap_test.go | 37 + internal/cli/up.go | 15 + internal/cli/vshell.go | 2 +- internal/cli/vzd.go | 31 +- internal/cli/workspace_defaults_test.go | 56 ++ internal/config/namespace.go | 6 + internal/config/types.go | 149 +++- internal/config/types_test.go | 41 + internal/guestcfg/manifest.go | 32 + internal/sandbox/agentstate_test.go | 162 ++++ internal/sandbox/firecracker_linux.go | 48 +- internal/sandbox/firecracker_linux_test.go | 27 + internal/sandbox/mcp.go | 153 ++++ internal/sandbox/mcp_test.go | 152 ++++ internal/sandbox/oci_sandbox.go | 19 +- internal/sandbox/oci_sandbox_test.go | 2 +- internal/sandbox/shares.go | 292 +++++-- internal/sandbox/shares_test.go | 58 +- internal/sandbox/swapdisk.go | 109 +++ internal/sandbox/swapdisk_test.go | 95 ++ internal/sandbox/vzprovider_oci_darwin.go | 7 + internal/serialfwd/serialfwd.go | 388 +++++++++ internal/serialfwd/serialfwd_test.go | 182 ++++ internal/serialport/serialport.go | 140 +++ internal/serialport/serialport_other.go | 20 + internal/serialport/serialport_test.go | 188 ++++ .../serialport/serialporttest/pty_darwin.go | 50 ++ .../serialport/serialporttest/pty_linux.go | 49 ++ internal/serialport/termios_darwin.go | 46 + internal/serialport/termios_linux.go | 54 ++ internal/serialport/termios_unix.go | 77 ++ internal/template/global.go | 407 +++++++++ internal/template/global_test.go | 371 ++++++++ internal/template/main_test.go | 16 + internal/template/mcp_test.go | 155 ++++ internal/template/parse.go | 403 ++++++++- internal/template/parse_test.go | 33 + internal/template/resources.go | 9 +- internal/template/serial_test.go | 51 ++ internal/template/workspace.go | 67 +- internal/vsockclient/client.go | 2 +- internal/vzdctl/vzdctl.go | 48 ++ internal/vzdctl/vzdctl_test.go | 64 ++ machine/vz/balloon.go | 95 +- machine/vz/balloon_test.go | 143 +++- machine/vz/memreport.go | 45 +- machine/vz/memreport_test.go | 42 + machine/vz/pressure_darwin.go | 15 +- 96 files changed, 9623 insertions(+), 223 deletions(-) create mode 100644 docs/mcp.md create mode 100644 docs/serial.md create mode 100644 internal/agentembed/serial_agent_test.go.in create mode 100644 internal/agentembed/serialfwd_lockstep_test.go create mode 100644 internal/cli/agents_test.go create mode 100644 internal/cli/compose_mcp.go create mode 100644 internal/cli/compose_mcp_test.go create mode 100644 internal/cli/compose_serial.go create mode 100644 internal/cli/compose_serial_test.go create mode 100644 internal/cli/global_defaults_test.go create mode 100644 internal/cli/main_test.go create mode 100644 internal/cli/serial.go create mode 100644 internal/cli/serial_proxy.go create mode 100644 internal/cli/serial_proxy_test.go create mode 100644 internal/cli/serial_test.go create mode 100644 internal/cli/swap_test.go create mode 100644 internal/cli/workspace_defaults_test.go create mode 100644 internal/sandbox/agentstate_test.go create mode 100644 internal/sandbox/mcp.go create mode 100644 internal/sandbox/mcp_test.go create mode 100644 internal/sandbox/swapdisk.go create mode 100644 internal/sandbox/swapdisk_test.go create mode 100644 internal/serialfwd/serialfwd.go create mode 100644 internal/serialfwd/serialfwd_test.go create mode 100644 internal/serialport/serialport.go create mode 100644 internal/serialport/serialport_other.go create mode 100644 internal/serialport/serialport_test.go create mode 100644 internal/serialport/serialporttest/pty_darwin.go create mode 100644 internal/serialport/serialporttest/pty_linux.go create mode 100644 internal/serialport/termios_darwin.go create mode 100644 internal/serialport/termios_linux.go create mode 100644 internal/serialport/termios_unix.go create mode 100644 internal/template/global.go create mode 100644 internal/template/global_test.go create mode 100644 internal/template/main_test.go create mode 100644 internal/template/mcp_test.go create mode 100644 internal/template/serial_test.go diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index a8a254f..9a59ef9 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -74,6 +74,18 @@ for the protocol, `internal/cli/reverse_forward.go` for the host end). The daemon pushes set changes down the same channel, so those edits apply to a running guest. vz only — firecracker's vsock is one-way. +Serial ports (`clawk serial add`) ride the same asymmetry and reuse its whole +shape: the guest agent creates a PTY per configured device, and holding that +PTY open is what makes the agent dial the daemon, which validates the device +name against the configured set and opens the physical tty (`internal/ +serialfwd` for the protocol, `internal/cli/serial_proxy.go` for the host end, +`internal/serialport` for the tty itself). Passing the USB device through +instead is not an option on either provider — Virtualization.framework +exposes no physical USB passthrough and firecracker has no USB bus — and it +isn't needed: the tooling wants a tty and a baud rate, not a USB endpoint. +Tying the host-side open to the guest-side open also reproduces the DTR edge +that resets an Arduino at the moment an upload expects it. + Live allow-list edits reach the running daemon over a control socket (`internal/vzdctl`); when the sandbox is down they apply on the next `up`. The same socket carries the VM lifecycle verbs: `clawk pause` / `resume` @@ -105,6 +117,7 @@ because it pins a vendored `gvisor-tap-vsock` fork; everything clawk-specific | `internal/agentembed` | The in-guest binaries (clawk-init, pty-agent, time-sync), cross-compiled and injected into the rootfs. | | `internal/vsockproto` / `internal/vsockclient` | The host↔guest vsock framing and the host-side client. | | `internal/revfwd` | Reverse-forward wire protocol (host loopback services exposed on the guest's loopback), mirrored in the guest agent. | +| `internal/serialfwd` / `internal/serialport` | Serial-forwarding wire protocol (mirrored in the guest agent) and the host-side tty open/termios. | | `internal/netfilter` | Egress allow-list (IPs/CIDRs/domains, DNS-aware) consumed by gvproxy. | | `internal/vzdctl` | Daemon control socket (live policy edits, denial ledger, VM pause/resume/suspend). | | `internal/worktree` / `internal/pr` | Multi-repo branch coordination and PR creation. | diff --git a/DESIGN.md b/DESIGN.md index 0fed5ee..634e8d3 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -174,9 +174,13 @@ Everything lives under `~/.clawk/`, namespace-first: `suspend/` directory holding the saved memory + device state, consumed one-shot by the next boot and discarded by `clawk down`. Wiped by `destroy`. -- `namespaces//state//` — agent state mounted into the guest - (Claude projects/memory, Codex state). **Survives `destroy`** — recreating - a sandbox restores history. +- `namespaces//state//` — agent state mounted into the guest, one + subdirectory per runner home (`claude/` → `~/.claude`, `codex/` → + `~/.codex`, `pi/` → `~/.pi`, and `opencode-data/` + `opencode-config/` → + opencode's two XDG dirs). **Survives `destroy`** — recreating a sandbox restores + history. It also survives `down`/`up`, which the rootfs does not: vz + re-clones `disk.raw` from the image master on every boot, so a runner + whose home isn't mounted here starts fresh each time the VM comes up. - `namespaces//worktrees//` — git worktrees for ticket-mode sandboxes. - `cache/` — built rootfs disks (CoW masters) and kernels, shared across @@ -210,6 +214,12 @@ lexer + recursive-descent parser in the go.mod two-form style - **Egress:** default-deny beyond a built-in allow-list of common registries + the configured domains/IPs, enforced in the userspace stack the guest can't reconfigure. +- **Host devices:** a serial port is reachable only if the user attached it, + and the guest names a *device*, never a host path — the mapping to + `/dev/cu.…` stays on the host, which validates every attach against the + configured set. The same shape as reverse forwarding, and deliberately not + a general "run something on the host" channel: the host picks the resource, + the guest only picks the bytes. - **Host credentials:** the ssh-agent is *forwarded* (keys stay on the host); the Claude OAuth token and any `files ( … )` secrets are pushed in deliberately and are the user's explicit choice. diff --git a/README.md b/README.md index 0795a9a..ce1354d 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ prompt every few seconds), or you run `--dangerously-skip-permissions` and hope nothing important is one `rm -rf` or one leaked token away. clawk is a third option. `cd` into a repo, type `clawk`, and Claude Code (or -Codex, or a shell) is working inside a disposable Linux VM (your code mounted +Codex, or pi, or a shell) is working inside a disposable Linux VM (your code mounted in, root in the guest, no permission prompts) while your files, your keychain, and the rest of your machine stay out of reach. **The agent gets its own machine instead of yours.** @@ -156,7 +156,7 @@ The everyday case, a sandbox for the directory you're in: cd ~/code/my-project clawk # boot a sandbox for this dir + attach claude clawk run shell # drop into a shell in the same sandbox -clawk run codex # or another agent: codex, opencode, shell +clawk run codex # or another agent: codex, pi, opencode, shell clawk down # stop the VM (repo + agent state persist) clawk attach # come back later — boots if stopped, reattaches claude clawk destroy # remove the VM (conversation history is kept) @@ -196,7 +196,7 @@ lives on the host.* | | `clawk down` | `clawk destroy` | | --- | :---: | :---: | | Your repo (mounted worktree; commits, branches) | ✅ | ✅ | -| Agent state (Claude/Codex conversations, memory) | ✅ | ✅ | +| Agent state (Claude/Codex/pi/opencode conversations, memory) | ✅ | ✅ | | The VM disk (apt installs, caches, `$HOME`) | ❌ (rebuilt fresh at every boot*) | ❌ (that's the point) | \* Two exceptions: resuming a `clawk snapshot` restores the disk and @@ -204,16 +204,23 @@ memory exactly as suspended, and the Linux/firecracker provider keeps its disk until destroy. Tools every boot needs belong in the image (`vm ( image … )`); per-boot setup belongs in `on up` hooks. -Agent state is host-mounted per sandbox: the guest's `~/.claude/projects/` -and `~/.claude/memory/` (and codex's `~/.codex/`) live under +Agent state is host-mounted per sandbox: each runner's home directory — +claude's `~/.claude/`, codex's `~/.codex/`, pi's `~/.pi/`, opencode's two XDG +dirs — live under `~/.clawk/namespaces/default/state//` on the host, so a recreated -sandbox picks up its old conversations with `--resume`. +sandbox picks up its old conversations with `--resume`. That mount is what +makes the promise real: the VM disk itself is re-cloned from the image on +every boot, so anything a runner writes outside those directories is gone +at the next `clawk up`. ## Full autonomy by default (and the `--safe` opt-out) Runners launch in their "externally sandboxed" modes: claude gets `--dangerously-skip-permissions`, codex gets -`--dangerously-bypass-approvals-and-sandbox`. On your own machine those flags +`--dangerously-bypass-approvals-and-sandbox`, pi gets `--approve` (it has no +approval prompts to bypass — it ships no sandbox at all — but it does gate +project-local `.pi/` settings and extensions behind a trust prompt), and +opencode gets `--auto`. On your own machine those flags would be reckless; here they are the point: the VM boundary and the network allow-list provide the containment, so the agent works at full speed without per-action prompts. The agent can only affect what you mounted and @@ -261,6 +268,9 @@ sandbox my-project ( forwards ( 3000 ) env ( DATABASE_URL ) # forward a host var; values come from your shell # also: GH=${OTHER_NAME}, LOG=${LOG:-info} defaults, API=${API:?required} + mcp ( # MCP servers, ready on first boot + linear https://mcp.linear.app/mcp header "Authorization: Bearer ${LINEAR_TOKEN}" + ) on create ( "go mod download" ) agent ( instructions "Ask before running destructive commands." @@ -271,8 +281,11 @@ sandbox my-project ( The block is a *template*: snapshotted when the sandbox is created, so a running sandbox never changes unexpectedly. The full reference (shares, secret files, skills, agent memory seeding, multi-repo workspace roots) is -in **[docs/configuration.md](docs/configuration.md)**; images and custom -guest kernels (including the KVM-enabled kernel used for nested +in **[docs/configuration.md](docs/configuration.md)**; MCP servers and how +their credentials stay off disk are in **[docs/mcp.md](docs/mcp.md)**; +putting a USB-serial board from your Mac inside the sandbox for +microcontroller work is in **[docs/serial.md](docs/serial.md)**; images +and custom guest kernels (including the KVM-enabled kernel used for nested virtualization) are in **[docs/images.md](docs/images.md)**. ## Lifecycle @@ -304,7 +317,7 @@ you ──▶ clawk CLI ──▶ per-sandbox daemon (detached; owns the VM) └─ VM: Virtualization.framework (macOS) / firecracker (Linux) ├─ clawk-init, PID 1 (no systemd, no cloud-init) ├─ your repo, live-mounted over virtio-fs - └─ claude / codex / shell on a PTY + └─ claude / codex / pi / shell on a PTY ``` A few deliberate choices, in brief: diff --git a/docs/commands.md b/docs/commands.md index 06e2d90..a7260a6 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -13,6 +13,10 @@ clawk resume [] # continue a paused or snapshotted sandbo clawk down [] # stop; discards any snapshot (next up is a cold boot) clawk destroy [] # remove (host-side state persists) +clawk serial add # a host serial port, inside the guest +clawk serial list [--json] # forwarded ports + whether they're present +clawk serial remove # stop forwarding one + clawk system info [--json] # host prereqs + active components clawk system df [--json] # disk usage by sandbox / cache clawk system prune [--image] # reap unreferenced OCI rootfs disks @@ -41,8 +45,8 @@ field is optional. ## Runners -Built-in runners: `claude`, `codex`, `opencode`, `shell`. The dispatch -shape is the same for all four: +Built-in runners: `claude`, `codex`, `pi`, `opencode`, `shell`. The dispatch +shape is the same for all five: ```sh clawk run [] [-- ] @@ -55,21 +59,36 @@ clawk run claude # cwd-sandbox clawk run claude foo # named sandbox clawk run claude -- --resume # pass-through args clawk run codex foo -- --model o4 +clawk run pi foo -- --resume # pi's own session picker clawk run shell foo # interactive bash ``` +The `clawk-dev` default image ships all four agents, so every name above works +out of the box. A runner an alternative image doesn't have fails with a plain +"command not found". + Each attach starts a fresh agent process in the guest and ends when you -disconnect; claude and codex resume from their own on-disk state next time, so +disconnect; every runner resumes from its own on-disk state next time, so detaching and reattaching is cheap. -State that should outlive the VM is kept on the host: +State that should outlive the VM is kept on the host — one directory per +runner, mounted over the guest's home: + +| Path on host (default namespace) | Mounted as | +|-------------------------------------------------------------|-----------------------------| +| `~/.clawk/namespaces/default/state//claude/` | `~/.claude/` | +| `~/.clawk/namespaces/default/state//codex/` | `~/.codex/` | +| `~/.clawk/namespaces/default/state//pi/` | `~/.pi/` | +| `~/.clawk/namespaces/default/state//opencode-data/` | `~/.local/share/opencode/` | +| `~/.clawk/namespaces/default/state//opencode-config/` | `~/.config/opencode/` | -| Path on host (default namespace) | Mounted as | -|---------------------------------------------------------------|-----------------------| -| `~/.clawk/namespaces/default/state//claude/projects/` | `~/.claude/projects/` | -| `~/.clawk/namespaces/default/state//claude/memory/` | `~/.claude/memory/` | -| `~/.clawk/namespaces/default/state//codex/` | `~/.codex/` | +opencode needs two because it follows the XDG split rather than keeping one +home directory. Its `~/.local/state/opencode` (locks) and `~/.cache/opencode` +are deliberately left on the disposable rootfs. +This is the only thing that persists a runner's sessions: the VM disk is +re-cloned from the image on every boot, so a runner writing anywhere else +loses its history at the next `clawk up`, not just at `clawk destroy`. `clawk destroy` wipes the VM disk but not the state directory, so a recreate returns the same conversation history. diff --git a/docs/configuration.md b/docs/configuration.md index 524a44e..92bf672 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -36,6 +36,7 @@ sandbox my-project ( memory 4GiB memory_max 8GiB disk 64GiB + swap 8GiB nested image golang:1.25 ) @@ -65,6 +66,16 @@ sandbox my-project ( ~/.terraform.d rw ) + serial ( + /dev/cu.usbmodem* ttyACM0 # an Arduino; the glob survives a re-enumeration + ) + + mcp ( + linear https://mcp.linear.app/mcp header "Authorization: Bearer ${LINEAR_TOKEN}" + sentry sse https://mcp.sentry.dev/sse + github stdio "npx -y @modelcontextprotocol/server-github" env GITHUB_TOKEN + ) + skills ( # A manifest of distributed skills for `clawk mod tidy` to pin. # Fetching them into the guest is not implemented yet — provision a @@ -102,7 +113,8 @@ sandbox my-project ( - `sandbox ( … )` — the header names the template (defaults to the directory when omitted: `sandbox ( … )`). - `vm ( … )` — runtime shape: `provider`, `cpu`, `memory`, `memory_max`, - `disk`, `nested`, `idle_timeout`, `image`, `kernel`. Memory and `disk` + `disk`, `swap`, `nested`, `idle_timeout`, `image`, `kernel`. Memory, + `disk` and `swap` sizes require an explicit unit, case-sensitive: IEC (`MiB`/`GiB`/`TiB`, shorthands `M`/`G`/`T`) or SI (`MB`/`GB`/`TB`); SI values convert to MiB rounding down (`1GB` → 953 MiB). `disk` sets the root filesystem ceiling @@ -112,8 +124,16 @@ sandbox my-project ( written up front. Raise it for repos with large dependency trees. Like `cpu` and `memory`, the value is snapshotted when the sandbox is created and baked into the rootfs, so editing it affects the next sandbox (or the - next rootfs rebuild), not a running one. See [Images](images.md) for - `image` and `kernel`, and + next rootfs rebuild), not a running one. `swap` sizes the guest's swap + device (default 2 GiB, minimum 64 MiB, `swap off` to disable). Every + sandbox gets one because clawk reclaims guest RAM under host memory + pressure, and a guest with nowhere to page out answers that with + multi-second stalls — long enough for a streaming API response to lose its + connection. It's a separate sparse file rather than part of the rootfs, so + it costs host bytes only as the guest actually swaps, and resizing it takes + effect on the next boot rather than on a rootfs rebuild. Across a workspace + the largest `swap` wins, but a single `swap off` anywhere disables it for + the whole VM. See [Images](images.md) for `image` and `kernel`, and [Commands & resource usage](commands.md#resource-usage) for `idle_timeout`. - `network ( … )` — egress policy: `allow` / `deny` a domain or `ip `, plus `use …` chains — see @@ -143,6 +163,28 @@ sandbox my-project ( shell-variable shaped (letters, digits, `_`; not starting with a digit) — lowercase names like `http_proxy` are fine. Whitespace around `=` is optional. + + A declaration here **overrides** any variable clawk would otherwise set + for the runner itself, including `CLAUDE_CODE_OAUTH_TOKEN`. That is how a + sandbox opts out of clawk's Anthropic credential when it points claude at + something else: + + ```text + env ( + ANTHROPIC_BASE_URL = https://gateway.example + CLAUDE_CODE_OAUTH_TOKEN = "" # don't send Anthropic's token there + ) + ``` + + This matters when the gateway needs no credential of its own, or + authenticates some other way. Claude Code's credential order is + `ANTHROPIC_AUTH_TOKEN` first, then `CLAUDE_CODE_OAUTH_TOKEN`, and an + empty value counts as unset — so a sandbox that *does* set + `ANTHROPIC_AUTH_TOKEN` already sends the gateway key regardless. Without + one, clawk's `sk-ant-oat-…` goes to that third-party endpoint as a + `Authorization: Bearer` credential, which is a token leak rather than a + broken request. The blunt alternative, `clawk auth clear`, disarms every + sandbox on the host. - `on create ( … )` / `on up ( … )` — shell hooks. `create` runs once after the first boot; `up` runs on every boot. Each command runs inside the guest via `bash -lc` as a login shell: variable expansion, globs, and @@ -156,6 +198,20 @@ sandbox my-project ( configs that rotate rarely). - `shares ( … )` — host directories live-mounted via virtio-fs (good for rotating secrets like AWS STS tokens). +- `serial ( … )` — host serial ports presented as devices in the guest, for + microcontroller work. Each line is ` []`; the name + defaults to the host device's basename, and the host device may be a glob + (resolved when the port is opened, which is how a board that re-enumerates + into its bootloader stays reachable). vz only, and applied live. See + [Serial devices](serial.md). +- `mcp ( … )` — MCP servers made available to the agent, ready on first + boot. See [MCP servers](mcp.md). Each line is + ` ` (http), ` http|sse `, or + ` stdio ""`, followed by any number of + `header "Name: value"` (http/sse) or `env NAME` (stdio) modifiers. + Declaring a server also allows its host in the egress policy, so it needs + no matching `network allow` line. Credentials are referenced, never + stored: write `${VAR}` and declare `VAR` in `env ( … )`. - `skills ( … )` — a manifest of **distributed** Claude skills (`/…` pinned to a version), maintained with `clawk mod tidy`. **Fetching skills into the guest is not implemented yet** — until it @@ -198,6 +254,81 @@ tied to any single repo's directory. A repo listed in `includes` keeps its own per-worktree `on up` / `on create`; the two scopes are independent and both run. +## Host-wide defaults + +Some settings are properties of *you*, not of the repo: a kernel path, a token +alias, a personal skill mount, a house rule about commit messages. Put those in +one file outside any repo and every sandbox on the machine picks them up — +including repos with no `clawk.mod` at all. + +```text +# ~/.config/clawk/clawk.mod — never committed, one per host +sandbox ( + vm ( + memory_max 8GiB + ) + network ( + allow *.internal.myco.com + ) + env ( + GITHUB_TOKEN = ${MYCO_GH_TOKEN} + ) + shares ( + ~/.claude/skills/idiomatic-go + ) + agent ( + instructions ./house-rules.md + ) + on up ( + "scripts/ensure-swap.sh" + ) +) +``` + +It is the lowest layer of the chain, narrowest wins: + +```text +built-in defaults < ~/.config/clawk/clawk.mod < namespace < repo clawk.mod + < clawk.mod. < flags +``` + +Lists union with the host-wide entries first (so a conflict message reads +scope-outward, and `on up` hooks from here run before a repo's). Scalars — +`provider`, `image`, `kernel`, `cpu`, `memory`, `disk`, `swap`, `idle_timeout` — +apply only where nothing narrower declared one. Like every other template it is +read at **sandbox-create time**, so editing it changes the next sandbox you +create, never one that already exists. + +The block must be **anonymous** (`sandbox ( … )`): a header name labels one +repo's phases, which means nothing for defaults, so a name here is an error +rather than a silent relabel of every repo on the host. `includes` is rejected +for the same reason. `policy ( … )` blocks are welcome — that's a +personal policy library. `namespace` blocks are not: this file declares defaults +for every sandbox, not named resources, and clawk owns those records itself. + +Relative paths (`./house-rules.md`, `./kernels/vmlinux`) resolve against the +file's own directory, so they keep working from any repo. + +**Where it lives**, in resolution order: + +| location | notes | +|---|---| +| `$CLAWK_GLOBAL_MOD` | explicit override; must exist, or it's an error | +| `$XDG_CONFIG_HOME/clawk/clawk.mod` | the documented home, default `~/.config/clawk/clawk.mod` | +| `~/.clawk/clawk.mod` | compatibility fallback | + +`~/.config` rather than `~/.clawk` because this is the one file in clawk's +footprint you hand-edit and might symlink out of a dotfiles repo: `~/.clawk` is +disposable machine state (VM disks, an image cache, per-sandbox records, a live +OAuth token) that people exclude from backups and delete to start clean. Having +both locations present is an error, never a silent precedence pick. + +**`--no-global`** ignores the layer entirely — for a CI run or a bug report +that should depend only on what's in the repo. `CLAWK_GLOBAL_MOD=/path/to/file` +does the opposite: pins the layer regardless of what the host has in +`~/.config`. When the layer applies, `clawk work` names the file it came from, +since it explains configuration nobody can find by reading the repo. + ## How clawk finds the file `clawk` in a repo uses that repo's own `clawk.mod` (beside its `.git`), diff --git a/docs/images.md b/docs/images.md index 0de9be2..a84c681 100644 --- a/docs/images.md +++ b/docs/images.md @@ -13,9 +13,9 @@ clawk image gc [--dry-run] [--layers] # reclaim disks no sandbox needs `clawk-dev` (`ghcr.io/clawkwork/clawk-dev`) bundles `go`, `node` + `pnpm`, `python3` + `uv`, `rustc` + `cargo`, `bun`, `zig`, plus `git`, `gh`, `jq`, -`ripgrep`, `claude`, and `codex`. The rootfs is rebuilt from the image each -boot, so bake system dependencies into the image and use `on up` for per-boot -setup. +`ripgrep`, `fd-find`, `claude`, `codex`, `pi`, and `opencode`. The rootfs is +rebuilt from the image each boot, so bake system dependencies into the image +and use `on up` for per-boot setup. A sandbox records the image *reference* it was created with, and the reference is re-resolved against the registry each time the rootfs is diff --git a/docs/mcp.md b/docs/mcp.md new file mode 100644 index 0000000..0f17a3b --- /dev/null +++ b/docs/mcp.md @@ -0,0 +1,127 @@ +# MCP servers + +Declare the MCP servers a project needs and every sandbox created from that +`clawk.mod` comes up with them configured — no per-sandbox setup step, no +interactive login inside the VM. + +```text +sandbox my-project ( + mcp ( + linear https://mcp.linear.app/mcp header "Authorization: Bearer ${LINEAR_TOKEN}" + github stdio "npx -y @modelcontextprotocol/server-github" env GITHUB_TOKEN + ) + + env ( + LINEAR_TOKEN = ${LINEAR_TOKEN:?create a Linear PAT and export it} + GITHUB_TOKEN + ) +) +``` + +That is the whole setup. On `clawk up`, clawk renders the server list into +the guest before the VM boots, allows each server's host in the egress +policy, and hands the runner the config file — so the agent's first tool +call works. + +## Line shapes + +| Written as | Transport | +| --- | --- | +| `name https://host/mcp` | `http` (the default) | +| `name http https://host/mcp` | `http`, explicit | +| `name sse https://host/sse` | `sse` | +| `name stdio "cmd --flag arg"` | `stdio`, a local process | + +Modifiers repeat and may be combined on one line: + +- `header "Name: value"` — an extra HTTP header, for `http` / `sse`. +- `env NAME` — an environment variable handed to a `stdio` server. + +## Credentials: use a token, not a browser + +Only static credentials are supported, and that's deliberate: a personal +access token is a value that can be in place *before* the VM boots, which is +what makes a fresh sandbox usable immediately. Interactive OAuth +(`claude mcp login`) can't be arranged ahead of time — it stores a grant +inside one guest, so every new sandbox would need its own login. + +Most services that offer an MCP endpoint also issue a PAT. Prefer it. + +Credential **values** never enter clawk's config or state: + +- `clawk.mod` holds a `${VAR}` reference; so does the config clawk renders + into the sandbox state dir on the host. +- The value is read from your shell at attach time and delivered straight to + the runner's process environment over the vsock handshake, where the + runner expands `${VAR}` as it connects. + +Declare the variable in `env ( … )` so it's carried. The `${VAR:?message}` +form is worth using: a missing PAT then fails sandbox creation with your +message instead of producing a server that quietly returns 401 mid-task. + +A URL with credentials in it — `https://user:token@host/mcp` — is rejected +for the same reason. The URL is stored verbatim on the sandbox record and in +the rendered guest config, both unencrypted on host disk, so it is the one +spelling that would defeat the guarantee above. Move the credential to a +`header` and it stays in the runner's environment. + +## Servers that need no declaration + +If a service is available as a **claude.ai connector**, use that instead of +declaring the server here. Connectors are authorized once against your +Anthropic account and proxied server-side, so they reach a sandbox with no +local credential, no config, and no egress rule of its own — they ride the +token clawk already forwards. Check with `claude mcp list` inside a sandbox; +anything listed as `claude.ai ` is already working. + +Declare a server in `mcp ( … )` when there is no connector for it, or when +you want to pin a specific endpoint. + +## Scopes + +`mcp ( … )` is valid in a repo `clawk.mod`, a workspace root, and a +`namespace` block. They merge by server name, narrowest scope winning — so a +namespace is the place for the org-wide set, and which namespace a sandbox +belongs to then decides what it can reach: + +```text +namespace acme ( + mcp ( + linear https://mcp.linear.app/mcp header "Authorization: Bearer ${LINEAR_TOKEN}" + ) +) +``` + +Two repos in one workspace declaring the same server identically is fine. +Declaring the same *name* with different targets is rejected, naming both +sources — clawk won't silently pick one. + +## Egress + +Every declared `http` / `sse` host is allowed automatically, in a network +block of origin `mcp`. It sits at the bottom of the precedence chain, just +above the namespace layer: a `deny` you write in `clawk.mod` or via +`clawk network deny` still wins, so the derivation is a convenience and +never a way around your own rules. + +`stdio` servers get no allow — they're a local process. They may still need +egress of their own for whatever they talk to, and one for the package +registry if the command is an `npx`-style fetch (already in the default +allowlist). + +## Notes + +- The rendered config lives at `~/.claude/mcp/clawk.json` in the guest, + passed to the runner with `--mcp-config`. It is not `.mcp.json` in your + repo (clawk won't write into your worktree) and not `~/.claude.json` + (concurrent-write races, and clawk uses it as the onboarding marker). +- Editing `mcp ( … )` takes effect on the next `clawk up` — the file is + rewritten every boot. Removing an entry retires the server. +- clawk does not pass `--strict-mcp-config`, so claude.ai connectors and any + plugin MCP servers your settings enable keep working alongside these. +- Only the `claude` runner is wired today. `codex` and `opencode` use their + own MCP config formats and `pi` loads MCP through an extension rather than + a flag; a sandbox declaring servers simply doesn't get a config flag for + those runners. +- An `npx`-style `stdio` server downloads on first use in each fresh VM. Put + the fetch in `on create ( … )` if you want the first tool call to be fast. diff --git a/docs/networking.md b/docs/networking.md index 40b9f6e..48d57dc 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -66,6 +66,12 @@ clawk policy refresh clawk policy delete ``` +One layer is derived rather than written: each `http`/`sse` server in an +`mcp ( … )` block contributes an allow for its host (see +[MCP servers](mcp.md)). It sits just above the namespace layer, below +everything you wrote yourself — so declaring a server saves you a `network +allow` line, but a `deny` of your own still wins. + `clawk apply -f ` registers `policy` and `namespace` blocks from manifest files (same grammar, no sandbox created). A directory applies every file independently — one broken manifest is reported by name @@ -120,6 +126,11 @@ rather than silently doing nothing. Reverse forwards can also be declared in `clawk.mod` — see [Configuration](configuration.md#reference). +A serial port is the same idea pointed at hardware rather than a socket: +`clawk serial add` puts a USB-serial device from your Mac into the guest as +`/dev/`, over the same vsock transport and with the same live-apply and +vz-only caveats. See [Serial devices](serial.md). + ### Recipe: the Claude Code IDE plugin The JetBrains and VS Code plugins run a websocket server on the host's diff --git a/docs/serial.md b/docs/serial.md new file mode 100644 index 0000000..10ea424 --- /dev/null +++ b/docs/serial.md @@ -0,0 +1,168 @@ +# Serial devices + +Present a serial port plugged into your Mac — an Arduino, an ESP32, a +USB-TTL adapter — as a device inside a sandbox, so `arduino-cli`, `esptool`, +`avrdude` and a serial monitor can run in there against real hardware. + +```sh +clawk serial add /dev/cu.usbmodem1101 # same name in the guest +clawk serial add /dev/cu.usbmodem1101:ttyACM0 # /dev/ttyACM0 in the guest +clawk serial list +clawk serial remove ttyACM0 +``` + +Like reverse forwards, these apply to a running sandbox immediately — no +`down`/`up` cycle — and they are vz (macOS) only, because the guest is the +end that dials and firecracker's vsock is one-way. + +## What actually crosses + +The USB device is **not** passed through. No hypervisor clawk targets can do +that: Virtualization.framework's USB controller carries virtual mass-storage +devices only, with no API for a physical device, and firecracker has no USB +at all — no PCI bus to hang a controller off. + +What crosses is the serial stream and its line settings, over vsock. Inside +the guest that arrives as a PTY symlinked to `/dev/`; on the host, +clawk opens the real port and pumps bytes between the two. This is what every +serial tool actually wants — none of them care that a tty is behind a USB +device rather than a 16550. + +Two consequences worth knowing up front: + +- **The port is only held while the guest is using it.** clawk opens the + physical device when a process in the sandbox opens `/dev/` and + closes it when that process lets go. The Arduino IDE on your Mac can have + the board the rest of the time. +- **Opening the device resets the board.** That is not a clawk behaviour, it + is the auto-reset circuit on the board: opening a serial port asserts DTR, + and DTR is wired to RESET through a capacitor. It's the same reason opening + the Arduino IDE's serial monitor reboots an Uno. Because the host open is + tied to the guest open, this happens at exactly the moment the tooling + expects it. + +## What doesn't cross: DTR and RTS + +A PTY has no modem-control lines. `TIOCMGET` and `TIOCMSET` return `ENOTTY` +on both ends of one, so a guest tool that toggles DTR or RTS explicitly gets +an error, and no amount of protocol work on clawk's side can change that. + +In practice this affects less than it sounds like, because the two things +those lines are used for both have another path: + +| Board style | How it enters the bootloader | Works? | +|---|---|---| +| Native USB (Leonardo, Micro, most ESP32-S3, RP2040) | 1200-baud touch — open at 1200 baud, close | **Yes.** A PTY does carry the baud rate, and clawk forwards the close too | +| Classic auto-reset (Uno, Nano, Mega) | DTR pulse | **Yes, via the open.** Opening the port asserts DTR, which is the pulse | +| ESP32 with the classic auto-program circuit | DTR *and* RTS in sequence, to drive GPIO0 and EN separately | **No.** Two lines in a specific order can't be expressed | + +For that last row — a plain ESP32 DevKit with `esptool` — hold the BOOT +button while the upload starts, or use a board with native USB. `esptool`'s +`--before no_reset` skips the sequence it can't perform. + +You may still see an `ioctl("TIOCMGET")` warning from `avrdude` even on a +board that uploads fine. It is telling the truth about the ioctl and is +harmless: the reset already happened when the port opened. + +## Boards that re-enumerate + +A board entering its bootloader drops off the USB bus and comes back, often +under a *different* device name — `cu.usbmodem1101` becomes +`cu.usbmodem14201` and back again. A literal path breaks on that; a glob +doesn't: + +```sh +clawk serial add '/dev/cu.usbmodem*:ttyACM0' +``` + +The pattern is resolved each time the port is opened, not when you configure +it. Quote it so your shell doesn't expand it first. A pattern matching two +boards is refused rather than guessed at — flashing the wrong device is the +one failure worth being loud about. + +The guest-side name never changes across a re-enumeration, so `arduino-cli +upload -p /dev/ttyACM0` keeps working through the whole cycle. + +## Declaring devices in clawk.mod + +``` +sandbox firmware ( + serial ( + /dev/cu.usbmodem1101 # /dev/cu.usbmodem1101 in the guest + /dev/cu.usbserial-A50285BI ttyUSB0 # /dev/ttyUSB0 in the guest + /dev/cu.usbmodem* ttyACM0 # resolved at open time + ) +) +``` + +Host device first, optional guest name second — space-separated, matching +`files` and `shares` rather than the CLI's colon form, because a colon inside +a path is ambiguous in a way a port number never is. + +Two entries claiming the same guest name, or the same host port, are refused +with an error naming both contributors rather than silently resolved. + +## Working with a board from the sandbox + +`arduino-cli board list` won't find anything: it enumerates USB VID/PID +through libusb, and there is no USB in there. Name the port and the board +explicitly instead — which is what you'd do in CI anyway: + +```sh +arduino-cli compile --fqbn arduino:avr:uno sketch/ +arduino-cli upload --fqbn arduino:avr:uno -p /dev/ttyACM0 sketch/ +arduino-cli monitor -p /dev/ttyACM0 -c baudrate=115200 +``` + +`esptool` and `avrdude` take `-p`/`-P` the same way. Anything that opens a +tty and sets a baud rate works; `screen`, `picocom` and `cat` are all fine. + +One-shot writes work too, but note what they cost: + +```sh +echo 'status?' > /dev/ttyACM0 # opens, writes, closes — and resets +``` + +The host port is open only while a process in the sandbox holds the device, +so a command like that opens and closes it around a single write — and since +opening asserts DTR, each one resets a board wired for auto-reset. Fine for a +one-off; wrong for a loop. Hold the device open instead, and the port stays +open with it: + +```sh +exec 3<>/dev/ttyACM0 # one open, one reset +echo 'status?' >&3 +cat <&3 & +exec 3<&- # release it +``` + +## On macOS: `cu.` not `tty.` + +Use the callout device (`/dev/cu.*`), not the dial-in device +(`/dev/tty.*`). The dial-in side blocks on carrier detect, which shows up as +a port that opens and then does nothing. `clawk serial add` warns if you name +a `tty.` device. + +## Troubleshooting + +**"no device matches"** — the board isn't plugged in, or is mid-reset. clawk +waits about three seconds for it during an attach, which covers a +re-enumeration; past that the guest retries on its own. `clawk serial list` +shows what's present right now. + +**"already in use"** — something else has the port. Inside the guest, only +one process can hold a device at a time; on the Mac, check for an open +Arduino IDE serial monitor. + +**Uploads hang at "not in sync"** — the board didn't reset. See the DTR table +above; for a classic ESP32 DevKit, hold BOOT. + +**Nothing appears at `/dev/`** — the sandbox has to be running vz and +its guest agent has to be current. `clawk down && clawk up` re-injects the +agent; `clawk serial add` says so explicitly when the daemon is too old. + +## See also + +- [Networking](networking.md) — port forwarding in both directions, which + serial forwarding is deliberately shaped like +- [Configuration](configuration.md) — the full `clawk.mod` reference diff --git a/images/clawk-dev/Dockerfile b/images/clawk-dev/Dockerfile index 4967360..151e6eb 100644 --- a/images/clawk-dev/Dockerfile +++ b/images/clawk-dev/Dockerfile @@ -40,8 +40,14 @@ FROM node:22-bookworm-slim COPY --from=go /usr/local/go /usr/local/go ENV PATH=/usr/local/go/bin:$PATH -# Coding agents. claude is what bare `clawk` attaches to; codex is the -# other built-in runner. +# Coding agents. claude is what bare `clawk` attaches to; codex, pi, and +# opencode are the other built-in runners, all installed here so every +# `clawk run ` in the registry works out of the box. +# +# opencode-ai resolves a large prebuilt platform binary (~180 MB for +# linux-arm64) rather than shipping JS, so it dominates this layer's size. +# Drop it from the list for a slimmer image; `clawk run opencode` then +# fails with a plain "command not found". # # The global npm prefix is relocated to /usr/local/npm-global and made # world-writable so claude/codex can self-update at runtime. Why not just @@ -58,6 +64,7 @@ ENV NPM_CONFIG_PREFIX=/usr/local/npm-global ENV PATH=/usr/local/npm-global/bin:$PATH RUN mkdir -p /usr/local/npm-global \ && npm install -g @anthropic-ai/claude-code @openai/codex \ + @earendil-works/pi-coding-agent opencode-ai \ && npm cache clean --force \ && chmod -R a+rwX /usr/local/npm-global @@ -68,10 +75,18 @@ RUN mkdir -p /usr/local/npm-global \ # runs root work through the agent and never needs it. sox is the audio # recorder Claude Code's voice dictation falls back to on Linux; it reads # the virtio-snd mic (CLAWK_AUDIO_INPUT, on by default) via ALSA. +# +# ripgrep and fd-find are here for pi as much as for the agents' own use: +# pi's search tools look for `rg` and `fd`/`fdfind` on PATH and, finding +# neither, download release tarballs from GitHub on first run ("fd not +# found. Downloading..."). Debian ships the fd binary as `fdfind` to avoid +# a name clash, which is precisely one of the names pi probes for, so the +# package satisfies it as-is — no symlink needed. RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ ca-certificates \ curl \ + fd-find \ git \ jq \ less \ diff --git a/internal/agentembed/agent_compile_test.go b/internal/agentembed/agent_compile_test.go index a576fb2..7f728d3 100644 --- a/internal/agentembed/agent_compile_test.go +++ b/internal/agentembed/agent_compile_test.go @@ -73,6 +73,38 @@ func TestAgentSourceCompiles(t *testing.T) { } } +// TestInitSourceCompiles is TestAgentSourceCompiles for clawk-init. The +// init is built by the same guestbuild path and injected into every OCI +// sandbox disk, but it went uncovered while only the agent was compiled +// here — and it is the harder of the two to review by eye: PID 1, raw +// syscalls, and no way to observe a failure except a guest that boots +// wrong. +func TestInitSourceCompiles(t *testing.T) { + if _, err := exec.LookPath("go"); err != nil { + t.Skip("`go` not on PATH") + } + if os.Getenv("AGENT_TEST_NO_BUILD") != "" { + t.Skip("AGENT_TEST_NO_BUILD set") + } + + tmp := t.TempDir() + mustWrite(t, filepath.Join(tmp, "main.go"), InitMainGo) + mustWrite(t, filepath.Join(tmp, "go.mod"), InitGoMod) + + if out, err := runIn(t, tmp, "go", "mod", "tidy"); err != nil { + t.Fatalf("go mod tidy failed: %v\n%s", err, out) + } + bin := filepath.Join(tmp, "clawk-init") + if out, err := runIn(t, tmp, "go", "build", "-o", bin, "."); err != nil { + t.Fatalf("go build failed: %v\n%s", err, out) + } + fi, err := os.Stat(bin) + require.NoError(t, err, "init binary not produced") + if fi.Size() < 100*1024 { + t.Fatalf("init binary suspiciously small (%d bytes); build probably hollow", fi.Size()) + } +} + // mustWrite writes content to path and fatals on error. Tiny helper so // the assertion-heavy tests above stay readable. func mustWrite(t *testing.T, path string, content []byte) { @@ -98,3 +130,45 @@ func runIn(t *testing.T, dir, cmdName string, args ...string) ([]byte, error) { err := cmd.Run() return buf.Bytes(), err } + +// TestSerialForwarderRuns compiles the agent into a throwaway module along +// with its own test file and runs `go test` there. +// +// TestAgentSourceCompiles proves the agent builds; this proves its serial +// half works. That half is a state machine over a PTY with several edges +// that are invisible to a compiler and awkward to reason about: whether a +// hangup really distinguishes "no client" from "idle client", whether a +// baud change with no accompanying event is noticed at all, and whether the +// attachment's lifetime tracks the client's open and close — which is what +// makes a board reset at the right moment. Those are worth executing. +// +// Linux-only and native-arch only: the tests open PTYs and issue termios +// ioctls, so they have to run rather than cross-compile. +func TestSerialForwarderRuns(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("the guest agent's serial tests need a Linux PTY; cross-compiling can't run them") + } + if _, err := exec.LookPath("go"); err != nil { + t.Skip("`go` not on PATH") + } + if os.Getenv("AGENT_TEST_NO_BUILD") != "" { + t.Skip("AGENT_TEST_NO_BUILD set") + } + + tmp := t.TempDir() + mustWrite(t, filepath.Join(tmp, "main.go"), AgentMainGo) + mustWrite(t, filepath.Join(tmp, "go.mod"), AgentGoMod) + mustWrite(t, filepath.Join(tmp, "serial_test.go"), SerialAgentTest) + + if out, err := runIn(t, tmp, "go", "mod", "tidy"); err != nil { + t.Fatalf("go mod tidy failed: %v\n%s", err, out) + } + // Native, not cross-compiled: runIn pins GOOS/GOARCH for the build + // tests, and these have to actually execute. + cmd := exec.Command("go", "test", "-count=1", "-timeout=120s", "./...") + cmd.Dir = tmp + cmd.Env = append(os.Environ(), "CGO_ENABLED=0") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("guest agent serial tests failed: %v\n%s", err, out) + } +} diff --git a/internal/agentembed/embed.go b/internal/agentembed/embed.go index e4948ba..82fdf22 100644 --- a/internal/agentembed/embed.go +++ b/internal/agentembed/embed.go @@ -54,3 +54,12 @@ var InitMainGo []byte // //go:embed init_go.mod.in var InitGoMod []byte + +// SerialAgentTest is the agent's own test suite for the serial forwarder, +// run inside a throwaway module built from AgentMainGo — see +// TestSerialForwarderRuns. Embedded rather than kept as a normal _test.go +// for the same reason main.go.in is: it is package main in the guest's +// module, not in this one. +// +//go:embed serial_agent_test.go.in +var SerialAgentTest []byte diff --git a/internal/agentembed/go.mod.in b/internal/agentembed/go.mod.in index 81d787d..5f2129c 100644 --- a/internal/agentembed/go.mod.in +++ b/internal/agentembed/go.mod.in @@ -5,4 +5,5 @@ go 1.25.0 require ( github.com/creack/pty v1.1.21 github.com/mdlayher/vsock v1.2.1 + golang.org/x/sys v0.47.0 ) diff --git a/internal/agentembed/init_main.go.in b/internal/agentembed/init_main.go.in index 64c360b..fba2134 100644 --- a/internal/agentembed/init_main.go.in +++ b/internal/agentembed/init_main.go.in @@ -25,9 +25,11 @@ package main import ( + "encoding/binary" "encoding/json" "errors" "fmt" + "io" "net" "os" "os/exec" @@ -37,6 +39,7 @@ import ( "strings" "sync" "time" + "unsafe" "github.com/vishvananda/netlink" "golang.org/x/sys/unix" @@ -51,11 +54,21 @@ type manifest struct { Hostname string `json:"hostname,omitempty"` Network *manifestNet `json:"network,omitempty"` User *manifestUser `json:"user,omitempty"` + Swap *swapSpec `json:"swap,omitempty"` Mounts []mountSpec `json:"mounts,omitempty"` Files []fileSpec `json:"files,omitempty"` Services []serviceSpec `json:"services,omitempty"` } +// swapSpec is the swap device to format and enable. Nil for a sandbox with +// swap disabled, and absent from manifests written by a clawk that predates +// swap — an additive field, so an old init ignores it and a new init sees +// nil, both booting without swap. +type swapSpec struct { + Device string `json:"device"` + Swappiness int `json:"swappiness,omitempty"` +} + type manifestNet struct { Interface string `json:"interface"` Address string `json:"address"` // CIDR, e.g. "192.168.127.2/24" @@ -143,6 +156,16 @@ func main() { } } + // Before the mounts and the services, so the guest already has somewhere + // to put cold anonymous pages by the time anything starts allocating. + // Non-fatal: a sandbox without swap works, it just has no cushion when the + // host's balloon controller reclaims its memory. + if m.Swap != nil { + if err := setupSwap(m.Swap); err != nil { + logf("swap: %v", err) + } + } + for _, mt := range m.Mounts { err := mountShare(mt, m.User) if err == nil { @@ -300,12 +323,18 @@ func loadManifest() (*manifest, error) { return &m, nil } -// waitForDevice opens dev, retrying briefly — virtio-blk probing can race -// the first userspace instructions on a fast direct-kernel boot. +// waitForDevice opens dev read-only, retrying briefly — virtio-blk probing +// can race the first userspace instructions on a fast direct-kernel boot. func waitForDevice(dev string, timeout time.Duration) (*os.File, error) { + return waitForDeviceFlags(dev, os.O_RDONLY, timeout) +} + +// waitForDeviceFlags is waitForDevice with an explicit open mode, for the +// swap device, whose header we have to write. +func waitForDeviceFlags(dev string, flags int, timeout time.Duration) (*os.File, error) { deadline := time.Now().Add(timeout) for { - f, err := os.Open(dev) + f, err := os.OpenFile(dev, flags, 0) if err == nil { return f, nil } @@ -316,6 +345,109 @@ func waitForDevice(dev string, timeout time.Duration) (*os.File, error) { } } +// ──────────────────────────────────────────────────────────────────────── +// Swap +// ──────────────────────────────────────────────────────────────────────── + +// setupSwap formats the swap device and enables it. +// +// The header is written here rather than shelled out to mkswap(8) for the +// same reason the user is created by editing /etc/passwd: the rootfs is an +// arbitrary OCI image, and nothing guarantees util-linux is in it. +func setupSwap(s *swapSpec) error { + if s.Device == "" { + return errors.New("manifest declares swap with no device") + } + f, err := waitForDeviceFlags(s.Device, os.O_RDWR, 5*time.Second) + if err != nil { + return err + } + size, err := f.Seek(0, io.SeekEnd) + if err != nil { + f.Close() + return fmt.Errorf("sizing %s: %w", s.Device, err) + } + err = writeSwapHeader(f, size) + f.Close() + if err != nil { + return err + } + if err := swapon(s.Device); err != nil { + return fmt.Errorf("swapon %s: %w", s.Device, err) + } + if s.Swappiness > 0 { + if err := os.WriteFile("/proc/sys/vm/swappiness", + []byte(strconv.Itoa(s.Swappiness)), 0o644); err != nil { + logf("swap: setting swappiness: %v", err) + } + } + logf("swap: enabled %s (%d MiB, swappiness %d)", s.Device, size>>20, s.Swappiness) + return nil +} + +// swapMagic is the signature that identifies a formatted swap area. It sits +// at the very end of the first page, not at its start. +const swapMagic = "SWAPSPACE2" + +// writeSwapHeader lays a version-1 swap header on the device, the layout +// swapon(2) validates (union swap_header in linux/include/linux/swap.h): +// +// [0:1024) bootbits, left zero +// [1024:1028) version = 1 +// [1028:1032) last_page, the highest usable page index +// [1032:1036) nr_badpages = 0 +// [1036:1068) uuid and label, both left zero +// [pagesize-10:pagesize) "SWAPSPACE2" +// +// The u32s are native-endian: the kernel reads this as a struct off the +// device, not as a wire format, and the page size is the running kernel's +// (arm64 may be 4K or 16K) for the same reason. +// +// Rewritten on every boot rather than validated first. It costs one page, +// and it keeps the header true after the host resizes the device — the +// alternative is a stale last_page that hands the kernel pages the device no +// longer has. +func writeSwapHeader(f *os.File, size int64) error { + pageSize := os.Getpagesize() + pages := size / int64(pageSize) + // The kernel refuses anything under 10 pages outright; a device that + // small means the host wrote the wrong file, not that swap is tiny. + if pages < 10 { + return fmt.Errorf("swap device is %d bytes, too small to format", size) + } + page := make([]byte, pageSize) + binary.NativeEndian.PutUint32(page[1024:1028], 1) // version + binary.NativeEndian.PutUint32(page[1028:1032], uint32(pages-1)) // last_page + binary.NativeEndian.PutUint32(page[1032:1036], 0) // nr_badpages + copy(page[pageSize-len(swapMagic):], swapMagic) + if _, err := f.WriteAt(page, 0); err != nil { + return fmt.Errorf("writing swap header: %w", err) + } + return f.Sync() +} + +// swapFlagDiscard is SWAP_FLAG_DISCARD: discard freed swap pages, which +// would let the sparse backing file give space back to the host. Neither +// vz's nor firecracker's virtio-blk advertises discard today and the kernel +// drops the flag when the device can't honor it, so this is currently a +// no-op that costs nothing and starts paying if either ever does. +const swapFlagDiscard = 0x10000 + +// swapon enables dev as swap. x/sys/unix wraps neither swapon(2) nor its +// flags, so this goes through the raw syscall. +func swapon(dev string) error { + p, err := unix.BytePtrFromString(dev) + if err != nil { + return err + } + _, _, errno := unix.Syscall( + unix.SYS_SWAPON, uintptr(unsafe.Pointer(p)), swapFlagDiscard, 0) + if errno != 0 { + return errno + } + return nil +} + func cmdlineValue(key string) string { data, err := os.ReadFile("/proc/cmdline") if err != nil { diff --git a/internal/agentembed/main.go.in b/internal/agentembed/main.go.in index d04a1fe..1cd5a30 100644 --- a/internal/agentembed/main.go.in +++ b/internal/agentembed/main.go.in @@ -31,6 +31,7 @@ import ( "os" "os/exec" "os/user" + "path/filepath" "strconv" "strings" "sync" @@ -39,6 +40,7 @@ import ( "github.com/creack/pty" "github.com/mdlayher/vsock" + "golang.org/x/sys/unix" ) // ──────────────────────────────────────────────────────────────────────── @@ -632,6 +634,14 @@ func main() { // proxy yet). go runReverseForwarder(logger) + // Serial forwarder: holds a control connection to the host, creates a + // PTY at /dev/ for every serial device the host publishes, and + // bridges each one to the physical port while a process in here has it + // open. Idle unless the sandbox has serial devices configured; retries + // forever if the host end isn't there, exactly like the reverse + // forwarder above. + go runSerialForwarder(logger) + var wg sync.WaitGroup for { conn, err := listener.Accept() @@ -997,7 +1007,7 @@ func revfwdReadLine(r *bufio.Reader, v any) error { // exactly memReportBytes of snapshot, then close. The host's idle watchdog // (internal/cli) polls the same snapshot for the activity fields — load and // network I/O — to tell a quiescent guest from one still working detached. -// The wire format is five big-endian uint64s and MUST stay in lock-step +// The wire format is seven big-endian uint64s and MUST stay in lock-step // with machine/vz/memreport.go (memReport / decodeMemReport): // // [0:8] MemTotal (KiB) @@ -1005,6 +1015,8 @@ func revfwdReadLine(r *bufio.Reader, v any) error { // [16:24] memory PSI "some avg10" × 100 (0 if PSI unavailable) // [24:32] /proc/loadavg load1 × 100 // [32:40] cumulative rx+tx bytes across non-lo interfaces +// [40:48] SwapTotal (KiB) +// [48:56] SwapFree (KiB) // // Hosts accept the legacy 24-byte prefix alone, so new fields may only be // appended, never reordered. @@ -1016,8 +1028,8 @@ const ( // (1026). Keep in sync with machine/vz/memreport.go's agentMemPort. memReportVSockPort = 1027 - // memReportBytes is the fixed snapshot size (5 × uint64). - memReportBytes = 40 + // memReportBytes is the fixed snapshot size (7 × uint64). + memReportBytes = 56 ) // runMemReporter listens on memReportVSockPort and writes a fresh memory @@ -1046,15 +1058,17 @@ func runMemReporter(logger *log.Logger) { // files, so this runs inline rather than in a goroutine. func serveMemReport(conn net.Conn, logger *log.Logger) { defer conn.Close() - total, avail := readMemInfo() + mem := readMemInfo() psi := readMemPSICenti() var buf [memReportBytes]byte - binary.BigEndian.PutUint64(buf[0:8], total) - binary.BigEndian.PutUint64(buf[8:16], avail) + binary.BigEndian.PutUint64(buf[0:8], mem.total) + binary.BigEndian.PutUint64(buf[8:16], mem.avail) binary.BigEndian.PutUint64(buf[16:24], psi) binary.BigEndian.PutUint64(buf[24:32], readLoad1Centi()) binary.BigEndian.PutUint64(buf[32:40], readNetIOBytes()) + binary.BigEndian.PutUint64(buf[40:48], mem.swapTotal) + binary.BigEndian.PutUint64(buf[48:56], mem.swapFree) _ = conn.SetWriteDeadline(time.Now().Add(2 * time.Second)) if _, err := conn.Write(buf[:]); err != nil && !isClosedConn(err) { @@ -1062,14 +1076,30 @@ func serveMemReport(conn net.Conn, logger *log.Logger) { } } -// readMemInfo returns MemTotal and MemAvailable from /proc/meminfo in KiB. -// Missing fields read as zero; a zero MemTotal tells the host "no usable -// data", which it treats as absent rather than "guest has no memory". -func readMemInfo() (total, avail uint64) { +// meminfo is the slice of /proc/meminfo the host's balloon controller +// reads. The swap pair matters because MemAvailable is by definition what +// the guest can reclaim *without* swapping: a guest that has pushed its +// cold anonymous pages out looks roomy by that measure alone, and the +// controller would answer by taking the memory back (see guestDesiredTarget +// in machine/vz/balloon.go). +type meminfo struct { + total uint64 + avail uint64 + swapTotal uint64 + swapFree uint64 +} + +// readMemInfo returns the meminfo fields in KiB. Missing fields read as +// zero; a zero total tells the host "no usable data", which it treats as +// absent rather than "guest has no memory". A guest with no swap device +// reports zero for both swap fields, which is also what a pre-swap guest +// agent's shorter report decodes to. +func readMemInfo() meminfo { data, err := os.ReadFile("/proc/meminfo") if err != nil { - return 0, 0 + return meminfo{} } + var m meminfo for _, line := range strings.Split(string(data), "\n") { key, rest, ok := strings.Cut(line, ":") if !ok { @@ -1077,12 +1107,16 @@ func readMemInfo() (total, avail uint64) { } switch key { case "MemTotal": - total = parseMeminfoKiB(rest) + m.total = parseMeminfoKiB(rest) case "MemAvailable": - avail = parseMeminfoKiB(rest) + m.avail = parseMeminfoKiB(rest) + case "SwapTotal": + m.swapTotal = parseMeminfoKiB(rest) + case "SwapFree": + m.swapFree = parseMeminfoKiB(rest) } } - return total, avail + return m } // parseMeminfoKiB parses a " 4033244 kB" meminfo value into its KiB number. @@ -1171,3 +1205,749 @@ func readMemPSICenti() uint64 { } return 0 } + +// ──────────────────────────────────────────────────────────────────────── +// Serial forwarder +// +// Wire protocol kept in lock-step with internal/serialfwd in the parent +// repo; any change there must be mirrored here. Two connection kinds, both +// guest-initiated, both starting with one JSON greeting line: +// +// op=control the host answers with the full device set now and again on +// every change, so `clawk serial add` applies to a running +// sandbox. +// op=attach the host validates the device name, opens the physical +// port, answers ok, and then speaks frames. +// +// For each device the host publishes we create a PTY, symlink it to +// /dev/, and hold the master. The attach connection exists for +// exactly as long as some process in here holds the slave open, which is +// what makes the host open and close the real port in step — see the +// serialfwd package comment for why that matters to an Arduino. +// ──────────────────────────────────────────────────────────────────────── + +const ( + // serialVSockPort is the host-side port serving both kinds. Mirrors + // serialfwd.VSockPort. Distinct from the pty agent (1024), time-sync + // (1025), ssh-agent (1026), mem-report (1027) and reverse-forward + // (1028). + serialVSockPort = 1029 + + // serialProtoVersion mirrors serialfwd.ProtoVersion. + serialProtoVersion = 1 + + // serialMaxLine mirrors serialfwd.MaxLineBytes. + serialMaxLine = 64 * 1024 + + // serialFrameHeaderBytes / serialMaxFrame mirror + // serialfwd.FrameHeaderBytes and serialfwd.MaxFrameBytes. + serialFrameHeaderBytes = 5 + serialMaxFrame = 256 * 1024 + + // Frame types, mirroring serialfwd.FrameData / serialfwd.FrameMode. + serialFrameData byte = 0x01 + serialFrameMode byte = 0x02 + + // serialControlMinBackoff / serialControlMaxBackoff bound the retry + // delay when the host end isn't answering — the first moments of every + // boot, and permanently under firecracker. Same reasoning as the + // reverse forwarder's. + serialControlMinBackoff = 1 * time.Second + serialControlMaxBackoff = 30 * time.Second + + // serialStableSession is how long a control connection must last to + // count as healthy and reset the backoff. + serialStableSession = 30 * time.Second + + // serialPollInterval is how long the per-device loop parks waiting for + // output from its PTY client. + // + // It is a timeout rather than a pure block because a PTY delivers no + // event when its client changes the line configuration: master and + // slave share the termios state, so the change is visible to a + // tcgetattr but wakes nothing. The loop therefore has to look, and this + // is how often. 100ms is far below human perception for "the monitor + // opened at the wrong baud" and cheap enough to run only while a + // process actually holds the device open. + serialPollInterval = 100 * time.Millisecond + + // serialReopenMinInterval / serialReopenMaxInterval bound how often a + // detached device checks whether a client has come back. + // + // It starts fast on purpose. A board being flashed goes through + // close-then-immediately-reopen (the 1200-baud touch, then the + // bootloader's port), and a slow first check would add dead time to + // every upload. It backs off because an idle sandbox with a forwarded + // board that nobody is using shouldn't wake the vCPU 50 times a second + // forever. + serialReopenMinInterval = 20 * time.Millisecond + serialReopenMaxInterval = 500 * time.Millisecond + + // serialAttachRetryInterval is the pause after the host refuses an + // attach — an unplugged board, or one held by something else on the + // Mac. The client in here still has the PTY open, so we keep trying, + // just not in a spin. + serialAttachRetryInterval = 1 * time.Second + + // serialDrainMaxReads caps how many buffered reads the hangup path + // flushes before giving up — see drainToHost. A PTY's input queue is a + // few kilobytes and forwardOutput moves 4 KiB a time, so this is roughly + // an order of magnitude of headroom over anything reachable: a backstop + // against spinning if a read ever reports data it does not deliver, not + // a limit on legitimate writes. + serialDrainMaxReads = 16 +) + +// serialDevDir is where forwarded devices are published, and +// serialDialHost is how the host end is reached. Both are variables solely +// so the agent's own tests can point them at a temp directory and a local +// listener: creating a node in /dev needs root, and AF_VSOCK needs a +// hypervisor. Nothing at runtime reassigns either. +var ( + serialDevDir = "/dev" + + serialDialHost = func() (net.Conn, error) { + return vsock.Dial(vsockHostCID, serialVSockPort, nil) + } +) + +type serialGreeting struct { + Op string `json:"op"` + V int `json:"v"` + Name string `json:"name,omitempty"` + Mode *serialMode `json:"mode,omitempty"` +} + +type serialSnapshot struct { + Devices []serialDeviceMsg `json:"devices"` +} + +type serialDeviceMsg struct { + Name string `json:"name"` +} + +type serialAttachReply struct { + OK bool `json:"ok"` + Error string `json:"error,omitempty"` +} + +type serialMode struct { + Baud int `json:"baud"` + Bits int `json:"bits"` + Parity string `json:"parity"` + Stop int `json:"stop"` +} + +// serialForwarder owns the per-device PTYs, keyed by guest-visible name. +type serialForwarder struct { + log *log.Logger + + mu sync.Mutex + devices map[string]*serialDevice +} + +// runSerialForwarder keeps the control connection to the host alive, +// applying every device set the host publishes. Never returns. +func runSerialForwarder(logger *log.Logger) { + f := &serialForwarder{log: logger, devices: make(map[string]*serialDevice)} + backoff := serialControlMinBackoff + quiet := false + for { + start := time.Now() + err := f.controlSession() + // Every PTY exists to reach a port through the host, so once the + // host is gone they must go too: a device node left in /dev would + // accept an open and then sit mute, which reads as broken hardware + // rather than an absent forward. + f.apply(nil) + + if time.Since(start) >= serialStableSession { + backoff, quiet = serialControlMinBackoff, false + } + if err != nil && !quiet { + logger.Printf("serial: %v (retrying, quietly from here)", err) + quiet = true + } + time.Sleep(backoff) + if backoff *= 2; backoff > serialControlMaxBackoff { + backoff = serialControlMaxBackoff + } + } +} + +// controlSession dials the host and applies snapshots until the connection +// drops. A clean EOF (the host shutting down) returns nil. +func (f *serialForwarder) controlSession() error { + conn, err := serialDialHost() + if err != nil { + return fmt.Errorf("vsock dial host:%d: %w", serialVSockPort, err) + } + defer conn.Close() + + if err := serialWriteLine(conn, serialGreeting{ + Op: "control", V: serialProtoVersion, + }); err != nil { + return fmt.Errorf("sending control greeting: %w", err) + } + r := bufio.NewReaderSize(conn, serialMaxLine) + for { + var snap serialSnapshot + if err := serialReadLine(r, &snap); err != nil { + if errors.Is(err, io.EOF) || isClosedConn(err) { + return nil + } + return fmt.Errorf("reading device set: %w", err) + } + f.apply(snap.Devices) + } +} + +// apply reconciles the live PTYs with want, which is always the complete +// desired set. +func (f *serialForwarder) apply(want []serialDeviceMsg) { + desired := make(map[string]bool, len(want)) + for _, d := range want { + if err := validSerialName(d.Name); err != nil { + f.log.Printf("serial: ignoring device: %v", err) + continue + } + desired[d.Name] = true + } + + f.mu.Lock() + defer f.mu.Unlock() + + for name, dev := range f.devices { + if desired[name] { + continue + } + dev.stop() + delete(f.devices, name) + } + for name := range desired { + if _, ok := f.devices[name]; ok { + continue + } + dev, err := newSerialDevice(f.log, name) + if err != nil { + f.log.Printf("serial: %s: %v", name, err) + continue + } + f.devices[name] = dev + f.log.Printf("serial: /dev/%s -> %s", name, dev.slavePath) + go dev.run() + } +} + +// serialDevice is one forwarded port: a PTY whose master we hold, a symlink +// in /dev pointing at its slave, and — whenever some process in here has +// that slave open — a vsock connection to the physical port on the host. +type serialDevice struct { + log *log.Logger + name string + + master *os.File + slavePath string + linkPath string + + done chan struct{} + once sync.Once + + // conn and mode belong to the current attachment. Only run() touches + // them, so they need no lock; the reader goroutine gets its own handle. + conn net.Conn + mode serialMode +} + +// newSerialDevice creates the PTY and publishes it at /dev/. +func newSerialDevice(logger *log.Logger, name string) (*serialDevice, error) { + master, slave, err := pty.Open() + if err != nil { + return nil, fmt.Errorf("opening pty: %w", err) + } + slavePath := slave.Name() + + // The agent runs as root but the tools that will open this don't, and + // devpts hands out slaves as root:tty 0620. A sandbox is single-user by + // construction, so widening this is not the concession it would be on a + // shared machine. + if err := os.Chmod(slavePath, 0o666); err != nil { + _ = slave.Close() + _ = master.Close() + return nil, fmt.Errorf("chmod %s: %w", slavePath, err) + } + + linkPath := filepath.Join(serialDevDir, name) + // Replace rather than fail: a stale link from a previous boot, or from + // a device that was removed and re-added, is the normal case. + if err := os.Remove(linkPath); err != nil && !errors.Is(err, os.ErrNotExist) { + _ = slave.Close() + _ = master.Close() + return nil, fmt.Errorf("clearing %s: %w", linkPath, err) + } + if err := os.Symlink(slavePath, linkPath); err != nil { + _ = slave.Close() + _ = master.Close() + return nil, fmt.Errorf("linking %s: %w", linkPath, err) + } + + // Put the line in raw mode before anyone can open it. + // + // A fresh PTY comes up cooked — ICANON, ECHO, OPOST — and that default + // is actively wrong for a forwarded serial port. Echo is the harmful + // part: with it on, every byte the board sends is echoed straight back + // out to the board, which turns a chatty device into a loop. Canonical + // mode is merely surprising, holding input back until a newline that a + // binary protocol never sends. + // + // A real tty has the same unhelpful defaults, and every serial tool + // fixes them on open — but the tools are not the only clients here, and + // a device that misbehaves for `cat` is a device people will assume is + // broken. Anything that does configure the port overrides this anyway. + if err := rawSerialPTY(int(master.Fd())); err != nil { + _ = slave.Close() + _ = master.Close() + return nil, fmt.Errorf("raw mode: %w", err) + } + + // Prime the hangup state by closing our own handle on the slave. + // + // This is what makes "is a client attached?" answerable. A PTY that has + // never had its slave opened polls exactly like one that is open and + // idle — no POLLHUP either way — so without this the loop would think a + // client was present from the moment the device appeared and would open + // the physical port for nobody. Opening and closing the slave once + // drives the master into the sticky hangup state, and from then on + // POLLHUP means precisely "nothing in here has it open". + _ = slave.Close() + + return &serialDevice{ + log: logger, + name: name, + master: master, + slavePath: slavePath, + linkPath: linkPath, + done: make(chan struct{}), + }, nil +} + +func (d *serialDevice) stop() { + d.once.Do(func() { close(d.done) }) +} + +// run is the device's whole life: watch the PTY for a client, hold an +// attachment to the host while one is there, and move bytes and mode +// changes between them. +func (d *serialDevice) run() { + defer func() { + d.detach() + _ = d.master.Close() + // Only remove the link if it is still ours. A device removed and + // re-added in quick succession may already have been relinked to a + // new PTY by the time this runs. + if target, err := os.Readlink(d.linkPath); err == nil && target == d.slavePath { + _ = os.Remove(d.linkPath) + } + }() + + reopen := serialReopenMinInterval + for { + select { + case <-d.done: + return + default: + } + + events, err := pollSerialMaster(d.master, serialPollInterval) + if err != nil { + d.log.Printf("serial: %s: poll: %v", d.name, err) + return + } + + if events&unix.POLLHUP != 0 { + // Nothing in here holds the device open — but it may have left + // something behind, so this is a flush-then-let-go, not a + // let-go. POLLIN and POLLHUP are independent: n_tty_poll reports + // the first from input_available_p() and the second from + // TTY_OTHER_CLOSED, so a writer that opened, wrote and closed + // inside one poll interval arrives here with its bytes still + // readable. Honouring the hangup without draining first threw + // them away, which lost `echo cmd > /dev/ttyACM0` — and every + // script that does printf into a device — with no error anywhere. + // + // Attach if that is what it takes to have somewhere to put them. + if events&unix.POLLIN != 0 && d.conn == nil { + d.attach() + } + // Mode first, for the reason the steady-state path below gives, + // and because a 1200-baud touch is a mode change immediately + // followed by a close where the mode is the entire message: + // losing it would mean a native-USB board never enters its + // bootloader. + d.syncMode() + if events&unix.POLLIN != 0 { + d.drainToHost() + } + d.detach() + + select { + case <-d.done: + return + case <-time.After(reopen): + } + if reopen *= 2; reopen > serialReopenMaxInterval { + reopen = serialReopenMaxInterval + } + continue + } + reopen = serialReopenMinInterval + + if d.conn == nil { + if !d.attach() { + select { + case <-d.done: + return + case <-time.After(serialAttachRetryInterval): + } + continue + } + } + + // Mode before data, every iteration. Neither order is right in + // general — the kernel gives no way to place a termios change + // within the byte stream — but this one matches how tools behave: + // they configure the port and then talk on it, so a change seen in + // the same wakeup as some output almost always preceded it. + d.syncMode() + + if events&unix.POLLIN != 0 { + if !d.forwardOutput() { + d.detach() + } + } + } +} + +// drainToHost flushes whatever the PTY still holds through to the physical +// port. Called on the hangup path, where a writer has already gone. +// +// forwardOutput moves at most one buffer per call, so a writer that left more +// than that behind needs several — hence the loop, with a non-blocking poll +// between reads to stop as soon as the queue is empty. serialDrainMaxReads +// bounds it: the PTY's own input queue is finite, so the cap is a backstop +// against spinning rather than a real limit, and it is logged if it is ever +// reached. +// +// Note what the caller's attach means for a board wired for auto-reset: +// opening the host port asserts DTR, so flushing a one-shot write pulses +// RESET. That is the right trade — the user explicitly wrote to the device, +// and dropping the write silently is worse than a reset. Holding the port +// open (`exec 3<>/dev/ttyACM0`) is how you write repeatedly without one. +func (d *serialDevice) drainToHost() { + if d.conn == nil { + return // attach failed; nothing to flush through + } + for i := 0; i < serialDrainMaxReads; i++ { + if !d.forwardOutput() { + return + } + // Timeout 0: ask what is pending right now and never park. The + // hangup is already latched, so a blocking poll here would return + // immediately anyway — but only because of that, and relying on it + // would be a trap for whoever next changes this loop. + events, err := pollSerialMaster(d.master, 0) + if err != nil || events&unix.POLLIN == 0 { + return + } + } + d.log.Printf("serial: %s: stopped draining after %d reads", d.name, serialDrainMaxReads) +} + +// attach opens a connection to the physical port on the host. Reports +// whether the device is ready to carry traffic. +func (d *serialDevice) attach() bool { + mode, err := readSerialMode(d.master) + if err != nil { + d.log.Printf("serial: %s: reading pty mode: %v", d.name, err) + return false + } + + conn, err := serialDialHost() + if err != nil { + d.log.Printf("serial: %s: vsock dial host:%d: %v", d.name, serialVSockPort, err) + return false + } + if err := serialWriteLine(conn, serialGreeting{ + Op: "attach", V: serialProtoVersion, Name: d.name, Mode: &mode, + }); err != nil { + d.log.Printf("serial: %s: sending attach greeting: %v", d.name, err) + _ = conn.Close() + return false + } + + r := bufio.NewReaderSize(conn, serialMaxLine) + var reply serialAttachReply + if err := serialReadLine(r, &reply); err != nil { + d.log.Printf("serial: %s: reading attach reply: %v", d.name, err) + _ = conn.Close() + return false + } + if !reply.OK { + d.log.Printf("serial: %s: host refused: %s", d.name, reply.Error) + _ = conn.Close() + return false + } + + d.conn, d.mode = conn, mode + go d.forwardInput(conn, r) + d.log.Printf("serial: %s attached (%d baud)", d.name, mode.Baud) + return true +} + +// detach tears down the current attachment, if any. The host closes the +// physical port when this connection goes, which is the other half of the +// open/close pairing that gives a board its reset pulse. +func (d *serialDevice) detach() { + if d.conn == nil { + return + } + _ = d.conn.Close() + d.conn = nil + d.log.Printf("serial: %s detached", d.name) +} + +// forwardInput pumps frames from the host into the PTY. It owns conn's read +// side for the lifetime of the attachment and exits when detach closes it. +func (d *serialDevice) forwardInput(conn net.Conn, r *bufio.Reader) { + for { + typ, payload, err := serialReadFrame(r) + if err != nil { + if !errors.Is(err, io.EOF) && !isClosedConn(err) { + d.log.Printf("serial: %s: reading frame: %v", d.name, err) + } + return + } + switch typ { + case serialFrameData: + if _, err := d.master.Write(payload); err != nil { + // EIO here means the client let go between the host + // sending and us writing. Not worth a log line: the poll + // loop is about to notice the hangup anyway. + return + } + default: + // The host only ever sends data today. Ignoring the rest keeps + // a future frame type from being a breaking change. + } + } +} + +// forwardOutput moves one batch of PTY output to the host. Reports whether +// the attachment is still good. +func (d *serialDevice) forwardOutput() bool { + buf := make([]byte, 4096) + n, err := d.master.Read(buf) + if n > 0 { + if werr := serialWriteFrame(d.conn, serialFrameData, buf[:n]); werr != nil { + d.log.Printf("serial: %s: writing to host: %v", d.name, werr) + return false + } + } + if err != nil { + // EIO is the client closing the slave, which the next poll reports + // as POLLHUP; everything else is the PTY going away under us. + return false + } + return true +} + +// syncMode forwards the PTY's line configuration to the host when it has +// changed since the last time we looked. +func (d *serialDevice) syncMode() { + if d.conn == nil { + return + } + mode, err := readSerialMode(d.master) + if err != nil || mode == d.mode { + return + } + if err := serialWriteFrame(d.conn, serialFrameMode, serialMarshalMode(mode)); err != nil { + d.log.Printf("serial: %s: sending mode: %v", d.name, err) + return + } + d.mode = mode + d.log.Printf("serial: %s reconfigured (%d baud)", d.name, mode.Baud) +} + +// pollSerialMaster waits for the PTY master to become readable or hang up. +func pollSerialMaster(f *os.File, timeout time.Duration) (int16, error) { + fds := []unix.PollFd{{Fd: int32(f.Fd()), Events: unix.POLLIN}} + for { + n, err := unix.Poll(fds, int(timeout.Milliseconds())) + if errors.Is(err, unix.EINTR) { + continue + } + if err != nil { + return 0, err + } + if n == 0 { + return 0, nil + } + return fds[0].Revents, nil + } +} + +// readSerialMode reads the line configuration the PTY's client has set. +// +// A PTY master and its slave share one termios, so this reads back exactly +// what the process on the other end asked for — which is the only reason +// forwarding a serial port through a PTY is possible at all. The one thing +// it cannot see is the modem-control lines: a PTY has none, and DTR and RTS +// are simply not expressible here. +func readSerialMode(f *os.File) (serialMode, error) { + t, err := unix.IoctlGetTermios(int(f.Fd()), unix.TCGETS2) + if err != nil { + return serialMode{}, err + } + + mode := serialMode{Baud: int(t.Ospeed), Bits: 8, Parity: "n", Stop: 1} + switch t.Cflag & unix.CSIZE { + case unix.CS5: + mode.Bits = 5 + case unix.CS6: + mode.Bits = 6 + case unix.CS7: + mode.Bits = 7 + } + if t.Cflag&unix.PARENB != 0 { + mode.Parity = "e" + if t.Cflag&unix.PARODD != 0 { + mode.Parity = "o" + } + } + if t.Cflag&unix.CSTOPB != 0 { + mode.Stop = 2 + } + // A rate of zero means the client hung the line up rather than picked a + // speed. Reporting it would fail validation on the host, so keep the + // last real value by reporting the tty default. + if mode.Baud <= 0 { + mode.Baud = 9600 + } + return mode, nil +} + +// rawSerialPTY takes the PTY out of cooked mode. See the call site in +// newSerialDevice for why that is the right default here. +func rawSerialPTY(fd int) error { + t, err := unix.IoctlGetTermios(fd, unix.TCGETS2) + if err != nil { + return err + } + t.Iflag &^= unix.IGNBRK | unix.BRKINT | unix.PARMRK | unix.ISTRIP | + unix.INLCR | unix.IGNCR | unix.ICRNL | unix.IXON + t.Oflag &^= unix.OPOST + t.Lflag &^= unix.ECHO | unix.ECHONL | unix.ICANON | unix.ISIG | unix.IEXTEN + t.Cc[unix.VMIN] = 1 + t.Cc[unix.VTIME] = 0 + return unix.IoctlSetTermios(fd, unix.TCSETS2, t) +} + +// validSerialName mirrors serialfwd.ValidDeviceName. The host validates +// first, but this string becomes a path in /dev on this side, so it is +// checked again here rather than trusted across the wire. +func validSerialName(name string) error { + switch { + case name == "": + return errors.New("device name is empty") + case len(name) > 64: + return fmt.Errorf("device name %q is too long", name) + case strings.ContainsAny(name, "/\\"): + return fmt.Errorf("device name %q must not contain a path separator", name) + case strings.HasPrefix(name, "."): + return fmt.Errorf("device name %q must not start with a dot", name) + } + for _, r := range name { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + case r == '.' || r == '-' || r == '_': + default: + return fmt.Errorf("device name %q contains %q", name, r) + } + } + return nil +} + +// serialMarshalMode encodes a mode for a FrameMode payload. Errors are +// impossible for this struct, so the signature doesn't carry one. +func serialMarshalMode(m serialMode) []byte { + b, err := json.Marshal(m) + if err != nil { + return []byte(`{}`) + } + return b +} + +// serialWriteLine mirrors serialfwd.WriteLine. +func serialWriteLine(w io.Writer, v any) error { + b, err := json.Marshal(v) + if err != nil { + return err + } + if len(b)+1 > serialMaxLine { + return errors.New("control line too long") + } + _, err = w.Write(append(b, '\n')) + return err +} + +// serialReadLine mirrors serialfwd.ReadLine. The reader must be reused for +// the frames that follow, or bytes buffered past the newline are lost. +func serialReadLine(r *bufio.Reader, v any) error { + line, err := r.ReadSlice('\n') + if errors.Is(err, bufio.ErrBufferFull) { + return errors.New("control line too long") + } + if err != nil { + return err + } + return json.Unmarshal(line, v) +} + +// serialWriteFrame mirrors serialfwd.WriteFrame. +func serialWriteFrame(w io.Writer, typ byte, payload []byte) error { + if len(payload) > serialMaxFrame { + return errors.New("frame too long") + } + var hdr [serialFrameHeaderBytes]byte + hdr[0] = typ + binary.BigEndian.PutUint32(hdr[1:], uint32(len(payload))) + if _, err := w.Write(hdr[:]); err != nil { + return err + } + if len(payload) == 0 { + return nil + } + _, err := w.Write(payload) + return err +} + +// serialReadFrame mirrors serialfwd.ReadFrame. +func serialReadFrame(r *bufio.Reader) (byte, []byte, error) { + var hdr [serialFrameHeaderBytes]byte + if _, err := io.ReadFull(r, hdr[:]); err != nil { + return 0, nil, err + } + n := binary.BigEndian.Uint32(hdr[1:]) + if n > serialMaxFrame { + return 0, nil, errors.New("frame too long") + } + if n == 0 { + return hdr[0], nil, nil + } + buf := make([]byte, n) + if _, err := io.ReadFull(r, buf); err != nil { + return 0, nil, err + } + return hdr[0], buf, nil +} diff --git a/internal/agentembed/revfwd_lockstep_test.go b/internal/agentembed/revfwd_lockstep_test.go index 27c95a0..296612b 100644 --- a/internal/agentembed/revfwd_lockstep_test.go +++ b/internal/agentembed/revfwd_lockstep_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/clawkwork/clawk/internal/revfwd" + "github.com/clawkwork/clawk/internal/serialfwd" "github.com/stretchr/testify/require" ) @@ -60,6 +61,7 @@ func TestGuestVSockPortsAreDistinct(t *testing.T) { {"sshAgentVSockPort", "1026"}, {"memReportVSockPort", "1027"}, {"reverseForwardVSockPort", fmt.Sprint(revfwd.VSockPort)}, + {"serialVSockPort", fmt.Sprint(serialfwd.VSockPort)}, } { require.Contains(t, src, decl.name+" = "+decl.port, "%s moved; update this test and every mirror of it", decl.name) diff --git a/internal/agentembed/serial_agent_test.go.in b/internal/agentembed/serial_agent_test.go.in new file mode 100644 index 0000000..fc088fc --- /dev/null +++ b/internal/agentembed/serial_agent_test.go.in @@ -0,0 +1,512 @@ +// Tests for the guest agent's serial forwarder, compiled and run inside a +// throwaway module built from main.go.in — see TestSerialForwarderRuns in +// the parent repo for how they get here. +// +// The agent can't be imported (it is package main in its own module), and +// its serial half is a state machine with genuinely tricky edges: the +// primed-hangup trick that makes "is a client attached?" answerable, the +// attachment lifetime that drives the host's open and close of the real +// port, and reading back a line configuration the kernel never announces. +// Compiling it proves none of that. Running it does. + +//go:build linux + +package main + +import ( + "bufio" + "encoding/json" + "io" + "log" + "net" + "os" + "path/filepath" + "testing" + "time" + + "golang.org/x/sys/unix" +) + +// fakeHost is the other end of the vsock the agent normally dials: it +// accepts attachments and records what arrives. +type fakeHost struct { + t *testing.T + ln net.Listener + + greetings chan serialGreeting + frames chan hostFrame + conns chan net.Conn + refuse bool +} + +type hostFrame struct { + typ byte + payload []byte +} + +func newFakeHost(t *testing.T) *fakeHost { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + h := &fakeHost{ + t: t, + ln: ln, + greetings: make(chan serialGreeting, 16), + frames: make(chan hostFrame, 256), + conns: make(chan net.Conn, 16), + } + t.Cleanup(func() { _ = ln.Close() }) + + // Point the agent's dialer at us. + prev := serialDialHost + serialDialHost = func() (net.Conn, error) { return net.Dial("tcp", ln.Addr().String()) } + t.Cleanup(func() { serialDialHost = prev }) + + go h.accept() + return h +} + +func (h *fakeHost) accept() { + for { + conn, err := h.ln.Accept() + if err != nil { + return + } + go h.serve(conn) + } +} + +func (h *fakeHost) serve(conn net.Conn) { + r := bufio.NewReaderSize(conn, serialMaxLine) + var g serialGreeting + if err := serialReadLine(r, &g); err != nil { + return + } + h.greetings <- g + if g.Op != "attach" { + return + } + if h.refuse { + _ = serialWriteLine(conn, serialAttachReply{OK: false, Error: "board unplugged"}) + _ = conn.Close() + return + } + if err := serialWriteLine(conn, serialAttachReply{OK: true}); err != nil { + return + } + h.conns <- conn + for { + typ, payload, err := serialReadFrame(r) + if err != nil { + return + } + h.frames <- hostFrame{typ: typ, payload: payload} + } +} + +func (h *fakeHost) waitGreeting(timeout time.Duration) (serialGreeting, bool) { + select { + case g := <-h.greetings: + return g, true + case <-time.After(timeout): + return serialGreeting{}, false + } +} + +func (h *fakeHost) waitFrame(typ byte, timeout time.Duration) (hostFrame, bool) { + deadline := time.After(timeout) + for { + select { + case f := <-h.frames: + if f.typ == typ { + return f, true + } + case <-deadline: + return hostFrame{}, false + } + } +} + +// startDevice brings up one forwarded device rooted in a temp dir, so the +// symlink lands somewhere writable without root. +func startDevice(t *testing.T, name string) *serialDevice { + t.Helper() + dir := t.TempDir() + prev := serialDevDir + serialDevDir = dir + t.Cleanup(func() { serialDevDir = prev }) + + dev, err := newSerialDevice(log.New(io.Discard, "", 0), name) + if err != nil { + t.Fatalf("newSerialDevice: %v", err) + } + t.Cleanup(dev.stop) + go dev.run() + return dev +} + +// openClient opens the device the way a tool in the sandbox would: through +// the published name, not the PTY path. +func openClient(t *testing.T, dev *serialDevice) *os.File { + t.Helper() + f, err := os.OpenFile(filepath.Join(serialDevDir, dev.name), os.O_RDWR|unix.O_NOCTTY, 0) + if err != nil { + t.Fatalf("opening client end: %v", err) + } + return f +} + +func setClientBaud(t *testing.T, f *os.File, cbaud uint32) { + t.Helper() + tio, err := unix.IoctlGetTermios(int(f.Fd()), unix.TCGETS) + if err != nil { + t.Fatalf("TCGETS: %v", err) + } + tio.Cflag = (tio.Cflag &^ unix.CBAUD) | cbaud + if err := unix.IoctlSetTermios(int(f.Fd()), unix.TCSETS, tio); err != nil { + t.Fatalf("TCSETS: %v", err) + } +} + +// The device node has to exist as soon as the host publishes it, whether or +// not anything ever opens it. +func TestSerialDevicePublishesNode(t *testing.T) { + newFakeHost(t) + dev := startDevice(t, "ttyTEST0") + + link := filepath.Join(serialDevDir, "ttyTEST0") + target, err := os.Readlink(link) + if err != nil { + t.Fatalf("device node not published: %v", err) + } + if target != dev.slavePath { + t.Fatalf("node points at %q, want %q", target, dev.slavePath) + } +} + +// Nothing has the device open, so nothing should be holding the physical +// port on the host. This is the primed-hangup trick working: without it the +// loop cannot tell "never opened" from "open and idle" and attaches to a +// board nobody asked for. +func TestSerialNoAttachUntilClientOpens(t *testing.T) { + host := newFakeHost(t) + startDevice(t, "ttyTEST0") + + if g, ok := host.waitGreeting(500 * time.Millisecond); ok { + t.Fatalf("attached with no client holding the device: %+v", g) + } +} + +// Opening the device is what makes the host open the real port — the event +// an Arduino sees as its reset pulse. +func TestSerialAttachOnClientOpen(t *testing.T) { + host := newFakeHost(t) + dev := startDevice(t, "ttyTEST0") + + client := openClient(t, dev) + defer client.Close() + + g, ok := host.waitGreeting(5 * time.Second) + if !ok { + t.Fatal("no attach after the client opened the device") + } + if g.Op != "attach" || g.Name != "ttyTEST0" { + t.Fatalf("unexpected greeting: %+v", g) + } + if g.Mode == nil { + t.Fatal("attach greeting carried no mode; the host would open at a guessed baud") + } +} + +// Closing the device must drop the attachment, which is what makes the host +// release the port — both so the reset pulse has a falling edge and so the +// Mac's own tooling can have the board back. +func TestSerialDetachOnClientClose(t *testing.T) { + host := newFakeHost(t) + dev := startDevice(t, "ttyTEST0") + + client := openClient(t, dev) + if _, ok := host.waitGreeting(5 * time.Second); !ok { + t.Fatal("no attach") + } + conn := <-host.conns + _ = client.Close() + + // The host sees the attachment end as its connection closing. + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + buf := make([]byte, 1) + if _, err := conn.Read(buf); err == nil { + t.Fatal("attachment outlived the client that opened the device") + } +} + +// Close then reopen is every upload to a native-USB board: the client drops +// the port, the board reboots into its bootloader, the client comes back. +// The host has to see a matching close and reopen of the physical port, or +// the board never gets its reset — so this asserts the detach lands +// *between* the two attaches rather than just that a second one happens. +func TestSerialReattachAfterClientReopens(t *testing.T) { + host := newFakeHost(t) + dev := startDevice(t, "ttyTEST0") + + client := openClient(t, dev) + if _, ok := host.waitGreeting(5 * time.Second); !ok { + t.Fatal("no first attach") + } + conn := <-host.conns + _ = client.Close() + + // Wait for the host to observe the detach before reopening. Any real + // close/reopen has a gap — the board is re-enumerating on the USB bus — + // and closing and reopening with no gap at all would be asking the + // agent to notice a state the kernel never held long enough to report. + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + if _, err := conn.Read(make([]byte, 1)); err == nil { + t.Fatal("attachment outlived the client") + } + + client2 := openClient(t, dev) + defer client2.Close() + if _, ok := host.waitGreeting(5 * time.Second); !ok { + t.Fatal("device never reattached after the client reopened it") + } +} + +func TestSerialForwardsBytesToHost(t *testing.T) { + host := newFakeHost(t) + dev := startDevice(t, "ttyTEST0") + + client := openClient(t, dev) + defer client.Close() + if _, ok := host.waitGreeting(5 * time.Second); !ok { + t.Fatal("no attach") + } + + if _, err := client.Write([]byte("hello board")); err != nil { + t.Fatalf("client write: %v", err) + } + f, ok := host.waitFrame(serialFrameData, 5*time.Second) + if !ok { + t.Fatal("client output never reached the host") + } + if string(f.payload) != "hello board" { + t.Fatalf("host got %q", f.payload) + } +} + +func TestSerialForwardsBytesToClient(t *testing.T) { + host := newFakeHost(t) + dev := startDevice(t, "ttyTEST0") + + client := openClient(t, dev) + defer client.Close() + if _, ok := host.waitGreeting(5 * time.Second); !ok { + t.Fatal("no attach") + } + conn := <-host.conns + + if err := serialWriteFrame(conn, serialFrameData, []byte("hello guest")); err != nil { + t.Fatalf("host write: %v", err) + } + + _ = client.SetReadDeadline(time.Now().Add(5 * time.Second)) + buf := make([]byte, 64) + n, err := client.Read(buf) + if err != nil { + t.Fatalf("client read: %v", err) + } + if string(buf[:n]) != "hello guest" { + t.Fatalf("client got %q", buf[:n]) + } +} + +// A baud change produces no event of any kind on a PTY — the loop has to +// notice it by looking. This is the mechanism the 1200-baud touch rides on, +// so it gets its own test. +func TestSerialReportsBaudChange(t *testing.T) { + host := newFakeHost(t) + dev := startDevice(t, "ttyTEST0") + + client := openClient(t, dev) + defer client.Close() + if _, ok := host.waitGreeting(5 * time.Second); !ok { + t.Fatal("no attach") + } + + setClientBaud(t, client, unix.B115200) + + f, ok := host.waitFrame(serialFrameMode, 5*time.Second) + if !ok { + t.Fatal("baud change never reached the host") + } + var mode serialMode + if err := json.Unmarshal(f.payload, &mode); err != nil { + t.Fatalf("bad mode frame: %v", err) + } + if mode.Baud != 115200 { + t.Fatalf("host told %d baud, want 115200", mode.Baud) + } +} + +// The 1200-baud touch in full: set 1200, then close immediately. The mode +// has to reach the host before the detach, or the board never enters its +// bootloader and every upload to a Leonardo or ESP32-S3 fails. +func TestSerial1200BaudTouchReachesHostBeforeDetach(t *testing.T) { + host := newFakeHost(t) + dev := startDevice(t, "ttyTEST0") + + client := openClient(t, dev) + if _, ok := host.waitGreeting(5 * time.Second); !ok { + t.Fatal("no attach") + } + + setClientBaud(t, client, unix.B1200) + _ = client.Close() + + f, ok := host.waitFrame(serialFrameMode, 5*time.Second) + if !ok { + t.Fatal("the 1200-baud touch was lost when the client closed") + } + var mode serialMode + if err := json.Unmarshal(f.payload, &mode); err != nil { + t.Fatalf("bad mode frame: %v", err) + } + if mode.Baud != 1200 { + t.Fatalf("host told %d baud, want 1200", mode.Baud) + } +} + +// An unchanged configuration must not produce a frame every time the loop +// looks — that would be 10 pointless wakeups a second on the host for as +// long as a monitor is open. +func TestSerialDoesNotRepeatUnchangedMode(t *testing.T) { + host := newFakeHost(t) + dev := startDevice(t, "ttyTEST0") + + client := openClient(t, dev) + defer client.Close() + if _, ok := host.waitGreeting(5 * time.Second); !ok { + t.Fatal("no attach") + } + + if _, ok := host.waitFrame(serialFrameMode, 700*time.Millisecond); ok { + t.Fatal("resent an unchanged mode") + } +} + +// A refused attach — an unplugged board, or one the Mac's own tooling holds +// — must not spin. The client keeps the device open, so the agent keeps +// trying, but at a pace that doesn't burn a core. +func TestSerialRefusedAttachRetriesWithoutSpinning(t *testing.T) { + host := newFakeHost(t) + host.refuse = true + dev := startDevice(t, "ttyTEST0") + + client := openClient(t, dev) + defer client.Close() + + if _, ok := host.waitGreeting(5 * time.Second); !ok { + t.Fatal("never tried to attach") + } + // Second attempt should come after the retry interval, not immediately. + start := time.Now() + if _, ok := host.waitGreeting(5 * time.Second); !ok { + t.Fatal("gave up after one refusal while the client still held the device") + } + if elapsed := time.Since(start); elapsed < serialAttachRetryInterval/2 { + t.Fatalf("retried after %v; that is a spin, not a retry", elapsed) + } +} + +// Removing a device from the published set has to take the node with it. A +// leftover /dev entry would accept an open and then sit mute, which reads +// as broken hardware rather than an absent forward. +func TestSerialStopRemovesNode(t *testing.T) { + newFakeHost(t) + dev := startDevice(t, "ttyTEST0") + link := filepath.Join(serialDevDir, "ttyTEST0") + + if _, err := os.Readlink(link); err != nil { + t.Fatalf("node was never published: %v", err) + } + dev.stop() + + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if _, err := os.Readlink(link); err != nil { + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatal("device node outlived the device") +} + +// The name arrives over the wire and becomes a path, so the guest validates +// it again rather than trusting the host to have done so. +func TestSerialApplyRejectsUnsafeNames(t *testing.T) { + newFakeHost(t) + dir := t.TempDir() + prev := serialDevDir + serialDevDir = dir + t.Cleanup(func() { serialDevDir = prev }) + + f := &serialForwarder{log: log.New(io.Discard, "", 0), devices: map[string]*serialDevice{}} + f.apply([]serialDeviceMsg{ + {Name: "../escape"}, + {Name: "sub/dir"}, + {Name: ""}, + {Name: ".hidden"}, + }) + t.Cleanup(func() { f.apply(nil) }) + + if len(f.devices) != 0 { + t.Fatalf("created %d device(s) from unsafe names", len(f.devices)) + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("unsafe names left %d entries behind", len(entries)) + } +} + +// A writer can be gone before the loop ever sees it open. `echo cmd > +// /dev/ttyACM0`, and every script that does printf into a device, opens, +// writes and closes in microseconds — far inside serialPollInterval — so the +// loop's first sight of it is POLLIN and POLLHUP together. The two flags are +// independent (input_available_p vs TTY_OTHER_CLOSED), and honouring the +// hangup first discarded the bytes with no error on either side. +func TestSerialShortLivedWriterIsNotLost(t *testing.T) { + host := newFakeHost(t) + dev := startDevice(t, "ttyTEST0") + + // Let the loop settle into its detached backoff first, so the whole + // open-write-close below lands between two polls. Catching the client + // while it still holds the device would exercise the ordinary attached + // path and prove nothing about this one. + time.Sleep(150 * time.Millisecond) + + client := openClient(t, dev) + if _, err := client.Write([]byte("reset\r\n")); err != nil { + t.Fatalf("client write: %v", err) + } + _ = client.Close() + + f, ok := host.waitFrame(serialFrameData, 5*time.Second) + if !ok { + t.Fatal("a write followed immediately by close never reached the host") + } + if string(f.payload) != "reset\r\n" { + t.Fatalf("host got %q, want %q", f.payload, "reset\r\n") + } + + // And the flush must not leave the port held: the host closes the + // physical device when the attachment goes, so a drain that forgot to + // let go would keep the board away from the Mac indefinitely. + if _, ok := host.waitGreeting(time.Second); !ok { + t.Fatal("no attach greeting for the flush") + } +} diff --git a/internal/agentembed/serialfwd_lockstep_test.go b/internal/agentembed/serialfwd_lockstep_test.go new file mode 100644 index 0000000..3e09bf0 --- /dev/null +++ b/internal/agentembed/serialfwd_lockstep_test.go @@ -0,0 +1,83 @@ +package agentembed + +import ( + "fmt" + "reflect" + "testing" + + "github.com/clawkwork/clawk/internal/serialfwd" + "github.com/stretchr/testify/require" +) + +// The guest agent builds standalone inside the guest, so it can't import +// internal/serialfwd — the serial protocol is transcribed into main.go.in by +// hand. Nothing else notices when the two drift: a renamed JSON field or a +// moved frame type compiles fine on both sides and fails only as a sandbox +// where a board is visible but mute. +// +// So: check the parts that have to match byte-for-byte against the +// parent-module definition, deriving them from the real declarations rather +// than restating them, so this test can't drift either. Same rule and same +// shape as TestReverseForwardProtocolMirroredInAgent. +func TestSerialProtocolMirroredInAgent(t *testing.T) { + src := string(AgentMainGo) + + require.Contains(t, src, fmt.Sprintf("serialVSockPort = %d", serialfwd.VSockPort), + "guest dials a different vsock port than the host listens on") + require.Contains(t, src, fmt.Sprintf("serialProtoVersion = %d", serialfwd.ProtoVersion), + "guest announces a protocol version the host will hang up on") + require.Contains(t, src, fmt.Sprintf("serialMaxLine = %d", serialfwd.MaxLineBytes/1024)+" * 1024", + "guest and host disagree on the control-line cap") + + // Framing constants. A mismatch here desynchronises the stream rather + // than failing cleanly, which is the worst way for this to break. + require.Contains(t, src, fmt.Sprintf("serialFrameHeaderBytes = %d", serialfwd.FrameHeaderBytes), + "guest frames a different header length than the host parses") + require.Contains(t, src, fmt.Sprintf("serialFrameData byte = 0x%02x", byte(serialfwd.FrameData)), + "guest tags data frames differently from the host") + require.Contains(t, src, fmt.Sprintf("serialFrameMode byte = 0x%02x", byte(serialfwd.FrameMode)), + "guest tags mode frames differently from the host") + + // Op values are string literals on the wire, sent by the guest and + // switched on by the host. + for _, op := range []string{serialfwd.OpControl, serialfwd.OpAttach} { + require.Contains(t, src, fmt.Sprintf("Op: %q", op), + "guest never sends the %q greeting", op) + } + + // Parity travels as a one-letter string; a guest that spelled it + // differently would be rejected by the host's mode validation and the + // port would silently keep its old settings. + for _, parity := range []string{serialfwd.ParityNone, serialfwd.ParityEven, serialfwd.ParityOdd} { + require.Contains(t, src, fmt.Sprintf("%q", parity), + "guest never produces parity %q", parity) + } + + for _, typ := range []any{ + serialfwd.Greeting{}, serialfwd.Snapshot{}, serialfwd.Device{}, + serialfwd.AttachReply{}, serialfwd.Mode{}, + } { + rt := reflect.TypeOf(typ) + for i := range rt.NumField() { + tag, ok := rt.Field(i).Tag.Lookup("json") + if !ok { + continue + } + want := fmt.Sprintf("`json:%q`", tag) + require.Contains(t, src, want, + "guest is missing the %s.%s field tag %s", + rt.Name(), rt.Field(i).Name, want) + } + } +} + +// The guest turns a published device name into /dev/. The host +// validates it first, but the guest must not rely on that — this checks the +// second line of defence is actually wired up rather than merely present. +func TestSerialDeviceNameValidatedInAgent(t *testing.T) { + src := string(AgentMainGo) + require.Contains(t, src, "func validSerialName(", + "guest has no device-name validation") + require.Contains(t, src, "if err := validSerialName(d.Name); err != nil", + "guest validates device names but never calls the validator on a published set") +} diff --git a/internal/cli/agent_session.go b/internal/cli/agent_session.go index 044ca35..412b397 100644 --- a/internal/cli/agent_session.go +++ b/internal/cli/agent_session.go @@ -228,10 +228,10 @@ func tryVSockAgent(sb *config.Sandbox, provider sandbox.Provider, agent Agent, e cfg := vsockclient.Config{ SocketPath: sockPath, Cmd: cmd, - Args: append(append([]string{}, agent.DefaultArgs...), extra...), + Args: launchArgs(sb, agent, extra), Cwd: agentStartDir(provider, sb), User: sandbox.GuestUser, - Env: buildVSockEnv(), + Env: buildVSockEnv(sb), // Agents are full-screen TUIs: clear so they don't overdraw the // CLI's boot narration. ClearScreen: true, @@ -271,15 +271,77 @@ func tryVSockAgent(sb *config.Sandbox, provider sandbox.Provider, agent Agent, e // vsock path the env must be threaded through the handshake here. // Loading on every dispatch picks up edits to the token file without // requiring a sandbox rebuild. -func buildVSockEnv() []string { +// +// The sandbox's own `env ( … )` entries ride along for the same reason: +// without them the runner process cannot see them at all, only its +// login-shell children can. Anything that reads the environment of the +// runner itself therefore needs this — notably `${VAR}` references in the +// generated MCP config (see sandbox.MCPConfigFile), whose expansion +// happens in the runner's process env, so a PAT delivered only via +// profile.d would silently expand to empty. +// +// Ordering and precedence: clawk.mod entries go first and clawk's own vars +// last, because the guest agent folds the slice into a map in order (last +// write wins) — EXCEPT for a name the sandbox declared itself, which clawk +// then does not emit at all. +// +// That exception is the whole point. clawk's defaults exist so the common +// case works with no configuration, not to overrule a user who wrote the +// variable down. Concretely: a sandbox pointed at a gateway that needs no +// credential of its own (ANTHROPIC_BASE_URL, no ANTHROPIC_AUTH_TOKEN) has +// claude fall back to CLAUDE_CODE_OAUTH_TOKEN, so clawk's injected +// sk-ant-oat-… goes to that third-party endpoint as an Authorization +// header — and before this, a clawk.mod declaration was silently +// overwritten, so no config could stop it. Nobody writes +// `CLAUDE_CODE_OAUTH_TOKEN = …` in a clawk.mod by accident, so treating it +// as intent costs nothing. Same precedence rule as launchArgs, where the +// user's `-- ` land after clawk's defaults. +// +// A declared-but-empty value is emitted as empty rather than dropped, so +// it still shadows whatever the image or a parent process might have set. +// Note that empty is not the same as absent: the guest agent folds these +// into a map (agentembed/main.go.in, buildChildEnv), so `NAME=` reaches the +// child set-but-empty. Whether that reads as "unset" is the consumer's +// call — claude treats an empty credential as unset, but not every program +// does, and clawk has no way to express a true unset today. +// +// Resolution here is best-effort by design: attach is the hot path and +// must not become unusable because a variable left the host shell since +// create (the guest still has the value profile.d captured then). A +// failure warns and the remaining entries still go through — but the +// failing name stays suppressed, because the alternative is clawk quietly +// re-injecting the credential the declaration existed to displace. +func buildVSockEnv(sb *config.Sandbox) []string { env := []string{} + modEnv, err := sandbox.ResolveEnv(sb) + if err != nil { + fmt.Fprintf(os.Stderr, + "warning: some clawk.mod env vars could not be resolved for "+ + "sandbox %q; the agent process will not see them: %v\n", + sb.DisplayName(), err) + } + env = append(env, modEnv...) + + // Names the sandbox set for itself; clawk yields on every one of them. + // + // Taken from the declarations, NOT from modEnv: an entry that failed to + // resolve is still a name the user spoke for, and reading it back out of + // the resolved set would hand it to clawk again — see + // sandbox.DeclaredEnvNames. + declared := sandbox.DeclaredEnvNames(sb) + add := func(k, v string) { + if !declared[k] { + env = append(env, k+"="+v) + } + } + if tok, _ := sandbox.LoadOAuthToken(clawkRoot()); tok != "" { - env = append(env, "CLAUDE_CODE_OAUTH_TOKEN="+tok) + add("CLAUDE_CODE_OAUTH_TOKEN", tok) } if v := os.Getenv("COLORTERM"); v != "" { - env = append(env, "COLORTERM="+v) + add("COLORTERM", v) } else { - env = append(env, "COLORTERM=truecolor") + add("COLORTERM", "truecolor") } // Forward terminal-identification env vars verbatim. Empty values // are dropped — we don't want to claim "I'm iTerm2" when the user @@ -293,14 +355,14 @@ func buildVSockEnv() []string { "ITERM_PROFILE", // ditto } { if v := os.Getenv(k); v != "" { - env = append(env, k+"="+v) + add(k, v) } } if v := os.Getenv("LANG"); v != "" { - env = append(env, "LANG="+v) + add("LANG", v) } if v := os.Getenv("LC_ALL"); v != "" { - env = append(env, "LC_ALL="+v) + add("LC_ALL", v) } return env } @@ -321,7 +383,7 @@ func attachAgentViaExec(sb *config.Sandbox, provider sandbox.Provider, agent Age if !ok { return fmt.Errorf("provider for %q cannot exec the agent", sb.Name) } - defaults := strings.Join(quoteAll(agent.DefaultArgs), " ") + defaults := strings.Join(quoteAll(launchArgs(sb, agent, nil)), " ") cmdline := `: "${TERM:=xterm-256color}"; : "${COLORTERM:=truecolor}"; ` + "export TERM COLORTERM; " + "cd " + agentStartDir(provider, sb) + " 2>/dev/null || true; " + diff --git a/internal/cli/agent_session_test.go b/internal/cli/agent_session_test.go index a2f4679..29d4753 100644 --- a/internal/cli/agent_session_test.go +++ b/internal/cli/agent_session_test.go @@ -8,6 +8,7 @@ import ( "strings" "testing" + "github.com/clawkwork/clawk/internal/config" "github.com/clawkwork/clawk/internal/sandbox" "github.com/stretchr/testify/require" ) @@ -38,7 +39,7 @@ func TestBuildVSockEnvForwardsOAuthToken(t *testing.T) { t.Setenv("CLAUDE_CODE_OAUTH_TOKEN", "") require.NoError(t, sandbox.SaveOAuthToken(filepath.Join(home, ".clawk"), "sk-test-vsock")) - env := buildVSockEnv() + env := buildVSockEnv(&config.Sandbox{Name: "sb"}) want := "CLAUDE_CODE_OAUTH_TOKEN=sk-test-vsock" for _, e := range env { if e == want { @@ -56,9 +57,157 @@ func TestBuildVSockEnvOmitsTokenWhenAbsent(t *testing.T) { _, err := os.Stat(filepath.Join(home, ".clawk", "claude-oauth-token")) require.True(t, os.IsNotExist(err), "expected no token file, got err=%v", err) - for _, e := range buildVSockEnv() { + for _, e := range buildVSockEnv(&config.Sandbox{Name: "sb"}) { if strings.HasPrefix(e, "CLAUDE_CODE_OAUTH_TOKEN=") { t.Errorf("unexpected token entry when unconfigured: %q", e) } } } + +// TestBuildVSockEnvForwardsModEnv is the guard for the same class of bug as +// the OAuth token above, for clawk.mod `env ( … )`: the runner process is +// spawned non-login, so a var delivered only through /etc/profile.d/ is +// invisible to it — visible to its shell children, but not to the runner +// itself. Anything reading the runner's own environment therefore breaks, +// notably `${VAR}` expansion inside the generated MCP config, where an +// unforwarded PAT silently becomes an empty Authorization header. +func TestBuildVSockEnvForwardsModEnv(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + t.Setenv("CLAUDE_CODE_OAUTH_TOKEN", "") + t.Setenv("CLAWK_TEST_PAT", "pat-abc123") + + sb := &config.Sandbox{Name: "sb", RequiredEnv: []string{ + "LINEAR_TOKEN=${CLAWK_TEST_PAT}", + "LITERAL=plain", + }} + env := buildVSockEnv(sb) + require.Contains(t, env, "LINEAR_TOKEN=pat-abc123", "aliased host var must reach the runner") + require.Contains(t, env, "LITERAL=plain", "literal must reach the runner") +} + +// TestBuildVSockEnvClawkVarsWinOrdering pins the layering for names the +// sandbox did NOT declare: the guest agent folds the handshake env into a +// map in slice order (agentembed/main.go.in, buildChildEnv), so the LAST +// occurrence wins, and clawk's own vars are emitted after the sandbox's +// declared env. +func TestBuildVSockEnvClawkVarsWinOrdering(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("CLAUDE_CODE_OAUTH_TOKEN", "") + t.Setenv("COLORTERM", "") + require.NoError(t, sandbox.SaveOAuthToken(filepath.Join(home, ".clawk"), "sk-real")) + + sb := &config.Sandbox{Name: "sb", RequiredEnv: []string{"UNRELATED=x"}} + env := buildVSockEnv(sb) + + idx := func(prefix string) int { + for i, e := range env { + if strings.HasPrefix(e, prefix) { + return i + } + } + return -1 + } + require.Contains(t, env, "CLAUDE_CODE_OAUTH_TOKEN=sk-real") + require.Greater(t, idx("CLAUDE_CODE_OAUTH_TOKEN="), idx("UNRELATED="), + "undeclared clawk vars still come after the sandbox's own entries") +} + +// TestBuildVSockEnvDeclarationOverridesClawkVar is the counterpart, and the +// reason the rule above carves out declared names: a sandbox pointed at a +// non-Anthropic provider must be able to stop clawk injecting an Anthropic +// credential, or the runner authenticates against the wrong service and +// every request fails with an unrecognized-model error. Before this, the +// declaration was silently overwritten and `clawk auth clear` — which +// disarms EVERY sandbox on the host — was the only lever. +func TestBuildVSockEnvDeclarationOverridesClawkVar(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("CLAUDE_CODE_OAUTH_TOKEN", "") + require.NoError(t, sandbox.SaveOAuthToken(filepath.Join(home, ".clawk"), "sk-real")) + + sb := &config.Sandbox{Name: "sb", RequiredEnv: []string{ + `CLAUDE_CODE_OAUTH_TOKEN=""`, + }} + env := buildVSockEnv(sb) + + var got []string + for _, e := range env { + if strings.HasPrefix(e, "CLAUDE_CODE_OAUTH_TOKEN=") { + got = append(got, e) + } + } + require.Equal(t, []string{"CLAUDE_CODE_OAUTH_TOKEN="}, got, + "the declared value must be the only occurrence — clawk must not re-add its own") +} + +// TestBuildVSockEnvDeclarationOverridesTerminalVar pins that the carve-out +// is a general precedence rule, not a special case for the OAuth token. +func TestBuildVSockEnvDeclarationOverridesTerminalVar(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + t.Setenv("CLAUDE_CODE_OAUTH_TOKEN", "") + t.Setenv("COLORTERM", "truecolor") + + sb := &config.Sandbox{Name: "sb", RequiredEnv: []string{"COLORTERM=256"}} + env := buildVSockEnv(sb) + + var got []string + for _, e := range env { + if strings.HasPrefix(e, "COLORTERM=") { + got = append(got, e) + } + } + require.Equal(t, []string{"COLORTERM=256"}, got) +} + +// TestBuildVSockEnvSurvivesUnresolvableEntry covers the best-effort +// contract: attach is the hot path, so a `${HOST:?msg}` that no longer +// resolves in this shell must not make the sandbox unreachable. The +// failing entry drops out; everything else still goes through. +func TestBuildVSockEnvSurvivesUnresolvableEntry(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + t.Setenv("CLAUDE_CODE_OAUTH_TOKEN", "") + t.Setenv("CLAWK_TEST_PRESENT", "here") + + sb := &config.Sandbox{Name: "sb", RequiredEnv: []string{ + "GONE=${CLAWK_TEST_ABSENT:?set it on the host}", + "KEPT=${CLAWK_TEST_PRESENT}", + }} + env := buildVSockEnv(sb) + require.Contains(t, env, "KEPT=here", "resolvable entries must survive a failing sibling") + for _, e := range env { + require.False(t, strings.HasPrefix(e, "GONE="), "unresolvable entry must be dropped: %q", e) + } +} + +// TestBuildVSockEnvDeclarationOverridesEvenWhenUnresolvable is the security +// counterpart to TestBuildVSockEnvSurvivesUnresolvableEntry: best-effort +// resolution must not become best-effort PRECEDENCE. +// +// A sandbox pointed at a non-Anthropic gateway disowns clawk's token by +// declaring the name itself. If that declaration reads a host variable that +// has since left the shell, the entry drops out of ResolveEnv's output — and +// deriving the "declared" set from that output handed the name straight back +// to clawk, which injected its own sk-ant-oat-… as an Authorization header to +// the third-party endpoint. Exactly the leak the declaration existed to stop, +// reachable from any shell missing the gateway variable. +func TestBuildVSockEnvDeclarationOverridesEvenWhenUnresolvable(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("CLAUDE_CODE_OAUTH_TOKEN", "") + // "sk-real" like the fixtures above, deliberately NOT the live sk-ant- + // prefix: this repo is public, and a literal shaped like a real Anthropic + // token trips secret scanners and push protection for no benefit. The + // assertion only needs a string distinctive enough to spot in the output. + require.NoError(t, sandbox.SaveOAuthToken(filepath.Join(home, ".clawk"), "sk-real")) + + sb := &config.Sandbox{Name: "sb", RequiredEnv: []string{ + "CLAUDE_CODE_OAUTH_TOKEN=${CLAWK_TEST_GATEWAY:?set it on the host}", + }} + env := buildVSockEnv(sb) + + for _, e := range env { + require.NotContains(t, e, "sk-real", + "a declared-but-unresolvable name must stay suppressed, not fall back to clawk's token") + } +} diff --git a/internal/cli/agents.go b/internal/cli/agents.go index d34e14b..3151181 100644 --- a/internal/cli/agents.go +++ b/internal/cli/agents.go @@ -3,13 +3,16 @@ package cli import ( "fmt" "sort" + + "github.com/clawkwork/clawk/internal/config" + "github.com/clawkwork/clawk/internal/sandbox" ) // Agent describes how a coding-agent runner is launched inside a -// sandbox. Treating each agent (claude, codex, opencode, ...) as a +// sandbox. Treating each agent (claude, codex, pi, opencode, ...) as a // data-driven entry rather than a per-agent code path keeps the CLI // surface symmetric: every agent's verb works the same way, and adding -// a fourth runner is a one-line registry change. +// another runner is a one-line registry change. type Agent struct { // Name is the user-facing runner argument (`clawk run `). // Must be unique. Lowercase ASCII, no whitespace. Also the command @@ -18,9 +21,23 @@ type Agent struct { Name string // DefaultArgs are prepended to whatever the user passes through - // `-- `. Claude and Codex both have explicit "externally - // sandboxed" modes, so clawk enables those by default. + // `-- `. Every runner that can be told "you are already + // externally sandboxed" is told so here: claude and codex have + // explicit modes for it, pi has project trust. DefaultArgs []string + + // MCPConfigFlag is the runner's flag for loading an external MCP + // server config file, or "" if clawk doesn't know how to hand this + // runner one. When set and the sandbox declares servers, the flag and + // sandbox.GuestMCPConfigPath are appended to the launch args. + // + // A per-runner flag rather than a line in DefaultArgs because the + // value is conditional (a sandbox with no `mcp ( … )` block must not + // get a flag pointing at a file that isn't there) and because each + // runner spells this differently — codex and opencode use their own + // config formats and pi loads MCP through an extension, so they stay + // empty until someone renders those too. + MCPConfigFlag string } // agents is the builtin runner registry. `clawk run ` consults @@ -29,16 +46,65 @@ var agents = []Agent{ { Name: "claude", DefaultArgs: []string{"--dangerously-skip-permissions"}, + // Not --strict-mcp-config: that would suppress every other MCP + // source, including the claude.ai account connectors that already + // work inside a sandbox (they're proxied server-side, so they need + // no local credential and no egress of their own) and any plugin + // the user's settings enable. + MCPConfigFlag: "--mcp-config", }, { Name: "codex", DefaultArgs: []string{"--dangerously-bypass-approvals-and-sandbox"}, }, + { + Name: "pi", + // pi has no approval prompts to bypass — it ships no built-in + // sandbox at all, by design (its docs/security.md: "Real isolation + // needs to come from the operating system or a virtualization + // boundary", which is exactly what a clawk VM is). What it does + // gate is project trust: a repo carrying .pi/settings.json, + // .pi/extensions or .agents/skills triggers an interactive + // "trust this project?" prompt before those load, and answering it + // is meaningless inside a VM whose whole premise is that the + // project already has the machine. --approve trusts the project for + // the run, the same posture as claude's and codex's flags above. + DefaultArgs: []string{"--approve"}, + // No MCPConfigFlag: pi speaks MCP through an extension + // (pi-mcp-adapter), not a config-file flag, so a sandbox with an + // `mcp ( … )` block gets nothing here until that's rendered too. + }, { Name: "opencode", + // opencode gates every tool call behind a permission prompt unless + // told otherwise; --auto approves anything not explicitly denied + // (its own help says "dangerous!", which is true on a laptop and + // beside the point inside a disposable VM). A deny rule in the + // user's opencode.jsonc still wins, so this widens the default + // without overriding a considered choice. + DefaultArgs: []string{"--auto"}, + // No MCPConfigFlag: opencode manages MCP servers through its own + // config file and `opencode mcp` subcommand, not a flag taking a + // path, so a sandbox's `mcp ( … )` block doesn't reach it yet. }, } +// launchArgs assembles the runner's argv tail: its permission-mode +// defaults, the MCP config flag when this sandbox declares servers and the +// runner knows how to take one, then the user's `-- ` last so an +// explicit flag always sits after (and therefore overrides) clawk's. +// +// Both attach paths — the vsock handshake and the `bash -lc` exec fallback +// — go through here, so a runner never gets a different command line +// depending on which transport happened to be available. +func launchArgs(sb *config.Sandbox, agent Agent, extra []string) []string { + args := append([]string{}, agent.DefaultArgs...) + if agent.MCPConfigFlag != "" && len(sb.MCP) > 0 { + args = append(args, agent.MCPConfigFlag, sandbox.GuestMCPConfigPath) + } + return append(args, extra...) +} + // agentByName returns the registry entry for a given runner name. // Used by `clawk run ` dispatch. func agentByName(name string) (Agent, error) { diff --git a/internal/cli/agents_test.go b/internal/cli/agents_test.go new file mode 100644 index 0000000..15e026d --- /dev/null +++ b/internal/cli/agents_test.go @@ -0,0 +1,58 @@ +package cli + +import ( + "strings" + "testing" + + "github.com/clawkwork/clawk/internal/config" + "github.com/clawkwork/clawk/internal/sandbox" + "github.com/stretchr/testify/require" +) + +// TestPiRunnerRegistered pins the pi harness's registry entry. pi ships no +// built-in sandbox (its security docs say isolation must come from a VM, +// which is what clawk provides), so the only thing to pre-answer is project +// trust: without --approve a repo carrying .pi/settings.json or .pi/extensions +// stops the run on an interactive prompt that means nothing inside a sandbox. +func TestPiRunnerRegistered(t *testing.T) { + pi, err := agentByName("pi") + require.NoError(t, err) + require.Equal(t, []string{"--approve"}, pi.DefaultArgs) + require.Empty(t, pi.MCPConfigFlag, + "pi loads MCP through an extension, not a config-file flag") + + // The user's own args land after clawk's, so an explicit --no-approve + // still wins. + withMCP := &config.Sandbox{MCP: []config.MCPServer{{Name: "linear"}}} + require.Equal(t, []string{"--approve", "--no-approve"}, + launchArgs(withMCP, pi, []string{"--no-approve"}), + "a runner clawk can't render MCP config for gets no flag, and user args go last") +} + +// TestAgentRegistryNames guards the invariants the registry's doc comment +// states: names are unique, and each is usable both as a `clawk run` argument +// and as a bare command on the guest's PATH. +func TestAgentRegistryNames(t *testing.T) { + seen := map[string]bool{} + for _, a := range agents { + require.Falsef(t, seen[a.Name], "duplicate runner name %q", a.Name) + seen[a.Name] = true + require.Equalf(t, strings.ToLower(a.Name), a.Name, "runner %q must be lowercase", a.Name) + require.NotContainsf(t, a.Name, " ", "runner %q must not contain whitespace", a.Name) + } + require.Subset(t, seen, map[string]bool{"claude": true, "codex": true, "pi": true, "opencode": true}) + + // Runner names are reserved as sandbox names — a sandbox called "pi" + // would make `clawk run pi` ambiguous. + require.Contains(t, reservedAgentNames(), "pi") +} + +// TestAgentStateDirsNameRealRunners keeps the persistence list honest: every +// home directory clawk mounts must belong to a runner someone can actually +// launch. A typo there costs a PCIe device per sandbox and persists nothing. +func TestAgentStateDirsNameRealRunners(t *testing.T) { + for _, d := range sandbox.AgentStateDirs { + _, err := agentByName(d.Agent) + require.NoErrorf(t, err, "AgentStateDirs names %q, which is not a registered runner", d.Agent) + } +} diff --git a/internal/cli/apply.go b/internal/cli/apply.go index 5a3cdfd..427f7ed 100644 --- a/internal/cli/apply.go +++ b/internal/cli/apply.go @@ -193,6 +193,14 @@ func namespaceFromDef(def template.NamespaceDef, dryRun bool) (*config.Namespace if err != nil { return nil, err } + var mcpSources []mcpSource + for _, s := range tmpl.MCP { + mcpSources = append(mcpSources, mcpSource{Origin: "manifest", Spec: s}) + } + mcpServers, err := composeMCP(mcpSources) + if err != nil { + return nil, err + } denied := make([]string, 0, len(tmpl.DenyDomains)) for _, d := range tmpl.DenyDomains { @@ -217,5 +225,6 @@ func namespaceFromDef(def template.NamespaceDef, dryRun bool) (*config.Namespace Files: files, Shares: shares, Env: dedupStrings(tmpl.Env), + MCP: mcpServers, }, nil } diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index c3d5a63..166d002 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -60,6 +60,7 @@ func setupTest(t *testing.T) (*config.Store, *sandbox.MockProvider) { statusJSON = false statusBrief = false forwardListJSON = false + serialListJSON = false networkListJSON = false worktreeListJSON = false imageFlag = "" diff --git a/internal/cli/complete.go b/internal/cli/complete.go index 2444e28..275fbca 100644 --- a/internal/cli/complete.go +++ b/internal/cli/complete.go @@ -109,7 +109,7 @@ func runnerNames() []string { // completeRunArgs is the position-aware ValidArgsFunction for `clawk run`: // -// - position 0: runner name (claude / codex / opencode / shell) +// - position 0: runner name (claude / codex / pi / opencode / shell) // - position 1: sandbox name // - position 2+: nothing (runner passthrough args follow `--`) func completeRunArgs(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { diff --git a/internal/cli/compose_mcp.go b/internal/cli/compose_mcp.go new file mode 100644 index 0000000..4eda4c7 --- /dev/null +++ b/internal/cli/compose_mcp.go @@ -0,0 +1,173 @@ +package cli + +import ( + "fmt" + "net/netip" + "net/url" + "slices" + "strings" + + "github.com/clawkwork/clawk/internal/config" + "github.com/clawkwork/clawk/internal/template" +) + +// mergeMCP gathers `mcp (...)` entries from the workspace and every repo +// Clawkfile into the sandbox's server list. +// +// Same conflict rule as mergeFiles/mergeShares: two sources declaring the +// same server name identically collapse to one entry, but a genuine +// disagreement about what that name means is a config bug rather than +// something to silently tie-break — the agent would otherwise get whichever +// definition happened to be composed first. +func mergeMCP(ws *template.Workspace) ([]config.MCPServer, error) { + var sources []mcpSource + for _, s := range ws.File.MCP { + sources = append(sources, mcpSource{Origin: "workspace", Spec: s}) + } + for _, r := range ws.Repos { + if r.Clawkfile == nil { + continue + } + for _, s := range r.Clawkfile.MCP { + sources = append(sources, mcpSource{Origin: r.Name, Spec: s}) + } + } + return composeMCP(sources) +} + +type mcpSource struct { + Origin string + Spec template.MCPSpec +} + +// composeMCP is the pure conflict-detection core, split out so the here-mode +// path (a single Clawkfile) and the workspace path share it. +func composeMCP(sources []mcpSource) ([]config.MCPServer, error) { + byName := make(map[string]struct { + Origin string + Server config.MCPServer + }) + var out []config.MCPServer + for _, s := range sources { + srv := config.MCPServer{ + Name: s.Spec.Name, + Transport: s.Spec.Transport, + URL: s.Spec.URL, + Command: s.Spec.Command, + Headers: s.Spec.Headers, + Env: s.Spec.Env, + } + if prev, dup := byName[srv.Name]; dup { + if sameMCPServer(prev.Server, srv) { + continue + } + return nil, fmt.Errorf( + "mcp server %q declared differently by %s (%s) and %s (%s) — rename one", + srv.Name, prev.Origin, mcpTarget(prev.Server), s.Origin, mcpTarget(srv)) + } + byName[srv.Name] = struct { + Origin string + Server config.MCPServer + }{s.Origin, srv} + out = append(out, srv) + } + return out, nil +} + +// sameMCPServer reports whether two declarations of one server name mean the +// same thing, so repeated identical entries collapse instead of erroring. +func sameMCPServer(a, b config.MCPServer) bool { + return a.Transport == b.Transport && + a.URL == b.URL && + slices.Equal(a.Command, b.Command) && + slices.Equal(a.Headers, b.Headers) && + slices.Equal(a.Env, b.Env) +} + +// mcpTarget renders a server's endpoint for error messages: the URL for a +// remote server, the command for a stdio one. +func mcpTarget(s config.MCPServer) string { + if s.URL != "" { + return s.URL + } + return strings.Join(s.Command, " ") +} + +// appendMissingMCP folds add into have, keeping entries already present by +// name. Used to layer the namespace's servers under a sandbox's own, matching +// how appendMissingShares treats namespace shares: the narrower scope wins a +// name collision. +func appendMissingMCP(have, add []config.MCPServer) []config.MCPServer { + seen := make(map[string]bool, len(have)) + for _, s := range have { + seen[s.Name] = true + } + for _, s := range add { + if seen[s.Name] { + continue + } + seen[s.Name] = true + have = append(have, s) + } + return have +} + +// mcpNetworkBlock derives the egress a sandbox's declared MCP servers need: +// one allow entry per distinct http/sse host. Without it every remote server +// would be refused by the default policy and the user would have to mirror +// each declaration with a `network allow` line — the failure mode being a +// bare "ConnectionRefused" from inside the guest, far from its cause. +// +// Loopback targets are skipped: a server on the guest's own 127.0.0.1 (or a +// host service reached through a reverse forward) never leaves the VM, so +// there is nothing to allow. Literal IPs land in AllowIPs and hostnames in +// AllowDomains, matching how the two are enforced — SYN-time for addresses, +// DNS-time for names. +// +// Returns ok=false when nothing needs allowing, so callers don't append an +// empty block to the policy. +func mcpNetworkBlock(servers []config.MCPServer) (config.NetworkBlock, bool) { + block := config.NetworkBlock{Origin: config.BlockOriginMCP, Name: "mcp servers"} + for _, s := range servers { + if s.URL == "" { + continue // stdio: a local process, no egress of its own + } + u, err := url.Parse(s.URL) + if err != nil { + continue // validated at parse time; nothing to derive if it ever isn't + } + host := u.Hostname() + if host == "" || host == "localhost" { + continue + } + if addr, err := netip.ParseAddr(host); err == nil { + if addr.IsLoopback() { + continue + } + block.AllowIPs = append(block.AllowIPs, host) + continue + } + block.AllowDomains = append(block.AllowDomains, host) + } + block.AllowDomains = dedupStrings(block.AllowDomains) + block.AllowIPs = dedupStrings(block.AllowIPs) + if len(block.AllowDomains)+len(block.AllowIPs) == 0 { + return config.NetworkBlock{}, false + } + return block, true +} + +// applyMCPNetwork installs (or refreshes) the derived allow layer on a +// sandbox's policy. Idempotent: the block is rebuilt from the current server +// list every time, so removing a server from clawk.mod and re-creating drops +// its allow entry rather than leaving it behind. +func applyMCPNetwork(sb *config.Sandbox) { + block, ok := mcpNetworkBlock(sb.MCP) + sb.Network.Blocks = slices.DeleteFunc(sb.Network.Blocks, + func(b config.NetworkBlock) bool { return b.Origin == config.BlockOriginMCP }) + if ok { + sb.Network.Blocks = append(sb.Network.Blocks, block) + } + // Restore the origin-order invariant the store relies on. + sb.Network.Normalize() +} diff --git a/internal/cli/compose_mcp_test.go b/internal/cli/compose_mcp_test.go new file mode 100644 index 0000000..b2b2be7 --- /dev/null +++ b/internal/cli/compose_mcp_test.go @@ -0,0 +1,226 @@ +package cli + +import ( + "testing" + + "github.com/clawkwork/clawk/internal/config" + "github.com/clawkwork/clawk/internal/sandbox" + "github.com/clawkwork/clawk/internal/template" + "github.com/stretchr/testify/require" +) + +// TestMCPPipelineEndToEnd walks a realistic clawk.mod through every stage +// that stands between the file and a working server — parse, compose, derive +// egress, render the guest config, build the runner's argv — because each of +// those lives in a different package and a field dropped between two of them +// is exactly the kind of gap unit tests on either side both pass through. +func TestMCPPipelineEndToEnd(t *testing.T) { + tmpl, err := template.ParseString(`sandbox proj ( + mcp ( + linear https://mcp.linear.app/mcp header "Authorization: Bearer ${LINEAR_TOKEN}" + github stdio "npx -y @modelcontextprotocol/server-github" env GITHUB_TOKEN + ) + env ( + LINEAR_TOKEN = ${LINEAR_TOKEN:?create a Linear PAT} + GITHUB_TOKEN + ) +) +`) + require.NoError(t, err) + + var sources []mcpSource + for _, s := range tmpl.MCP { + sources = append(sources, mcpSource{Origin: "clawk.mod", Spec: s}) + } + servers, err := composeMCP(sources) + require.NoError(t, err) + + sb := &config.Sandbox{Name: "proj", MCP: servers, RequiredEnv: tmpl.Env} + applyMCPNetwork(sb) + + // The remote server's host is reachable; the stdio one adds no egress. + var allowed []string + for _, b := range sb.Network.Blocks { + if b.Origin == config.BlockOriginMCP { + allowed = b.AllowDomains + } + } + require.Equal(t, []string{"mcp.linear.app"}, allowed) + + // The rendered guest config names both servers and carries references, + // not values, even with the credential present in this process's env. + t.Setenv("LINEAR_TOKEN", "leaked-if-this-appears") + content, ok, err := sandbox.RenderMCPConfig(sb.MCP) + require.NoError(t, err) + require.True(t, ok) + require.NotContains(t, string(content), "leaked-if-this-appears") + require.Contains(t, string(content), "${LINEAR_TOKEN}") + require.Contains(t, string(content), "${GITHUB_TOKEN}") + + // The runner is told where to find it. + claude, err := agentByName("claude") + require.NoError(t, err) + require.Contains(t, launchArgs(sb, claude, nil), sandbox.GuestMCPConfigPath) + + // And the credential the config references is one the sandbox carries, + // so ${LINEAR_TOKEN} has something to expand to at connect time. + resolved, err := sandbox.ResolveEnv(sb) + require.NoError(t, err) + require.Contains(t, resolved, "LINEAR_TOKEN=leaked-if-this-appears", + "the value travels via the env path, never the config file") +} + +func httpSpec(name, url string, headers ...string) template.MCPSpec { + return template.MCPSpec{ + Name: name, + Transport: config.MCPTransportHTTP, + URL: url, + Headers: headers, + } +} + +// TestComposeMCPCollapsesIdenticalDeclarations: two repos in one workspace +// both needing the same server is normal, not a conflict. +func TestComposeMCPCollapsesIdenticalDeclarations(t *testing.T) { + spec := httpSpec("linear", "https://mcp.linear.app/mcp", "Authorization: Bearer ${T}") + got, err := composeMCP([]mcpSource{ + {Origin: "api", Spec: spec}, + {Origin: "web", Spec: spec}, + }) + require.NoError(t, err) + require.Len(t, got, 1) + require.Equal(t, "linear", got[0].Name) +} + +// TestComposeMCPRejectsConflict: the same name meaning two different things +// is a config bug. Silently picking one would hand the agent whichever +// definition composed first — the kind of thing you only notice when a tool +// call goes to the wrong place. +func TestComposeMCPRejectsConflict(t *testing.T) { + _, err := composeMCP([]mcpSource{ + {Origin: "api", Spec: httpSpec("linear", "https://mcp.linear.app/mcp")}, + {Origin: "web", Spec: httpSpec("linear", "https://staging.linear.app/mcp")}, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "declared differently") + require.Contains(t, err.Error(), "api") + require.Contains(t, err.Error(), "web") +} + +// TestAppendMissingMCPNarrowerScopeWins mirrors appendMissingShares: a +// namespace supplies the org-wide default, a repo's own declaration of the +// same name overrides it. +func TestAppendMissingMCPNarrowerScopeWins(t *testing.T) { + own := []config.MCPServer{{Name: "linear", Transport: config.MCPTransportHTTP, URL: "https://own/mcp"}} + ns := []config.MCPServer{ + {Name: "linear", Transport: config.MCPTransportHTTP, URL: "https://ns/mcp"}, + {Name: "notion", Transport: config.MCPTransportHTTP, URL: "https://mcp.notion.com/mcp"}, + } + got := appendMissingMCP(own, ns) + require.Len(t, got, 2) + require.Equal(t, "https://own/mcp", got[0].URL, "sandbox's own entry wins the name") + require.Equal(t, "notion", got[1].Name, "namespace-only entries still arrive") +} + +// TestMCPNetworkBlockDerivesHosts is the fix for the failure this feature +// exists to prevent: a declared remote server that the egress ACL refuses, +// surfacing inside the guest as a bare ConnectionRefused. +func TestMCPNetworkBlockDerivesHosts(t *testing.T) { + block, ok := mcpNetworkBlock([]config.MCPServer{ + {Name: "linear", Transport: config.MCPTransportHTTP, URL: "https://mcp.linear.app/mcp"}, + {Name: "sentry", Transport: config.MCPTransportSSE, URL: "https://mcp.sentry.dev/sse"}, + {Name: "dup", Transport: config.MCPTransportHTTP, URL: "https://mcp.linear.app/other"}, + {Name: "byip", Transport: config.MCPTransportHTTP, URL: "http://10.20.0.7:8080/mcp"}, + {Name: "local", Transport: config.MCPTransportHTTP, URL: "http://127.0.0.1:9000/mcp"}, + {Name: "named-local", Transport: config.MCPTransportHTTP, URL: "http://localhost:9000/mcp"}, + {Name: "gh", Transport: config.MCPTransportStdio, Command: []string{"npx", "server"}}, + }) + require.True(t, ok) + require.Equal(t, config.BlockOriginMCP, block.Origin) + require.ElementsMatch(t, []string{"mcp.linear.app", "mcp.sentry.dev"}, block.AllowDomains, + "hostnames are matched at DNS time and deduped") + require.Equal(t, []string{"10.20.0.7"}, block.AllowIPs, + "a literal address is enforced at SYN time, so it belongs in AllowIPs") +} + +func TestMCPNetworkBlockEmptyWhenNothingToAllow(t *testing.T) { + _, ok := mcpNetworkBlock(nil) + require.False(t, ok) + _, ok = mcpNetworkBlock([]config.MCPServer{ + {Name: "gh", Transport: config.MCPTransportStdio, Command: []string{"npx", "server"}}, + {Name: "local", Transport: config.MCPTransportHTTP, URL: "http://127.0.0.1:9000/mcp"}, + }) + require.False(t, ok, "a stdio server and a loopback URL need no egress") +} + +// TestApplyMCPNetworkIsIdempotent: the block is rebuilt from the current +// server list on every create, so removing a server retires its allow entry +// instead of leaving it behind forever. +func TestApplyMCPNetworkIsIdempotent(t *testing.T) { + sb := &config.Sandbox{ + MCP: []config.MCPServer{{Name: "linear", Transport: config.MCPTransportHTTP, URL: "https://mcp.linear.app/mcp"}}, + } + applyMCPNetwork(sb) + applyMCPNetwork(sb) + var mcpBlocks int + for _, b := range sb.Network.Blocks { + if b.Origin == config.BlockOriginMCP { + mcpBlocks++ + require.Equal(t, []string{"mcp.linear.app"}, b.AllowDomains) + } + } + require.Equal(t, 1, mcpBlocks, "re-applying must replace, not accumulate") + + sb.MCP = nil + applyMCPNetwork(sb) + for _, b := range sb.Network.Blocks { + require.NotEqual(t, config.BlockOriginMCP, b.Origin, "dropping the server drops its allow") + } +} + +// TestApplyMCPNetworkRanksBelowUserRules pins the precedence choice: the +// derived allow is a convenience, so an explicit deny the user wrote in +// clawk.mod or via the CLI has to outrank it. +func TestApplyMCPNetworkRanksBelowUserRules(t *testing.T) { + sb := &config.Sandbox{ + MCP: []config.MCPServer{{Name: "linear", Transport: config.MCPTransportHTTP, URL: "https://mcp.linear.app/mcp"}}, + Network: config.NetworkPolicy{Blocks: []config.NetworkBlock{ + {Origin: config.BlockOriginCustom, DenyDomains: []string{"mcp.linear.app"}}, + {Origin: config.BlockOriginMod, Name: "clawk.mod"}, + }}, + } + applyMCPNetwork(sb) + + var order []string + for _, b := range sb.Network.Blocks { + order = append(order, b.Origin) + } + require.Equal(t, + []string{config.BlockOriginMCP, config.BlockOriginMod, config.BlockOriginCustom}, + order, "mcp sits below mod and custom in increasing precedence") +} + +// TestLaunchArgsMCPConfig covers both directions: a sandbox with servers gets +// the flag, one without must not (the file wouldn't exist), and the user's +// own args always land last so they can override clawk. +func TestLaunchArgsMCPConfig(t *testing.T) { + claude, err := agentByName("claude") + require.NoError(t, err) + + withMCP := &config.Sandbox{MCP: []config.MCPServer{{Name: "linear"}}} + args := launchArgs(withMCP, claude, []string{"--resume"}) + require.Equal(t, []string{ + "--dangerously-skip-permissions", + "--mcp-config", sandbox.GuestMCPConfigPath, + "--resume", + }, args) + + bare := &config.Sandbox{} + require.Equal(t, []string{"--dangerously-skip-permissions"}, launchArgs(bare, claude, nil), + "no declared servers means no flag pointing at a missing file") + + codex, err := agentByName("codex") + require.NoError(t, err) + require.NotContains(t, launchArgs(withMCP, codex, nil), "--mcp-config", + "a runner clawk can't render config for gets no flag") +} diff --git a/internal/cli/compose_serial.go b/internal/cli/compose_serial.go new file mode 100644 index 0000000..54ae508 --- /dev/null +++ b/internal/cli/compose_serial.go @@ -0,0 +1,110 @@ +package cli + +import ( + "fmt" + + "github.com/clawkwork/clawk/internal/config" + "github.com/clawkwork/clawk/internal/serialfwd" + "github.com/clawkwork/clawk/internal/template" +) + +type serialSource struct { + Origin string + Spec template.SerialSpec +} + +// mergeSerials gathers `serial (...)` entries from the workspace file and +// every repo's Clawkfile and resolves them into the sandbox's device list. +func mergeSerials(ws *template.Workspace) ([]config.SerialDevice, error) { + var sources []serialSource + for _, s := range ws.File.Serials { + sources = append(sources, serialSource{Origin: "workspace", Spec: s}) + } + for _, r := range ws.Repos { + if r.Clawkfile == nil { + continue + } + for _, s := range r.Clawkfile.Serials { + sources = append(sources, serialSource{Origin: r.Name, Spec: s}) + } + } + return composeSerials(sources) +} + +// composeSerials is the pure conflict-detection core split out from +// mergeSerials so the here-mode path (a single Clawkfile) and the workspace +// path share it — same split as composeFiles and composeShares. +// +// Two things can collide, and both are refused with a message naming the +// contributors rather than resolved by precedence: a guest name, because +// the guest creates /dev/ exactly once and a second claim would +// silently lose in there, and a host device, because forwarding one port +// under two names would let the sandbox open it twice. +func composeSerials(sources []serialSource) ([]config.SerialDevice, error) { + type claim struct { + Origin string + Device config.SerialDevice + } + byName := make(map[string]claim) + byHost := make(map[string]claim) + + var out []config.SerialDevice + for _, s := range sources { + dev, err := resolveSerialSpec(s.Spec) + if err != nil { + return nil, fmt.Errorf("%s serial %q: %w", s.Origin, s.Spec.HostPath, err) + } + if prev, dup := byName[dev.GuestName]; dup { + if prev.Device == dev { + // The same device declared twice — by a repo and the + // workspace, say. Harmless; keep the first. + continue + } + return nil, fmt.Errorf( + "serial device %q is claimed by both %s (%s) and %s (%s)", + dev.GuestName, prev.Origin, prev.Device.HostPath, s.Origin, dev.HostPath) + } + if prev, dup := byHost[dev.HostPath]; dup { + return nil, fmt.Errorf( + "serial port %s is forwarded twice: as %q by %s and as %q by %s", + dev.HostPath, prev.Device.GuestName, prev.Origin, dev.GuestName, s.Origin) + } + byName[dev.GuestName] = claim{Origin: s.Origin, Device: dev} + byHost[dev.HostPath] = claim{Origin: s.Origin, Device: dev} + out = append(out, dev) + } + return out, nil +} + +// resolveSerialSpec turns one parsed clawk.mod entry into a device, +// applying the same defaults and validation the CLI's HOST:GUEST spec +// parser does. +func resolveSerialSpec(spec template.SerialSpec) (config.SerialDevice, error) { + hostPath, err := template.ExpandPath(spec.HostPath) + if err != nil { + return config.SerialDevice{}, err + } + if !isAbsSerialPath(hostPath) { + return config.SerialDevice{}, fmt.Errorf( + "must be an absolute path (e.g. /dev/cu.usbmodem1101)") + } + + guestName := spec.GuestName + if guestName == "" { + guestName = config.DefaultSerialGuestName(hostPath) + if guestName == "" { + return config.SerialDevice{}, fmt.Errorf( + "a device pattern needs an explicit guest name (e.g. '%s ttyACM0')", hostPath) + } + } + if err := serialfwd.ValidDeviceName(guestName); err != nil { + return config.SerialDevice{}, err + } + return config.SerialDevice{HostPath: hostPath, GuestName: guestName}, nil +} + +// isAbsSerialPath is filepath.IsAbs pinned to POSIX semantics. Serial +// devices are named the same way on both hosts clawk runs on, and a +// clawk.mod is shared across machines, so this shouldn't vary by where it +// happens to be parsed. +func isAbsSerialPath(p string) bool { return len(p) > 0 && p[0] == '/' } diff --git a/internal/cli/compose_serial_test.go b/internal/cli/compose_serial_test.go new file mode 100644 index 0000000..e715c02 --- /dev/null +++ b/internal/cli/compose_serial_test.go @@ -0,0 +1,93 @@ +package cli + +import ( + "testing" + + "github.com/clawkwork/clawk/internal/config" + "github.com/clawkwork/clawk/internal/template" + "github.com/stretchr/testify/require" +) + +func src(origin, host, guest string) serialSource { + return serialSource{ + Origin: origin, + Spec: template.SerialSpec{HostPath: host, GuestName: guest}, + } +} + +func TestComposeSerials(t *testing.T) { + got, err := composeSerials([]serialSource{ + src("workspace", "/dev/cu.usbmodem1101", ""), + src("firmware", "/dev/cu.usbserial-A50285BI", "ttyUSB0"), + }) + require.NoError(t, err) + require.Equal(t, []config.SerialDevice{ + {HostPath: "/dev/cu.usbmodem1101", GuestName: "cu.usbmodem1101"}, + {HostPath: "/dev/cu.usbserial-A50285BI", GuestName: "ttyUSB0"}, + }, got) +} + +// The workspace and a repo both declaring the same board is ordinary, not a +// conflict — it only becomes one when they disagree. +func TestComposeSerialsIdenticalDuplicateIsFine(t *testing.T) { + got, err := composeSerials([]serialSource{ + src("workspace", "/dev/cu.usbmodem1101", "ttyACM0"), + src("firmware", "/dev/cu.usbmodem1101", "ttyACM0"), + }) + require.NoError(t, err) + require.Len(t, got, 1) +} + +// Both collision kinds must name their contributors: with several repos in +// a workspace, "something clashed" is not actionable. +func TestComposeSerialsRejectsNameClash(t *testing.T) { + _, err := composeSerials([]serialSource{ + src("workspace", "/dev/cu.usbmodem1101", "ttyACM0"), + src("firmware", "/dev/cu.usbmodem2201", "ttyACM0"), + }) + require.Error(t, err) + require.Contains(t, err.Error(), "workspace") + require.Contains(t, err.Error(), "firmware") + require.Contains(t, err.Error(), "ttyACM0") +} + +func TestComposeSerialsRejectsSamePortTwice(t *testing.T) { + _, err := composeSerials([]serialSource{ + src("workspace", "/dev/cu.usbmodem1101", "ttyACM0"), + src("firmware", "/dev/cu.usbmodem1101", "ttyACM1"), + }) + require.Error(t, err) + require.Contains(t, err.Error(), "forwarded twice") + require.Contains(t, err.Error(), "/dev/cu.usbmodem1101") +} + +func TestComposeSerialsRejectsRelativePath(t *testing.T) { + _, err := composeSerials([]serialSource{src("workspace", "cu.usbmodem1101", "")}) + require.Error(t, err) + require.Contains(t, err.Error(), "absolute path") +} + +func TestComposeSerialsRejectsGlobWithoutName(t *testing.T) { + _, err := composeSerials([]serialSource{src("workspace", "/dev/cu.usbmodem*", "")}) + require.Error(t, err) + require.Contains(t, err.Error(), "needs an explicit guest name") +} + +// The name becomes /dev/ in the guest, so a clawk.mod is just as much +// an untrusted-ish input as the wire is — it may come from a cloned repo. +func TestComposeSerialsRejectsUnsafeGuestName(t *testing.T) { + for _, name := range []string{"../escape", "sub/dir", ".hidden", "has space"} { + t.Run(name, func(t *testing.T) { + _, err := composeSerials([]serialSource{ + src("workspace", "/dev/cu.usbmodem1101", name), + }) + require.Error(t, err) + }) + } +} + +func TestComposeSerialsEmpty(t *testing.T) { + got, err := composeSerials(nil) + require.NoError(t, err) + require.Empty(t, got) +} diff --git a/internal/cli/daemon.go b/internal/cli/daemon.go index 2f57103..a34f7f4 100644 --- a/internal/cli/daemon.go +++ b/internal/cli/daemon.go @@ -472,12 +472,20 @@ type reverseForwardSink interface { Set([]config.PortForward) } +// serialSink publishes a serial-device set to the in-guest agent. An +// interface for the same reason reverseForwardSink is one: the only +// implementation is the darwin serialProxy, and the firecracker daemon +// passes nil. +type serialSink interface { + Set([]config.SerialDevice) +} + // controlHandlers builds the control-socket callbacks shared by both VM // daemons: the denial ledger, a live network-policy reload from the store, // the VM lifecycle surface (pause/resume/suspend), reverse-forward reloads // when the daemon has a sink for them, and (when the allow list has one) // the interactive gate. -func controlHandlers(sb *config.Sandbox, allow *netfilter.AllowList, lc *vmLifecycle, rev reverseForwardSink, logger *log.Logger) vzdctl.Handlers { +func controlHandlers(sb *config.Sandbox, allow *netfilter.AllowList, lc *vmLifecycle, rev reverseForwardSink, ser serialSink, logger *log.Logger) vzdctl.Handlers { h := vzdctl.Handlers{ Denials: allow.Denials, Lifecycle: lc.lifecycleHandlers(), @@ -508,6 +516,17 @@ func controlHandlers(sb *config.Sandbox, allow *netfilter.AllowList, lc *vmLifec return nil } } + if ser != nil { + h.ReloadSerials = func() error { + cur, err := store.Load(sb.Name) + if err != nil { + return fmt.Errorf("reloading sandbox record: %w", err) + } + ser.Set(cur.Serials) + logger.Printf("serial devices reloaded: %d", len(cur.Serials)) + return nil + } + } return h } diff --git a/internal/cli/fcd.go b/internal/cli/fcd.go index a4fcfc1..a07768d 100644 --- a/internal/cli/fcd.go +++ b/internal/cli/fcd.go @@ -91,7 +91,7 @@ func runFcd(_ *cobra.Command, args []string) (retErr error) { // No reverse-forward sink: firecracker's vsock is one-way (guest listens, // host dials), so there is nothing for the guest agent to dial. The // endpoint 404s and the CLI says so. - ctl, err := vzdctl.Start(vzdctl.SocketPath(vmDir), controlHandlers(sb, allow, lc, nil, logger)) + ctl, err := vzdctl.Start(vzdctl.SocketPath(vmDir), controlHandlers(sb, allow, lc, nil, nil, logger)) if err != nil { logger.Printf("control socket: disabled (%v) — network edits apply on next up", err) } else { diff --git a/internal/cli/global_defaults_test.go b/internal/cli/global_defaults_test.go new file mode 100644 index 0000000..710c8f2 --- /dev/null +++ b/internal/cli/global_defaults_test.go @@ -0,0 +1,207 @@ +package cli + +import ( + "os" + "path/filepath" + "slices" + "testing" + + "github.com/clawkwork/clawk/internal/config" + "github.com/clawkwork/clawk/internal/template" + "github.com/stretchr/testify/require" +) + +// withGlobalMod writes src as the host-wide clawk.mod inside a fresh +// XDG_CONFIG_HOME and enables the layer for the duration of the test (the +// package otherwise runs with it off — see TestMain). Returns its directory. +func withGlobalMod(t *testing.T, src string) string { + t.Helper() + home := t.TempDir() + dir := filepath.Join(home, ".config", "clawk") + require.NoError(t, os.MkdirAll(dir, 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(dir, template.RepoFileName), []byte(src), 0o644)) + + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config")) + t.Setenv(template.GlobalModEnvVar, "") + + template.GlobalDisabled = false + noGlobalFlag = false + t.Cleanup(func() { + template.GlobalDisabled = true + noGlobalFlag = true + }) + return dir +} + +// End to end through `clawk work`: a host-wide clawk.mod reaches the sandbox +// record for every directive group, and the repo's own file still wins. +func TestGlobalDefaultsReachTheRecord(t *testing.T) { + setupTest(t) + dir := withGlobalMod(t, `sandbox ( + vm ( + memory_max 8GiB + provider vz + ) + network ( + allow global.example.com + ) + env ( + GLOBAL_TOKEN + ) + files ( + ./house.netrc /home/agent/.netrc + ) + agent ( + instructions "house rule" + ) +) +`) + require.NoError(t, os.WriteFile(filepath.Join(dir, "house.netrc"), []byte("x"), 0o600)) + + repo := filepath.Join(t.TempDir(), "proj") + require.NoError(t, os.MkdirAll(repo, 0o755)) + gitInit(t, repo) + require.NoError(t, os.WriteFile(filepath.Join(repo, template.RepoFileName), []byte(`sandbox ( + vm ( + memory_max 16GiB + ) + network ( + allow repo.example.com + ) + env ( + REPO_TOKEN + ) + agent ( + instructions "repo rule" + ) +) +`), 0o644)) + + _, err := executeCommand("work", filepath.Join(repo, template.RepoFileName), "TICKET-9", "--bare") + require.NoError(t, err) + + sb, err := store.Load("TICKET-9") + require.NoError(t, err) + + require.EqualValues(t, 16384, sb.MemoryMaxMiB, "the repo's own memory_max must win") + require.Equal(t, config.Provider("vz"), sb.Provider, "global provider applies where the repo is silent") + + mod := sb.Network.Block(config.BlockOriginMod) + require.Contains(t, mod.AllowDomains, "global.example.com") + require.Contains(t, mod.AllowDomains, "repo.example.com") + + require.True(t, slices.Contains(sb.RequiredEnv, "GLOBAL_TOKEN"), "env: %v", sb.RequiredEnv) + require.True(t, slices.Contains(sb.RequiredEnv, "REPO_TOKEN"), "env: %v", sb.RequiredEnv) + + // The relative host path resolved against the global file's directory, not + // the repo or the process CWD. + var netrc *config.HostFile + for i := range sb.Files { + if sb.Files[i].GuestPath == "/home/agent/.netrc" { + netrc = &sb.Files[i] + } + } + require.NotNil(t, netrc, "global file entry missing: %v", sb.Files) + require.Equal(t, filepath.Join(dir, "house.netrc"), netrc.HostPath) + + require.Equal(t, []string{"house rule", "repo rule"}, sb.Instructions, + "the broader scope's instructions read first") +} + +// A broken host-wide file must be reported as such. resolveSource walks +// workspace → standalone → bare-git-repo and treats each failure as "not this +// shape", which swallowed the real error and answered "no clawk.mod found, and +// X is not a git repo" — for a directory that had a clawk.mod and was a git +// repo. See template.ErrGlobalMod. +func TestBrokenGlobalIsReportedNotSwallowed(t *testing.T) { + setupTest(t) + // A header name is the scope violation with the most misleading fallback: + // the file parses, so nothing else complains. + withGlobalMod(t, "sandbox house (\n vm (\n cpu 2\n )\n)\n") + + repo := filepath.Join(t.TempDir(), "proj") + require.NoError(t, os.MkdirAll(repo, 0o755)) + gitInit(t, repo) + require.NoError(t, os.WriteFile(filepath.Join(repo, template.RepoFileName), + []byte("sandbox (\n)\n"), 0o644)) + + t.Chdir(repo) + _, err := resolveSource("", "") + require.ErrorIs(t, err, template.ErrGlobalMod) + require.ErrorContains(t, err, "must be anonymous") + require.NotContains(t, err.Error(), "is not a git repo") +} + +func TestNoGlobalFlagDropsTheLayer(t *testing.T) { + setupTest(t) + withGlobalMod(t, `sandbox ( + network ( + allow global.example.com + ) + env ( + GLOBAL_TOKEN + ) +) +`) + repo := filepath.Join(t.TempDir(), "proj") + require.NoError(t, os.MkdirAll(repo, 0o755)) + gitInit(t, repo) + require.NoError(t, os.WriteFile(filepath.Join(repo, template.RepoFileName), + []byte("sandbox (\n network (\n allow repo.example.com\n )\n)\n"), 0o644)) + + _, err := executeCommand("work", filepath.Join(repo, template.RepoFileName), + "TICKET-10", "--bare", "--no-global") + require.NoError(t, err) + + sb, err := store.Load("TICKET-10") + require.NoError(t, err) + mod := sb.Network.Block(config.BlockOriginMod) + require.Contains(t, mod.AllowDomains, "repo.example.com") + require.NotContains(t, mod.AllowDomains, "global.example.com") + require.Empty(t, sb.RequiredEnv) +} + +// The here-mode path (`clawk` bare) must refuse a broken host-wide file for +// the same reason resolveSource does — and it matters more here, because the +// standalone loader fails as a WHOLE: degrading to "no defaults" threw away +// the repo's own clawk.mod too, creating a sandbox with none of its forwards, +// env or instructions and only a line on stderr to say so. +func TestHereModeRefusesBrokenGlobal(t *testing.T) { + setupTest(t) + // A header name parses fine and violates only the global scope, so + // nothing but the ErrGlobalMod check stands between it and a silently + // misconfigured sandbox. + withGlobalMod(t, "sandbox house (\n vm (\n cpu 2\n )\n)\n") + + repo := filepath.Join(t.TempDir(), "proj") + require.NoError(t, os.MkdirAll(repo, 0o755)) + gitInit(t, repo) + require.NoError(t, os.WriteFile(filepath.Join(repo, template.RepoFileName), + []byte("sandbox (\n forwards ( 3000 )\n env ( MY_TOKEN )\n)\n"), 0o644)) + + _, _, _, err := loadHereClawkfile(repo) + require.ErrorIs(t, err, template.ErrGlobalMod) + require.ErrorContains(t, err, "must be anonymous") +} + +// The repo's own clawk.mod must still come back intact when the host-wide +// layer is merely absent — the overwhelmingly common case, and the one the +// error path above must not swallow. +func TestHereModeKeepsRepoConfigWithoutGlobal(t *testing.T) { + setupTest(t) + + repo := filepath.Join(t.TempDir(), "proj") + require.NoError(t, os.MkdirAll(repo, 0o755)) + gitInit(t, repo) + require.NoError(t, os.WriteFile(filepath.Join(repo, template.RepoFileName), + []byte("sandbox (\n forwards ( 3000 )\n env ( MY_TOKEN )\n)\n"), 0o644)) + + tmpl, _, globalPath, err := loadHereClawkfile(repo) + require.NoError(t, err) + require.Empty(t, globalPath) + require.NotNil(t, tmpl) + require.Equal(t, []string{"MY_TOKEN"}, tmpl.Env) + require.Len(t, tmpl.Forwards, 1) +} diff --git a/internal/cli/here.go b/internal/cli/here.go index 1d7eccd..a2e59ce 100644 --- a/internal/cli/here.go +++ b/internal/cli/here.go @@ -157,7 +157,13 @@ func loadOrCreateHereSandbox(name, cwd string) (*config.Sandbox, bool, error) { return sb, false, nil } - clawkfile, policies := loadHereClawkfile(cwd) + clawkfile, policies, globalPath, err := loadHereClawkfile(cwd) + if err != nil { + return nil, false, err + } + if globalPath != "" { + fmt.Printf("Using host-wide defaults from %s\n", globalPath) + } // Register the file's `policy ( ... )` blocks before composing // the network policy — `use` references resolve against the store at @@ -187,6 +193,8 @@ func loadOrCreateHereSandbox(name, cwd string) (*config.Sandbox, bool, error) { var memory string var fileSources []fileSource var shareSources []shareSource + var serialSources []serialSource + var mcpSources []mcpSource if clawkfile != nil { forwardSpecs = append(forwardSpecs, clawkfile.Forwards...) reverseForwardSpecs = append(reverseForwardSpecs, clawkfile.ReverseForwards...) @@ -209,6 +217,12 @@ func loadOrCreateHereSandbox(name, cwd string) (*config.Sandbox, bool, error) { for _, s := range clawkfile.Shares { shareSources = append(shareSources, shareSource{Origin: "clawk.mod", Spec: s}) } + for _, s := range clawkfile.Serials { + serialSources = append(serialSources, serialSource{Origin: "clawk.mod", Spec: s}) + } + for _, s := range clawkfile.MCP { + mcpSources = append(mcpSources, mcpSource{Origin: "clawk.mod", Spec: s}) + } } forwards, err := parseForwardSpecs(forwardSpecs, cwd) @@ -227,18 +241,27 @@ func loadOrCreateHereSandbox(name, cwd string) (*config.Sandbox, bool, error) { if err != nil { return nil, false, err } + serials, err := composeSerials(serialSources) + if err != nil { + return nil, false, err + } + mcpServers, err := composeMCP(mcpSources) + if err != nil { + return nil, false, err + } var nested bool var cpu uint var memoryMiB, memoryMaxMiB, diskMiB uint64 var image, kernel string - var idleTimeoutSec int64 + var idleTimeoutSec, swapMiB int64 if clawkfile != nil { nested = clawkfile.Nested cpu = clawkfile.CPU memoryMiB = clawkfile.MemoryMiB memoryMaxMiB = clawkfile.MemoryMaxMiB diskMiB = clawkfile.DiskMiB + swapMiB = clawkfile.SwapMiB image = clawkfile.Image kernel = clawkfile.Kernel idleTimeoutSec = clawkfile.IdleTimeoutSec @@ -265,6 +288,8 @@ func loadOrCreateHereSandbox(name, cwd string) (*config.Sandbox, bool, error) { ReverseForwards: reverseForwards, Files: files, Shares: shares, + Serials: serials, + MCP: mcpServers, RequiredEnv: requiredEnv, Instructions: instructions, Memory: memory, @@ -273,6 +298,7 @@ func loadOrCreateHereSandbox(name, cwd string) (*config.Sandbox, bool, error) { MemoryMiB: memoryMiB, MemoryMaxMiB: memoryMaxMiB, DiskMiB: diskMiB, + SwapMiB: swapMiB, // IdleTimeoutSec rides the same snapshot-at-create rule as every // other clawk.mod value; it was the one vm(...) field this path // forgot to copy when idle-stop landed, which silently pinned every @@ -295,6 +321,8 @@ func loadOrCreateHereSandbox(name, cwd string) (*config.Sandbox, bool, error) { }}, CreatedAt: time.Now(), } + // Derived MCP egress, before the first Save — see applyMCPNetwork. + applyMCPNetwork(sb) if err := store.Save(sb); err != nil { return nil, false, fmt.Errorf("saving sandbox: %w", err) } @@ -302,21 +330,44 @@ func loadOrCreateHereSandbox(name, cwd string) (*config.Sandbox, bool, error) { } // loadHereClawkfile reads cwd/clawk.mod if present and returns its sandbox -// template plus any policy blocks declared beside it. Missing or unreadable -// clawk.mod is not an error — callers fall back to defaults. -func loadHereClawkfile(cwd string) (*template.Template, []template.PolicyDef) { +// template — with the host-wide clawk.mod folded in underneath — plus any +// policy blocks declared beside either, and the host-wide file's path for the +// create-time note. Missing or unreadable clawk.mod is not an error: callers +// fall back to defaults, which is where the host-wide layer alone lands. +// +// A broken host-wide layer IS an error, and it has to be, for the same reason +// resolveSource surfaces it: the standalone loader fails as a whole, so +// degrading to "no defaults" here would throw away the repo's OWN clawk.mod +// too — its forwards, env and instructions — and create the sandbox anyway +// with nothing but a line on stderr to say so. See template.ErrGlobalMod. +func loadHereClawkfile(cwd string) (*template.Template, []template.PolicyDef, string, error) { ws, err := template.LoadStandaloneClawkfileWithProfile(cwd, "") if err != nil { + // The host-wide layer carries its own context, so it is not prefixed + // with a clawk.mod the user may not even have. + if errors.Is(err, template.ErrGlobalMod) { + return nil, nil, "", err + } // Only surface non-ENOENT errors; absence is expected. if !errors.Is(err, os.ErrNotExist) { fmt.Fprintf(os.Stderr, "warning: reading clawk.mod: %v\n", err) + return nil, nil, "", nil } - return nil, nil + // No clawk.mod here — the host-wide defaults are the whole template. + // The standalone loader never got far enough to fold them in. + g, gerr := template.LoadGlobal() + if gerr != nil { + if !errors.Is(gerr, template.ErrNoGlobalMod) { + return nil, nil, "", gerr + } + return nil, nil, "", nil + } + return g.Template, g.Policies, g.Path, nil } if ws == nil || len(ws.Repos) == 0 { - return nil, nil + return nil, nil, "", nil } - return ws.Repos[0].Clawkfile, ws.Policies + return ws.Repos[0].Clawkfile, ws.Policies, ws.GlobalPath, nil } // parseForwardSpecs turns a list of "PORT" or "HOST:GUEST" strings into diff --git a/internal/cli/main_test.go b/internal/cli/main_test.go new file mode 100644 index 0000000..2e70961 --- /dev/null +++ b/internal/cli/main_test.go @@ -0,0 +1,23 @@ +package cli + +import ( + "os" + "testing" + + "github.com/clawkwork/clawk/internal/template" +) + +// TestMain runs the whole package as if --no-global had been passed: a +// developer's own ~/.config/clawk/clawk.mod must never leak into the sandbox +// records these tests compose. +// +// Both knobs are set. GlobalDisabled covers tests that call the loaders +// directly; noGlobalFlag covers those going through executeCommand, whose +// PersistentPreRunE re-derives GlobalDisabled from the flag on every +// invocation. Tests for the layer itself flip both (see withGlobalMod in +// global_defaults_test.go). +func TestMain(m *testing.M) { + template.GlobalDisabled = true + noGlobalFlag = true + os.Exit(m.Run()) +} diff --git a/internal/cli/namespace.go b/internal/cli/namespace.go index 8d50575..8630825 100644 --- a/internal/cli/namespace.go +++ b/internal/cli/namespace.go @@ -64,6 +64,7 @@ func applyNamespaceDefaults(sb *config.Sandbox) error { sb.RequiredEnv = dedupStrings(append(sb.RequiredEnv, ns.Env...)) sb.Files = appendMissingFiles(sb.Files, ns.Files) sb.Shares = appendMissingShares(sb.Shares, ns.Shares) + sb.MCP = appendMissingMCP(sb.MCP, ns.MCP) // Namespace scope is broader than the sandbox's own (clawk.mod) entries, // so it reads first: namespace instructions, then sandbox-specific. sb.Instructions = append(append([]string{}, ns.Instructions...), sb.Instructions...) diff --git a/internal/cli/resources.go b/internal/cli/resources.go index 1ea0d0a..b1db8b1 100644 --- a/internal/cli/resources.go +++ b/internal/cli/resources.go @@ -95,6 +95,31 @@ func resolveDisk(ws *template.Workspace) uint64 { return disk } +// resolveSwap merges the swap directive across a workspace and its repos. +// An explicit "off" (negative) wins over any size, mirroring how +// resolveIdleTimeout lets "off" win: a repo that declares `swap off` is +// saying its workload must not be swapped — a latency measurement, a +// benchmark — and honoring a different repo's larger size would break that, +// while the reverse only costs the other repo some headroom. Among sizes the +// max wins, the same rule as resolveResources: the VM is shared. +func resolveSwap(ws *template.Workspace) int64 { + swap := ws.File.SwapMiB + for _, r := range ws.Repos { + if r.Clawkfile == nil { + continue + } + v := r.Clawkfile.SwapMiB + if v < 0 || swap < 0 { + swap = -1 + continue + } + if v > swap { + swap = v + } + } + return swap +} + // minDiskMiB is the floor for a per-sandbox `vm ( disk )` override. // Below ~1 GiB there is no room for the base image plus any writes, and // such a value is almost always a unit typo (disk 32M meaning 32G). Zero diff --git a/internal/cli/root.go b/internal/cli/root.go index 40ffb5a..6c66fcb 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -135,8 +135,16 @@ func init() { "OCI image for new sandboxes (registry ref or docker-save tarball path; overrides clawk.mod)") rootCmd.PersistentFlags().StringVar(&kernelFlag, "kernel", "", "guest kernel override for new sandboxes (local vmlinux path or http(s) URL; overrides clawk.mod). Default: the Kata kernel") + rootCmd.PersistentFlags().BoolVar(&noGlobalFlag, "no-global", false, + "ignore the host-wide clawk.mod (~/.config/clawk/clawk.mod) — the repo's own config only, for a reproducible run") } +// noGlobalFlag (--no-global) drops the host-wide defaults layer. It exists +// because that file makes a sandbox's shape depend on host-local config: a CI +// run or a bug report needs a way to say "just what's in the repo". Applied in +// PersistentPreRunE (see setup.go), before any template load. +var noGlobalFlag bool + // testProvider, if non-nil, is returned from providerFor — used by tests to // inject a MockProvider without going through provider selection. var testProvider sandbox.Provider diff --git a/internal/cli/run.go b/internal/cli/run.go index f7be58d..01f51ca 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -53,8 +53,9 @@ func registerSafeFlag(cmd *cobra.Command) { cmd.Flags().BoolVar(&runSafe, "safe", false, "attach the runner without its permission-bypass default args "+ "(claude's --dangerously-skip-permissions, codex's "+ - "--dangerously-bypass-approvals-and-sandbox), so the agent asks "+ - "for confirmation as it would on the host") + "--dangerously-bypass-approvals-and-sandbox, pi's --approve, "+ + "opencode's --auto), so the agent asks for confirmation as it "+ + "would on the host") } // applySafeMode drops a runner's permission-bypass DefaultArgs when --safe is @@ -239,6 +240,12 @@ func resolveSource(source, profile string) (*template.Workspace, error) { } if ws, err := template.LoadStandaloneClawkfileWithProfile(cwd, profile); err == nil { return ws, nil + } else if errors.Is(err, template.ErrGlobalMod) { + // The repo's own file may well be absent — but the host-wide layer + // being broken is not a reason to try the next shape, it's the + // answer. Without this the rung below swallows it and the user gets + // "no clawk.mod found, and X is not a git repo". + return nil, err } // Last resort: cwd is itself a git repo. Treat it as a single-repo // workspace inheriting only defaults — keeps `clawk work ` @@ -248,11 +255,22 @@ func resolveSource(source, profile string) (*template.Workspace, error) { return nil, fmt.Errorf( "--profile requires a clawk.mod (none found in %s)", cwd) } - if ws, err := template.WorkspaceFromGitRepo(cwd); err == nil { - fmt.Fprintf(os.Stderr, - "note: no clawk.mod in %s; using defaults "+ - "(no forwards, no setup). Add clawk.mod to declare them.\n", - ws.Root) + ws, gerr := template.WorkspaceFromGitRepo(cwd) + if errors.Is(gerr, template.ErrGlobalMod) { + return nil, gerr + } + if gerr == nil { + if ws.GlobalPath != "" { + fmt.Fprintf(os.Stderr, + "note: no clawk.mod in %s; using the host-wide defaults in %s. "+ + "Add clawk.mod to declare repo-specific settings.\n", + ws.Root, ws.GlobalPath) + } else { + fmt.Fprintf(os.Stderr, + "note: no clawk.mod in %s; using defaults "+ + "(no forwards, no setup). Add clawk.mod to declare them.\n", + ws.Root) + } return ws, nil } return nil, fmt.Errorf( @@ -354,6 +372,14 @@ func loadOrCreateSandboxFromWorkspace(name string, ws *template.Workspace) (*con if err != nil { return nil, err } + serials, err := mergeSerials(ws) + if err != nil { + return nil, err + } + mcpServers, err := mergeMCP(ws) + if err != nil { + return nil, err + } // Nested virt: workspace-level `nested` directive OR any repo's // Clawkfile requesting it turns it on for the whole sandbox. This @@ -398,11 +424,14 @@ func loadOrCreateSandboxFromWorkspace(name string, ws *template.Workspace) (*con ReverseForwards: reverseForwards, Files: files, Shares: shares, + Serials: serials, + MCP: mcpServers, NestedVirt: nested, CPU: cpu, MemoryMiB: memoryMiB, MemoryMaxMiB: memoryMaxMiB, DiskMiB: disk, + SwapMiB: resolveSwap(ws), IdleTimeoutSec: resolveIdleTimeout(ws), Image: image, Kernel: kernel, @@ -418,6 +447,14 @@ func loadOrCreateSandboxFromWorkspace(name string, ws *template.Workspace) (*con if err := applyNamespaceDefaults(sb); err != nil { return nil, err } + // After the namespace, so its entries stay narrower than the workspace's. + if err := applyWorkspaceLevelDefaults(sb, ws); err != nil { + return nil, err + } + // After the namespace has folded its own servers in, so the derived + // allow layer covers the full set. Before the first Save, so the very + // first boot already has the egress its servers need. + applyMCPNetwork(sb) if err := store.Save(sb); err != nil { return nil, err } @@ -427,9 +464,41 @@ func loadOrCreateSandboxFromWorkspace(name string, ws *template.Workspace) (*con } else { fmt.Printf("Created sandbox %q (provider: %s)\n", sb.DisplayName(), sb.Provider) } + if ws.GlobalPath != "" { + // Named at create because the layer is host-local: it explains config + // nobody can find by reading the repo. --no-global excludes it. + fmt.Printf(" host-wide defaults from %s\n", ws.GlobalPath) + } return sb, nil } +// applyWorkspaceLevelDefaults folds the workspace-position `env ( … )` and +// `agent ( … )` blocks into the record — the workspace file's own, plus the +// host-wide clawk.mod folded underneath it (see template/global.go). +// +// Both were previously dropped on the floor: only repo Clawkfiles fed +// RequiredEnv and the agent docs (see addPhases), so a workspace root +// declaring `env ( GITHUB_TOKEN )` silently got nothing. Ordering is +// scope-outward — workspace, then namespace, then repo — which for env is what +// decides precedence, since both the profile.d exports and exec's environment +// let the LAST occurrence of a name win. +func applyWorkspaceLevelDefaults(sb *config.Sandbox, ws *template.Workspace) error { + if len(ws.File.Env) > 0 { + sb.RequiredEnv = unionStrings(ws.File.Env, sb.RequiredEnv) + } + instr, err := resolveAgentDocs(ws.Root, ws.File.Instructions) + if err != nil { + return fmt.Errorf("workspace agent instructions: %w", err) + } + sb.Instructions = append(instr, sb.Instructions...) + mem, err := resolveAgentDocs(ws.Root, ws.File.Memory) + if err != nil { + return fmt.Errorf("workspace agent memory: %w", err) + } + sb.Memory = joinMemory(joinMemory(mem...), sb.Memory) + return nil +} + // resolveProvider picks the provider for a new sandbox, erroring on // workspace-vs-repo or repo-vs-repo disagreement when the workspace itself // is silent. This matches the design: config should reject ambiguity. @@ -466,7 +535,7 @@ func resolveProvider(ws *template.Workspace) (config.Provider, error) { // defaultImage is the rootfs new sandboxes boot when neither --image // nor clawk.mod chooses one: clawk-dev — our own image bundling a -// development toolchain (Go/Node/Rust/etc. + claude/codex), built +// development toolchain (Go/Node/Rust/etc. + claude/codex/pi/opencode), built // and published by .github/workflows/publish-clawk-dev.yml. Override // per invocation with --image, or per repo/workspace with // `vm ( image ... )`. diff --git a/internal/cli/serial.go b/internal/cli/serial.go new file mode 100644 index 0000000..52a3c88 --- /dev/null +++ b/internal/cli/serial.go @@ -0,0 +1,368 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "runtime" + "strings" + "text/tabwriter" + "time" + + "github.com/clawkwork/clawk/internal/config" + "github.com/clawkwork/clawk/internal/serialfwd" + "github.com/clawkwork/clawk/internal/serialport" + "github.com/clawkwork/clawk/internal/template" + "github.com/clawkwork/clawk/internal/vzdctl" + "github.com/spf13/cobra" +) + +var serialListJSON bool + +func init() { + rootCmd.AddCommand(serialCmd) + serialCmd.AddCommand(serialAddCmd) + serialCmd.AddCommand(serialRemoveCmd) + serialCmd.AddCommand(serialListCmd) + serialListCmd.Flags().BoolVar(&serialListJSON, "json", false, "emit JSON") +} + +var serialCmd = &cobra.Command{ + Use: "serial", + Short: "Expose a host serial port inside a sandbox", + Long: `Present a serial port plugged into this machine — an Arduino, an +ESP32, a USB-TTL adapter — as a device inside the sandbox, so tooling in +there can flash and monitor it. + +The USB device itself is not passed through; nothing clawk runs on can do +that. What crosses is the serial stream and its line settings, which is all +avrdude, esptool and a serial monitor ever wanted. See docs/serial.md for +what that does and doesn't cover.`, +} + +var serialAddCmd = &cobra.Command{ + ValidArgsFunction: completeSandboxNames, + Use: "add [device-spec...]", + Short: "Forward a host serial port into the sandbox", + Long: `Device specs read host-side first, like the forward commands: + + /dev/cu.usbmodem1101 — same path inside the guest + /dev/cu.usbmodem1101:ttyACM0 — /dev/ttyACM0 inside the guest + '/dev/cu.usbmodem*:ttyACM0' — resolved at open time, not now + +The glob form is worth knowing about: a board that reboots into its +bootloader leaves /dev and comes back, often under a neighbouring name, and +a pattern survives that where a literal path doesn't. Quote it so the shell +doesn't expand it first. A pattern that matches two boards is refused rather +than guessed at. + +These apply to a running sandbox immediately — no down/up cycle. The port is +opened on the host only while a process in the guest holds the device open, +so the Arduino IDE on this machine can still have it the rest of the time. + +vz (macOS) only: firecracker's vsock is one-way, so the guest has no channel +to connect back through.`, + Args: cobra.MinimumNArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + sb, err := store.Load(args[0]) + if err != nil { + return err + } + // Every spec is parsed and checked before anything is printed or + // saved. The command is already all-or-nothing on the store — a + // later bad spec returns before Save — so reporting devices as + // "added" mid-loop announced work that then didn't persist. + var added []config.SerialDevice + for _, spec := range args[1:] { + dev, err := parseSerialSpec(spec) + if err != nil { + return err + } + if prev, dup := serialByGuestName(sb.Serials, dev.GuestName); dup { + if prev.HostPath == dev.HostPath { + fmt.Fprintf(cmd.OutOrStdout(), " (already forwarded: %s)\n", describeSerial(dev)) + continue + } + // One name, one device. A second spec claiming it would + // silently lose to the first inside the guest, so reject it + // here where both can be named. + return fmt.Errorf( + "guest device %q is already forwarded from %s — remove that first", + dev.GuestName, prev.HostPath) + } + // The same port under two names would let the guest open it + // twice, and the second open would fail in a way that points at + // the wrong thing. + if prev, dup := serialByHostPath(sb.Serials, dev.HostPath); dup { + return fmt.Errorf( + "%s is already forwarded as %q — remove that first", + dev.HostPath, prev.GuestName) + } + sb.Serials = append(sb.Serials, dev) + added = append(added, dev) + } + if len(added) == 0 { + return nil + } + if err := store.Save(sb); err != nil { + return err + } + for _, dev := range added { + fmt.Fprintf(cmd.OutOrStdout(), "Serial device added: %s\n", describeSerial(dev)) + for _, w := range warnAboutDevice(dev) { + fmt.Fprintf(cmd.OutOrStdout(), " note: %s\n", w) + } + } + applySerials(cmd, sb) + return nil + }, +} + +var serialRemoveCmd = &cobra.Command{ + ValidArgsFunction: completeSandboxNames, + Use: "remove [more...]", + Aliases: []string{"rm"}, + Short: "Stop forwarding a host serial port", + Long: `Accepts whatever identifies the device: the spec as added, the host +path on its own, or the guest name on its own.`, + Args: cobra.MinimumNArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + sb, err := store.Load(args[0]) + if err != nil { + return err + } + var kept, removed []config.SerialDevice + for _, d := range sb.Serials { + if serialMatchesAny(d, args[1:]) { + removed = append(removed, d) + continue + } + kept = append(kept, d) + } + // Nothing matched: say so and leave the record untouched rather than + // rewriting it identically. + if len(removed) == 0 { + fmt.Fprintf(cmd.OutOrStdout(), " (nothing matched: %s)\n", strings.Join(args[1:], ", ")) + return nil + } + sb.Serials = kept + if err := store.Save(sb); err != nil { + return err + } + for _, d := range removed { + fmt.Fprintf(cmd.OutOrStdout(), "Serial device removed: %s\n", describeSerial(d)) + } + applySerials(cmd, sb) + return nil + }, +} + +var serialListCmd = &cobra.Command{ + ValidArgsFunction: completeSandboxNames, + Use: "list ", + Aliases: []string{"ls"}, + Short: "List a sandbox's forwarded serial ports", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + sb, err := store.Load(args[0]) + if err != nil { + return err + } + if serialListJSON { + enc := json.NewEncoder(cmd.OutOrStdout()) + enc.SetIndent("", " ") + // Never null: a caller iterating the result shouldn't have to + // special-case a sandbox with no devices. + devices := sb.Serials + if devices == nil { + devices = []config.SerialDevice{} + } + return enc.Encode(devices) + } + if len(sb.Serials) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "No serial devices forwarded.") + return nil + } + w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "HOST\tGUEST\tPRESENT") + for _, d := range sb.Serials { + resolved, err := serialport.Resolve(d.HostPath) + present := "yes" + switch { + case err != nil: + present = "no" + case resolved != d.HostPath: + // A glob that currently resolves somewhere is worth showing + // resolved: "which board is this right now" is the question + // the column exists to answer. + present = resolved + } + fmt.Fprintf(w, "%s\t/dev/%s\t%s\n", d.HostPath, d.GuestName, present) + } + return w.Flush() + }, +} + +// parseSerialSpec parses HOST[:GUEST]. +// +// Split on the last colon rather than the first: a host path may contain +// one (rare but legal), while a guest name may not contain anything that +// isn't a bare filename, so the right-hand side is unambiguous. +func parseSerialSpec(spec string) (config.SerialDevice, error) { + hostPath, guestName := spec, "" + if i := strings.LastIndex(spec, ":"); i >= 0 { + hostPath, guestName = spec[:i], spec[i+1:] + } + + expanded, err := template.ExpandPath(hostPath) + if err != nil { + return config.SerialDevice{}, fmt.Errorf("device %q: %w", spec, err) + } + hostPath = expanded + + if hostPath == "" { + return config.SerialDevice{}, fmt.Errorf("device spec %q has no host path", spec) + } + if !filepath.IsAbs(hostPath) { + return config.SerialDevice{}, fmt.Errorf( + "device %q must be an absolute path (e.g. /dev/cu.usbmodem1101)", hostPath) + } + + if guestName == "" { + guestName = config.DefaultSerialGuestName(hostPath) + if guestName == "" { + // Only reachable for a glob, which has no basename to borrow. + return config.SerialDevice{}, fmt.Errorf( + "device pattern %q needs an explicit guest name (e.g. %s:ttyACM0)", + hostPath, hostPath) + } + } + if err := serialfwd.ValidDeviceName(guestName); err != nil { + return config.SerialDevice{}, err + } + return config.SerialDevice{HostPath: hostPath, GuestName: guestName}, nil +} + +// warnAboutDevice returns advisory notes for a device just added. None of +// these are errors: a board that is currently unplugged is a perfectly +// reasonable thing to configure, and the forward starts working when it +// appears. +func warnAboutDevice(dev config.SerialDevice) []string { + var notes []string + if _, err := serialport.Resolve(dev.HostPath); err != nil { + if errors.Is(err, serialport.ErrNoMatch) { + notes = append(notes, fmt.Sprintf( + "%s isn't there right now — it'll be picked up when it appears", dev.HostPath)) + } else { + notes = append(notes, err.Error()) + } + } + // The /dev/tty.* device on macOS blocks on carrier detect and is meant + // for dial-in; /dev/cu.* is the callout side and the one every serial + // tool uses. Getting this wrong looks like a port that opens and then + // does nothing. + if runtime.GOOS == "darwin" && strings.HasPrefix(dev.HostPath, "/dev/tty.") { + notes = append(notes, fmt.Sprintf( + "prefer the callout device /dev/cu.%s over /dev/tty.%[1]s", + strings.TrimPrefix(dev.HostPath, "/dev/tty."))) + } + return notes +} + +// applySerials pushes the just-saved device set into the running daemon, +// which relays it to the in-guest agent. Reports what happened but never +// fails the command: the store is already updated, so the worst case is +// that the edit lands on the next boot. Mirrors applyReverseForwards. +// +// Takes the sandbox rather than its name because ErrSerialUnsupported is +// ambiguous by construction (see vzdctl.ErrSerialUnsupported): it means +// either "this daemon predates serial forwarding" or "this backend has no +// host-side vsock listener". Only the record distinguishes them, and telling +// a firecracker user to restart the daemon sends them after a fix that can +// never work. +func applySerials(cmd *cobra.Command, sb *config.Sandbox) { + ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Second) + defer cancel() + err := vzdctl.NewClient(vzdctl.SocketPath(store.VMDir(sb.Name))).ReloadSerials(ctx) + switch { + case err == nil: + fmt.Fprintln(cmd.OutOrStdout(), "Applied to running sandbox.") + case errors.Is(err, vzdctl.ErrNotRunning): + if !serialSupported(sb) { + fmt.Fprintf(cmd.OutOrStdout(), + "Note: this sandbox uses the %s provider, which cannot forward "+ + "serial devices — the guest has no channel to dial back through. "+ + "The entry is recorded but will not appear in the sandbox.\n", + sb.Provider) + return + } + fmt.Fprintln(cmd.OutOrStdout(), "Sandbox not running — applies on next 'clawk up'.") + case errors.Is(err, vzdctl.ErrSerialUnsupported): + if !serialSupported(sb) { + fmt.Fprintf(cmd.OutOrStdout(), + "Note: this sandbox uses the %s provider, which cannot forward "+ + "serial devices — the guest has no channel to dial back through. "+ + "The entry is recorded but will not appear in the sandbox.\n", + sb.Provider) + return + } + fmt.Fprintln(cmd.OutOrStdout(), + "Sandbox is running an older daemon — restart it ('clawk down && clawk up') to apply.") + default: + fmt.Fprintf(cmd.ErrOrStderr(), + "clawk: live apply failed (%v) — applies on next 'clawk up'\n", err) + } +} + +// serialSupported reports whether sb's provider can forward serial devices +// at all. Only vz can: the guest is the end that dials (see +// internal/serialfwd), and firecracker's vsock carries host→guest only. +func serialSupported(sb *config.Sandbox) bool { + return sb.Provider.Normalize() == config.ProviderVZ +} + +// describeSerial spells out a device in the direction it is used, because +// the HOST:GUEST spec alone doesn't say which name belongs where. +func describeSerial(d config.SerialDevice) string { + return fmt.Sprintf("/dev/%s in the guest → %s", d.GuestName, d.HostPath) +} + +func serialByGuestName(devs []config.SerialDevice, name string) (config.SerialDevice, bool) { + for _, d := range devs { + if d.GuestName == name { + return d, true + } + } + return config.SerialDevice{}, false +} + +func serialByHostPath(devs []config.SerialDevice, path string) (config.SerialDevice, bool) { + for _, d := range devs { + if d.HostPath == path { + return d, true + } + } + return config.SerialDevice{}, false +} + +// serialMatchesAny reports whether d is named by any of the given +// identifiers — the spec it was added as, its host path, or its guest name. +func serialMatchesAny(d config.SerialDevice, identifiers []string) bool { + for _, id := range identifiers { + if id == d.String() || id == d.HostPath || id == d.GuestName { + return true + } + // The guest name is also accepted with the /dev/ prefix people + // naturally type when copying it out of an error message. + if strings.TrimPrefix(id, "/dev/") == d.GuestName && strings.HasPrefix(id, "/dev/") { + return true + } + // A spec whose host path needs expanding (~) won't match literally. + if dev, err := parseSerialSpec(id); err == nil && dev == d { + return true + } + } + return false +} diff --git a/internal/cli/serial_proxy.go b/internal/cli/serial_proxy.go new file mode 100644 index 0000000..66a7ce0 --- /dev/null +++ b/internal/cli/serial_proxy.go @@ -0,0 +1,425 @@ +package cli + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net" + "slices" + "sync" + "time" + + "github.com/clawkwork/clawk/internal/config" + "github.com/clawkwork/clawk/internal/serialfwd" + "github.com/clawkwork/clawk/internal/serialport" + "github.com/clawkwork/clawk/machine" +) + +// serialProxy is the host end of serial forwarding: it publishes the +// sandbox's serial devices to the in-guest agent and, for as long as a +// guest process holds the matching PTY open, bridges bytes to and from the +// physical port. See internal/serialfwd for the wire protocol and why the +// guest is the end that dials. +// +// It is a near-twin of reverseProxy — same lifecycle, same subscribe/ +// snapshot machinery, same reason for existing before the VM does. The +// bridging half is where they part company: a reverse forward hands off two +// sockets and copies until one ends, while an attachment owns a tty whose +// line configuration changes under it mid-stream. +type serialProxy struct { + logger *log.Logger + + ctx context.Context + cancel context.CancelFunc + listener net.Listener + wg sync.WaitGroup + + mu sync.Mutex + devices []config.SerialDevice + // subs are the live control connections' wakeup channels. Each is + // buffered by one and written non-blockingly, so a burst of edits + // collapses into a single resend of the (complete) current set. + subs map[chan struct{}]struct{} + // busy is the set of device names with an attachment in flight. A + // serial port takes exactly one reader: a second attach has to be + // refused rather than queued, or two guest processes would silently + // steal each other's bytes. + busy map[string]bool +} + +func newSerialProxy(logger *log.Logger) *serialProxy { + return &serialProxy{ + logger: logger, + subs: make(map[chan struct{}]struct{}), + busy: make(map[string]bool), + } +} + +// Set replaces the published device set and wakes every connected guest. +func (p *serialProxy) Set(devices []config.SerialDevice) { + p.mu.Lock() + p.devices = slices.Clone(devices) + for sub := range p.subs { + select { + case sub <- struct{}{}: + default: // already pending — it will read the latest set anyway + } + } + p.mu.Unlock() +} + +// Start begins accepting guest connections on serialfwd.VSockPort. It is a +// no-op (with one log line) on a backend that can't accept guest-initiated +// vsock connections, matching reverseProxy.Start. +func (p *serialProxy) Start(ctx context.Context, m machine.Machine) error { + listener, ok := m.(machine.VSockListener) + if !ok { + p.logger.Printf("serial: backend doesn't expose VSockListen; not starting") + return nil + } + pCtx, cancel := context.WithCancel(ctx) + l, err := listener.VSockListen(pCtx, serialfwd.VSockPort) + if err != nil { + cancel() + return fmt.Errorf("vsock listen port=%d: %w", serialfwd.VSockPort, err) + } + p.serve(pCtx, cancel, l) + + p.mu.Lock() + n := len(p.devices) + p.mu.Unlock() + p.logger.Printf("serial: listening on guest vsock port %d (%d device(s))", + serialfwd.VSockPort, n) + return nil +} + +// serve takes ownership of l and starts accepting. Split out from Start so +// the protocol can be exercised over any net.Listener — see reverseProxy. +func (p *serialProxy) serve(ctx context.Context, cancel context.CancelFunc, l net.Listener) { + p.ctx, p.cancel, p.listener = ctx, cancel, l + p.wg.Add(1) + go p.acceptLoop() +} + +// Stop closes the listener and waits for in-flight attachments. Idempotent, +// and safe on a proxy that was never started. +func (p *serialProxy) Stop() { + if p == nil || p.listener == nil { + return + } + p.cancel() + _ = p.listener.Close() + p.wg.Wait() +} + +func (p *serialProxy) acceptLoop() { + defer p.wg.Done() + for { + conn, err := p.listener.Accept() + if err != nil { + if errors.Is(err, net.ErrClosed) || p.ctx.Err() != nil { + return + } + p.logger.Printf("serial: accept: %v", err) + return + } + p.wg.Add(1) + go p.handle(conn) + } +} + +// handle reads the greeting and dispatches to the control stream or a +// device attachment. +func (p *serialProxy) handle(conn net.Conn) { + defer p.wg.Done() + defer conn.Close() + + // Shutdown has to reach an attachment that is simply idle — a serial + // monitor on a quiet board sends nothing for hours, and Stop waits on + // this goroutine. Closing the conn from the context unblocks it. + defer context.AfterFunc(p.ctx, func() { _ = conn.Close() })() + + r := serialfwd.NewReader(conn) + var g serialfwd.Greeting + if err := serialfwd.ReadLine(r, &g); err != nil { + if !errors.Is(err, io.EOF) { + p.logger.Printf("serial: reading greeting: %v", err) + } + return + } + if g.V != serialfwd.ProtoVersion { + // The guest agent is re-injected from the host's sources every cold + // boot, so this only happens across a resumed suspend state — where + // a down/up is exactly the fix. + p.logger.Printf("serial: guest speaks protocol v%d, host speaks v%d — "+ + "'clawk down && clawk up' to refresh the guest agent", g.V, serialfwd.ProtoVersion) + return + } + switch g.Op { + case serialfwd.OpControl: + p.serveControl(conn, r) + case serialfwd.OpAttach: + p.serveAttach(conn, r, g) + default: + p.logger.Printf("serial: unknown op %q", g.Op) + } +} + +// serveControl streams the device set to one guest until it disconnects or +// the daemon shuts down. The guest treats every snapshot as the complete +// desired state, so no delta bookkeeping is needed on either side. +func (p *serialProxy) serveControl(conn net.Conn, r *bufio.Reader) { + updates, unsubscribe := p.subscribe() + defer unsubscribe() + + // The guest never writes again on a control connection, so a read that + // returns is the guest going away. + gone := make(chan struct{}) + go func() { + defer close(gone) + _, _ = io.Copy(io.Discard, r) + }() + + for { + if err := serialfwd.WriteLine(conn, p.snapshot()); err != nil { + if p.ctx.Err() == nil && !errors.Is(err, net.ErrClosed) { + p.logger.Printf("serial: sending snapshot: %v", err) + } + return + } + select { + case <-updates: + case <-gone: + return + case <-p.ctx.Done(): + return + } + } +} + +// openRetryWindow bounds how long an attach waits for a device that isn't +// in /dev yet, and openRetryInterval is how often it looks. +// +// This window is not about patience with an unplugged board — it is the +// re-enumeration gap. A board told to enter its bootloader (the 1200-baud +// touch) drops off the USB bus and comes back a moment later, and the guest +// tool reopens the port straight away. Failing that reopen would break +// every upload to a native-USB board; waiting a couple of seconds makes it +// work. Past the window the attach fails and the guest retries on its own, +// so nothing is lost by keeping it short. +const ( + openRetryWindow = 3 * time.Second + openRetryInterval = 100 * time.Millisecond +) + +// serveAttach bridges one guest PTY to a physical serial port for as long as +// the guest holds it open. +func (p *serialProxy) serveAttach(conn net.Conn, r *bufio.Reader, g serialfwd.Greeting) { + dev, ok := p.lookup(g.Name) + if !ok { + // Reachable when an edit races an attach the guest already started; + // also the backstop if a guest ever names a device nobody + // configured. + p.logger.Printf("serial: refused attach to %q (not configured)", g.Name) + p.refuse(conn, fmt.Sprintf("no serial device named %q is forwarded", g.Name)) + return + } + if !p.claim(dev.GuestName) { + p.logger.Printf("serial: refused attach to %q (already attached)", dev.GuestName) + p.refuse(conn, fmt.Sprintf("serial device %q is already in use", dev.GuestName)) + return + } + defer p.release(dev.GuestName) + + mode := serialfwd.DefaultMode() + if g.Mode != nil { + mode = *g.Mode + } + + port, err := p.openWithRetry(dev.HostPath, mode) + if err != nil { + p.logger.Printf("serial: attach %s (%s): %v", dev.GuestName, dev.HostPath, err) + p.refuse(conn, err.Error()) + return + } + defer port.Close() + + if err := serialfwd.WriteLine(conn, serialfwd.AttachReply{OK: true}); err != nil { + return + } + p.logger.Printf("serial: %s attached to %s at %s", dev.GuestName, port.Path(), mode) + defer p.logger.Printf("serial: %s detached from %s", dev.GuestName, port.Path()) + + p.pump(conn, r, port, dev.GuestName) +} + +// pump moves bytes and mode changes between one guest attachment and one +// open port until either end stops. +// +// The two directions are deliberately asymmetric. Device → guest is only +// ever data, so it gets a plain goroutine that owns the connection's write +// side outright — no lock, because nothing else writes after the handshake. +// Guest → device is a frame loop, because a mode change has to be applied +// in the right place in the byte stream rather than whenever it happens to +// arrive. +func (p *serialProxy) pump(conn net.Conn, r *bufio.Reader, port *serialport.Port, name string) { + done := make(chan struct{}, 2) + + go func() { + defer func() { done <- struct{}{} }() + buf := make([]byte, 4096) + for { + n, err := port.Read(buf) + if n > 0 { + if werr := serialfwd.WriteFrame(conn, serialfwd.FrameData, buf[:n]); werr != nil { + return + } + } + if err != nil { + // Any read error ends the attachment: the board was + // unplugged, or Close raced us during teardown. Both mean + // this port is finished, and the guest reattaches if its + // client is still there. + if p.ctx.Err() == nil && !errors.Is(err, io.EOF) && !errors.Is(err, net.ErrClosed) { + p.logger.Printf("serial: %s: reading %s: %v", name, port.Path(), err) + } + return + } + } + }() + + go func() { + defer func() { done <- struct{}{} }() + for { + typ, payload, err := serialfwd.ReadFrame(r) + if err != nil { + if p.ctx.Err() == nil && !errors.Is(err, io.EOF) && !errors.Is(err, net.ErrClosed) { + p.logger.Printf("serial: %s: reading frame: %v", name, err) + } + return + } + switch typ { + case serialfwd.FrameData: + if _, err := port.Write(payload); err != nil { + p.logger.Printf("serial: %s: writing %s: %v", name, port.Path(), err) + return + } + case serialfwd.FrameMode: + var mode serialfwd.Mode + if err := json.Unmarshal(payload, &mode); err != nil { + p.logger.Printf("serial: %s: bad mode frame: %v", name, err) + continue + } + if err := port.Configure(mode); err != nil { + // Not fatal: a board that won't do 250000 baud is worth + // a log line, but tearing the attachment down would + // lose the session over a setting the tool may not even + // depend on. + p.logger.Printf("serial: %s: %v", name, err) + continue + } + p.logger.Printf("serial: %s reconfigured to %s", name, mode) + default: + p.logger.Printf("serial: %s: ignoring %s frame", name, typ) + } + } + }() + + // The first direction to finish ends the attachment. Closing the port + // unblocks the reader goroutine (its fd is registered with the Go + // poller); closing the conn unblocks the frame loop. Both happen in the + // deferred cleanup of serveAttach and handle respectively, so returning + // here is enough. + <-done +} + +// openWithRetry opens the device, tolerating a short absence — see +// openRetryWindow. +func (p *serialProxy) openWithRetry(pattern string, mode serialfwd.Mode) (*serialport.Port, error) { + deadline := time.Now().Add(openRetryWindow) + for { + port, err := serialport.Open(pattern, mode) + if err == nil { + return port, nil + } + // Only absence is retried. A device that exists but refuses to open + // — held by the Arduino IDE on the Mac, or a permissions problem — + // will refuse again in 100ms, and reporting it at once gives the + // user something to act on. + if !errors.Is(err, serialport.ErrNoMatch) || time.Now().After(deadline) { + return nil, err + } + select { + case <-time.After(openRetryInterval): + case <-p.ctx.Done(): + return nil, err + } + } +} + +// refuse sends a rejected AttachReply. Errors are dropped: the guest is +// about to see the connection close either way. +func (p *serialProxy) refuse(conn net.Conn, reason string) { + _ = serialfwd.WriteLine(conn, serialfwd.AttachReply{OK: false, Error: reason}) +} + +// snapshot renders the current set in wire form. Host paths are left out — +// the guest has no use for them. +func (p *serialProxy) snapshot() serialfwd.Snapshot { + p.mu.Lock() + defer p.mu.Unlock() + snap := serialfwd.Snapshot{Devices: make([]serialfwd.Device, 0, len(p.devices))} + for _, d := range p.devices { + snap.Devices = append(snap.Devices, serialfwd.Device{Name: d.GuestName}) + } + return snap +} + +// lookup resolves a guest-visible name to its configured device. The guest +// names a device and never a path, and an unlisted name is refused — +// otherwise any process in the sandbox could open anything in the Mac's +// /dev. +func (p *serialProxy) lookup(name string) (config.SerialDevice, bool) { + p.mu.Lock() + defer p.mu.Unlock() + for _, d := range p.devices { + if d.GuestName == name { + return d, true + } + } + return config.SerialDevice{}, false +} + +// claim reserves a device for one attachment, reporting false if another +// already holds it. +func (p *serialProxy) claim(name string) bool { + p.mu.Lock() + defer p.mu.Unlock() + if p.busy[name] { + return false + } + p.busy[name] = true + return true +} + +func (p *serialProxy) release(name string) { + p.mu.Lock() + delete(p.busy, name) + p.mu.Unlock() +} + +func (p *serialProxy) subscribe() (<-chan struct{}, func()) { + ch := make(chan struct{}, 1) + p.mu.Lock() + p.subs[ch] = struct{}{} + p.mu.Unlock() + return ch, func() { + p.mu.Lock() + delete(p.subs, ch) + p.mu.Unlock() + } +} diff --git a/internal/cli/serial_proxy_test.go b/internal/cli/serial_proxy_test.go new file mode 100644 index 0000000..2df0b5c --- /dev/null +++ b/internal/cli/serial_proxy_test.go @@ -0,0 +1,379 @@ +//go:build darwin || linux + +package cli + +import ( + "bufio" + "context" + "io" + "log" + "net" + "os" + "path/filepath" + "testing" + "time" + + "github.com/clawkwork/clawk/internal/config" + "github.com/clawkwork/clawk/internal/serialfwd" + "github.com/clawkwork/clawk/internal/serialport/serialporttest" + "github.com/stretchr/testify/require" +) + +// startTestSerialProxy runs a serialProxy over a loopback TCP listener +// instead of vsock, the same trick startTestProxy uses for reverse +// forwarding: the protocol is transport-agnostic, so everything below +// exercises the real accept/handshake/bridge path. +func startTestSerialProxy(t *testing.T, devices ...config.SerialDevice) (*serialProxy, string) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + + p := newSerialProxy(log.New(io.Discard, "", 0)) + p.Set(devices) + ctx, cancel := context.WithCancel(context.Background()) + p.serve(ctx, cancel, ln) + t.Cleanup(p.Stop) + return p, ln.Addr().String() +} + +// dialSerial opens a connection and sends one greeting, returning the +// connection plus the reader that must be used for everything after it. +func dialSerial(t *testing.T, addr string, g serialfwd.Greeting) (net.Conn, *bufio.Reader) { + t.Helper() + conn, err := net.Dial("tcp", addr) + require.NoError(t, err) + t.Cleanup(func() { conn.Close() }) + require.NoError(t, conn.SetDeadline(time.Now().Add(10*time.Second))) + require.NoError(t, serialfwd.WriteLine(conn, g)) + return conn, serialfwd.NewReader(conn) +} + +// fakeBoard is a PTY standing in for a plugged-in microcontroller: the +// proxy opens the slave as its serial port, and the test drives the master. +type fakeBoard struct { + master *os.File + path string +} + +func newFakeBoard(t *testing.T) fakeBoard { + t.Helper() + master, slavePath := serialporttest.OpenPTYPair(t) + return fakeBoard{master: master, path: slavePath} +} + +// device is the config entry pointing at this board. +func (b fakeBoard) device(guestName string) config.SerialDevice { + return config.SerialDevice{HostPath: b.path, GuestName: guestName} +} + +func TestSerialControlSendsSnapshotAndUpdates(t *testing.T) { + p, addr := startTestSerialProxy(t, + config.SerialDevice{HostPath: "/dev/cu.usbmodem1101", GuestName: "ttyACM0"}) + + _, r := dialSerial(t, addr, serialfwd.Greeting{Op: serialfwd.OpControl, V: serialfwd.ProtoVersion}) + + var snap serialfwd.Snapshot + require.NoError(t, serialfwd.ReadLine(r, &snap)) + require.Equal(t, []serialfwd.Device{{Name: "ttyACM0"}}, snap.Devices) + + // An edit to a running sandbox pushes a fresh full set — the guest + // reconciles rather than applying deltas. + p.Set([]config.SerialDevice{ + {HostPath: "/dev/cu.usbmodem1101", GuestName: "ttyACM0"}, + {HostPath: "/dev/cu.usbserial-A50285BI", GuestName: "ttyUSB0"}, + }) + require.NoError(t, serialfwd.ReadLine(r, &snap)) + require.Equal(t, []serialfwd.Device{{Name: "ttyACM0"}, {Name: "ttyUSB0"}}, snap.Devices) +} + +// The host path is the guest's business to never learn: it names devices, +// and the mapping to the Mac's /dev stays on the host. +func TestSerialSnapshotOmitsHostPaths(t *testing.T) { + _, addr := startTestSerialProxy(t, + config.SerialDevice{HostPath: "/dev/cu.usbmodem1101", GuestName: "ttyACM0"}) + + conn, r := dialSerial(t, addr, serialfwd.Greeting{Op: serialfwd.OpControl, V: serialfwd.ProtoVersion}) + _ = conn + + line, err := r.ReadString('\n') + require.NoError(t, err) + require.NotContains(t, line, "usbmodem") + require.Contains(t, line, "ttyACM0") +} + +func TestSerialAttachRefusesUnknownDevice(t *testing.T) { + _, addr := startTestSerialProxy(t, + config.SerialDevice{HostPath: "/dev/cu.usbmodem1101", GuestName: "ttyACM0"}) + + _, r := dialSerial(t, addr, serialfwd.Greeting{ + Op: serialfwd.OpAttach, V: serialfwd.ProtoVersion, Name: "ttyACM9", + }) + + var reply serialfwd.AttachReply + require.NoError(t, serialfwd.ReadLine(r, &reply)) + require.False(t, reply.OK) + require.Contains(t, reply.Error, "ttyACM9") +} + +// A guest speaking a version the host doesn't gets hung up on rather than +// half-served — the alternative is a stream neither end can parse. +func TestSerialRejectsProtocolMismatch(t *testing.T) { + _, addr := startTestSerialProxy(t) + + _, r := dialSerial(t, addr, serialfwd.Greeting{ + Op: serialfwd.OpControl, V: serialfwd.ProtoVersion + 1, + }) + + _, err := r.ReadString('\n') + require.Error(t, err, "host should close the connection") +} + +func TestSerialAttachBridgesBytesBothWays(t *testing.T) { + board := newFakeBoard(t) + _, addr := startTestSerialProxy(t, board.device("ttyACM0")) + + conn, r := dialSerial(t, addr, serialfwd.Greeting{ + Op: serialfwd.OpAttach, V: serialfwd.ProtoVersion, Name: "ttyACM0", + Mode: &serialfwd.Mode{Baud: 115200, Bits: 8, Parity: serialfwd.ParityNone, Stop: 1}, + }) + + var reply serialfwd.AttachReply + require.NoError(t, serialfwd.ReadLine(r, &reply)) + require.True(t, reply.OK, "attach refused: %s", reply.Error) + + // Guest → board. + require.NoError(t, serialfwd.WriteFrame(conn, serialfwd.FrameData, []byte("AT\r\n"))) + buf := make([]byte, 64) + require.NoError(t, board.master.SetDeadline(time.Now().Add(5*time.Second))) + n, err := board.master.Read(buf) + require.NoError(t, err) + require.Equal(t, "AT\r\n", string(buf[:n])) + + // Board → guest. + _, err = board.master.WriteString("OK\r\n") + require.NoError(t, err) + typ, payload, err := serialfwd.ReadFrame(r) + require.NoError(t, err) + require.Equal(t, serialfwd.FrameData, typ) + require.Equal(t, "OK\r\n", string(payload)) +} + +// The greeting's mode has to reach the hardware before the first byte does. +// A tool that opens at 115200 and immediately writes must not have that +// write go out at the port's previous rate. +func TestSerialAttachAppliesGreetingMode(t *testing.T) { + board := newFakeBoard(t) + _, addr := startTestSerialProxy(t, board.device("ttyACM0")) + + _, r := dialSerial(t, addr, serialfwd.Greeting{ + Op: serialfwd.OpAttach, V: serialfwd.ProtoVersion, Name: "ttyACM0", + Mode: &serialfwd.Mode{Baud: 115200, Bits: 8, Parity: serialfwd.ParityNone, Stop: 1}, + }) + + var reply serialfwd.AttachReply + require.NoError(t, serialfwd.ReadLine(r, &reply)) + require.True(t, reply.OK, "attach refused: %s", reply.Error) + + require.Equal(t, 115200, serialporttest.Speed(t, int(board.master.Fd()))) +} + +// The 1200-baud touch: a mid-stream mode change is how a native-USB board is +// told to reboot into its bootloader, so it has to actually reach the tty. +func TestSerialModeFrameReconfiguresPort(t *testing.T) { + board := newFakeBoard(t) + _, addr := startTestSerialProxy(t, board.device("ttyACM0")) + + conn, r := dialSerial(t, addr, serialfwd.Greeting{ + Op: serialfwd.OpAttach, V: serialfwd.ProtoVersion, Name: "ttyACM0", + Mode: &serialfwd.Mode{Baud: 115200, Bits: 8, Parity: serialfwd.ParityNone, Stop: 1}, + }) + var reply serialfwd.AttachReply + require.NoError(t, serialfwd.ReadLine(r, &reply)) + require.True(t, reply.OK, "attach refused: %s", reply.Error) + + require.NoError(t, serialfwd.WriteModeFrame(conn, serialfwd.Mode{ + Baud: 1200, Bits: 8, Parity: serialfwd.ParityNone, Stop: 1, + })) + + require.Eventually(t, func() bool { + return serialporttest.Speed(t, int(board.master.Fd())) == 1200 + }, 5*time.Second, 20*time.Millisecond, "mode frame never reached the port") +} + +// Ordering is the reason frames exist here rather than a raw byte stream +// with mode on the side. Data written before a mode change must be on the +// wire before the port is reconfigured. +func TestSerialDataBeforeModeChangeIsNotReordered(t *testing.T) { + board := newFakeBoard(t) + _, addr := startTestSerialProxy(t, board.device("ttyACM0")) + + conn, r := dialSerial(t, addr, serialfwd.Greeting{ + Op: serialfwd.OpAttach, V: serialfwd.ProtoVersion, Name: "ttyACM0", + Mode: &serialfwd.Mode{Baud: 115200, Bits: 8, Parity: serialfwd.ParityNone, Stop: 1}, + }) + var reply serialfwd.AttachReply + require.NoError(t, serialfwd.ReadLine(r, &reply)) + require.True(t, reply.OK) + + require.NoError(t, serialfwd.WriteFrame(conn, serialfwd.FrameData, []byte("before"))) + require.NoError(t, serialfwd.WriteModeFrame(conn, serialfwd.Mode{ + Baud: 1200, Bits: 8, Parity: serialfwd.ParityNone, Stop: 1, + })) + + buf := make([]byte, 64) + require.NoError(t, board.master.SetDeadline(time.Now().Add(5*time.Second))) + n, err := board.master.Read(buf) + require.NoError(t, err) + require.Equal(t, "before", string(buf[:n])) + + require.Eventually(t, func() bool { + return serialporttest.Speed(t, int(board.master.Fd())) == 1200 + }, 5*time.Second, 20*time.Millisecond) +} + +// One port, one reader. A second attach has to be refused rather than +// queued, or two guest processes silently steal each other's bytes. +func TestSerialSecondAttachIsRefused(t *testing.T) { + board := newFakeBoard(t) + _, addr := startTestSerialProxy(t, board.device("ttyACM0")) + + greeting := serialfwd.Greeting{ + Op: serialfwd.OpAttach, V: serialfwd.ProtoVersion, Name: "ttyACM0", + } + _, r1 := dialSerial(t, addr, greeting) + var first serialfwd.AttachReply + require.NoError(t, serialfwd.ReadLine(r1, &first)) + require.True(t, first.OK, "first attach refused: %s", first.Error) + + _, r2 := dialSerial(t, addr, greeting) + var second serialfwd.AttachReply + require.NoError(t, serialfwd.ReadLine(r2, &second)) + require.False(t, second.OK) + require.Contains(t, second.Error, "already in use") +} + +// Detaching has to release the claim, or a board is usable exactly once per +// boot. +func TestSerialReattachAfterDetach(t *testing.T) { + board := newFakeBoard(t) + _, addr := startTestSerialProxy(t, board.device("ttyACM0")) + + greeting := serialfwd.Greeting{ + Op: serialfwd.OpAttach, V: serialfwd.ProtoVersion, Name: "ttyACM0", + } + conn, r := dialSerial(t, addr, greeting) + var reply serialfwd.AttachReply + require.NoError(t, serialfwd.ReadLine(r, &reply)) + require.True(t, reply.OK) + require.NoError(t, conn.Close()) + + // The release happens as the handler unwinds, so give it a moment. + require.Eventually(t, func() bool { + c, rr := dialSerial(t, addr, greeting) + defer c.Close() + var again serialfwd.AttachReply + if err := serialfwd.ReadLine(rr, &again); err != nil { + return false + } + return again.OK + }, 5*time.Second, 50*time.Millisecond, "device never became reattachable") +} + +// A board that is mid-reset isn't there yet. The attach waits out the +// re-enumeration gap rather than failing, which is what makes an upload to +// a native-USB board work. +func TestSerialAttachWaitsForDeviceToAppear(t *testing.T) { + board := newFakeBoard(t) + + // A symlink under a temp dir stands in for the device node: it can be + // created after the fact, which a PTY path cannot. + dir := t.TempDir() + link := filepath.Join(dir, "cu.usbmodem1101") + + _, addr := startTestSerialProxy(t, + config.SerialDevice{HostPath: link, GuestName: "ttyACM0"}) + + appeared := make(chan struct{}) + go func() { + time.Sleep(300 * time.Millisecond) + _ = os.Symlink(board.path, link) + close(appeared) + }() + + _, r := dialSerial(t, addr, serialfwd.Greeting{ + Op: serialfwd.OpAttach, V: serialfwd.ProtoVersion, Name: "ttyACM0", + }) + + var reply serialfwd.AttachReply + require.NoError(t, serialfwd.ReadLine(r, &reply)) + <-appeared + require.True(t, reply.OK, "attach gave up before the device came back: %s", reply.Error) +} + +// Past the retry window the attach fails with something the user can act +// on, rather than hanging until the guest gives up. +func TestSerialAttachFailsWhenDeviceNeverAppears(t *testing.T) { + dir := t.TempDir() + _, addr := startTestSerialProxy(t, config.SerialDevice{ + HostPath: filepath.Join(dir, "cu.usbmodem1101"), GuestName: "ttyACM0", + }) + + start := time.Now() + _, r := dialSerial(t, addr, serialfwd.Greeting{ + Op: serialfwd.OpAttach, V: serialfwd.ProtoVersion, Name: "ttyACM0", + }) + + var reply serialfwd.AttachReply + require.NoError(t, serialfwd.ReadLine(r, &reply)) + require.False(t, reply.OK) + require.Contains(t, reply.Error, "no device matches") + require.GreaterOrEqual(t, time.Since(start), openRetryWindow, + "should have used the whole retry window before giving up") +} + +// A glob that matches two boards must refuse rather than guess — flashing +// the wrong device is the worst outcome available here. And because it is +// not an absence, it must not burn the retry window first. +func TestSerialAttachRefusesAmbiguousGlob(t *testing.T) { + dir := t.TempDir() + for _, n := range []string{"cu.usbmodem1101", "cu.usbmodem2201"} { + require.NoError(t, os.WriteFile(filepath.Join(dir, n), nil, 0o600)) + } + _, addr := startTestSerialProxy(t, config.SerialDevice{ + HostPath: filepath.Join(dir, "cu.usbmodem*"), GuestName: "ttyACM0", + }) + + start := time.Now() + _, r := dialSerial(t, addr, serialfwd.Greeting{ + Op: serialfwd.OpAttach, V: serialfwd.ProtoVersion, Name: "ttyACM0", + }) + + var reply serialfwd.AttachReply + require.NoError(t, serialfwd.ReadLine(r, &reply)) + require.False(t, reply.OK) + require.Contains(t, reply.Error, "matches 2 devices") + require.Less(t, time.Since(start), openRetryWindow, "should have failed immediately") +} + +// Stop has to reach an attachment parked on a silent board, or the daemon +// hangs on shutdown waiting for a byte that never comes. +func TestSerialStopUnblocksIdleAttachment(t *testing.T) { + board := newFakeBoard(t) + p, addr := startTestSerialProxy(t, board.device("ttyACM0")) + + _, r := dialSerial(t, addr, serialfwd.Greeting{ + Op: serialfwd.OpAttach, V: serialfwd.ProtoVersion, Name: "ttyACM0", + }) + var reply serialfwd.AttachReply + require.NoError(t, serialfwd.ReadLine(r, &reply)) + require.True(t, reply.OK) + + stopped := make(chan struct{}) + go func() { p.Stop(); close(stopped) }() + + select { + case <-stopped: + case <-time.After(5 * time.Second): + t.Fatal("Stop hung on an idle attachment") + } +} diff --git a/internal/cli/serial_test.go b/internal/cli/serial_test.go new file mode 100644 index 0000000..ccb8fd5 --- /dev/null +++ b/internal/cli/serial_test.go @@ -0,0 +1,283 @@ +package cli + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/clawkwork/clawk/internal/config" + "github.com/stretchr/testify/require" +) + +func TestSerialAdd(t *testing.T) { + s, _ := setupTest(t) + require.NoError(t, s.Save(&config.Sandbox{ + Name: "ard", Provider: config.ProviderVZ, VMState: config.VMStateStopped, + })) + + out, err := executeCommand("serial", "add", "ard", + "/dev/cu.usbmodem1101", "/dev/cu.usbserial-A50285BI:ttyUSB0") + require.NoError(t, err) + // Which name belongs on which side is the confusion the wording exists + // to prevent, so it has to show up in the output. + require.Contains(t, out, "/dev/cu.usbmodem1101 in the guest → /dev/cu.usbmodem1101") + require.Contains(t, out, "/dev/ttyUSB0 in the guest → /dev/cu.usbserial-A50285BI") + require.Contains(t, out, "applies on next 'clawk up'") + + sb, err := s.Load("ard") + require.NoError(t, err) + require.Equal(t, []config.SerialDevice{ + {HostPath: "/dev/cu.usbmodem1101", GuestName: "cu.usbmodem1101"}, + {HostPath: "/dev/cu.usbserial-A50285BI", GuestName: "ttyUSB0"}, + }, sb.Serials) +} + +// A board that isn't plugged in right now is a perfectly reasonable thing +// to configure — it must be a note, not a failure. +func TestSerialAddAbsentDeviceIsANoteNotAnError(t *testing.T) { + s, _ := setupTest(t) + require.NoError(t, s.Save(&config.Sandbox{ + Name: "ard", Provider: config.ProviderVZ, VMState: config.VMStateStopped, + })) + + out, err := executeCommand("serial", "add", "ard", "/dev/cu.definitelyNotThere") + require.NoError(t, err) + require.Contains(t, out, "isn't there right now") + + sb, err := s.Load("ard") + require.NoError(t, err) + require.Len(t, sb.Serials, 1) +} + +func TestSerialAddIsIdempotent(t *testing.T) { + s, _ := setupTest(t) + require.NoError(t, s.Save(&config.Sandbox{ + Name: "ard", Provider: config.ProviderVZ, VMState: config.VMStateStopped, + })) + + _, err := executeCommand("serial", "add", "ard", "/dev/cu.usbmodem1101") + require.NoError(t, err) + out, err := executeCommand("serial", "add", "ard", "/dev/cu.usbmodem1101") + require.NoError(t, err) + require.Contains(t, out, "already forwarded") + + sb, err := s.Load("ard") + require.NoError(t, err) + require.Len(t, sb.Serials, 1) +} + +// One guest name can only be one device — the guest creates /dev/ +// once, and a second claim would silently lose in there. +func TestSerialAddRejectsGuestNameClash(t *testing.T) { + s, _ := setupTest(t) + require.NoError(t, s.Save(&config.Sandbox{ + Name: "ard", Provider: config.ProviderVZ, VMState: config.VMStateStopped, + })) + + _, err := executeCommand("serial", "add", "ard", "/dev/cu.usbmodem1101:ttyACM0") + require.NoError(t, err) + _, err = executeCommand("serial", "add", "ard", "/dev/cu.usbmodem2201:ttyACM0") + require.Error(t, err) + require.Contains(t, err.Error(), "already forwarded from /dev/cu.usbmodem1101") + + sb, err := s.Load("ard") + require.NoError(t, err) + require.Len(t, sb.Serials, 1) +} + +// The same port under two names would let the guest hold it twice, and the +// second open fails pointing at the wrong thing. +func TestSerialAddRejectsDuplicateHostPath(t *testing.T) { + s, _ := setupTest(t) + require.NoError(t, s.Save(&config.Sandbox{ + Name: "ard", Provider: config.ProviderVZ, VMState: config.VMStateStopped, + })) + + _, err := executeCommand("serial", "add", "ard", "/dev/cu.usbmodem1101:ttyACM0") + require.NoError(t, err) + _, err = executeCommand("serial", "add", "ard", "/dev/cu.usbmodem1101:ttyACM1") + require.Error(t, err) + require.Contains(t, err.Error(), `already forwarded as "ttyACM0"`) +} + +func TestSerialRemove(t *testing.T) { + s, _ := setupTest(t) + require.NoError(t, s.Save(&config.Sandbox{ + Name: "ard", Provider: config.ProviderVZ, VMState: config.VMStateStopped, + Serials: []config.SerialDevice{ + {HostPath: "/dev/cu.usbmodem1101", GuestName: "ttyACM0"}, + {HostPath: "/dev/cu.usbserial-A50285BI", GuestName: "ttyUSB0"}, + }, + })) + + // Removal by guest name, which is what an error message inside the + // guest would have shown the user. + out, err := executeCommand("serial", "remove", "ard", "ttyACM0") + require.NoError(t, err) + require.Contains(t, out, "Serial device removed") + + sb, err := s.Load("ard") + require.NoError(t, err) + require.Equal(t, []config.SerialDevice{ + {HostPath: "/dev/cu.usbserial-A50285BI", GuestName: "ttyUSB0"}, + }, sb.Serials) +} + +func TestSerialRemoveAcceptsEveryIdentifier(t *testing.T) { + for _, id := range []string{ + "/dev/cu.usbmodem1101", // host path + "ttyACM0", // guest name + "/dev/ttyACM0", // guest name as it appears in the VM + "/dev/cu.usbmodem1101:ttyACM0", // the spec as added + } { + t.Run(id, func(t *testing.T) { + s, _ := setupTest(t) + require.NoError(t, s.Save(&config.Sandbox{ + Name: "ard", Provider: config.ProviderVZ, VMState: config.VMStateStopped, + Serials: []config.SerialDevice{ + {HostPath: "/dev/cu.usbmodem1101", GuestName: "ttyACM0"}, + }, + })) + + _, err := executeCommand("serial", "remove", "ard", id) + require.NoError(t, err) + + sb, err := s.Load("ard") + require.NoError(t, err) + require.Empty(t, sb.Serials) + }) + } +} + +func TestSerialRemoveUnknownSaysSo(t *testing.T) { + s, _ := setupTest(t) + require.NoError(t, s.Save(&config.Sandbox{ + Name: "ard", Provider: config.ProviderVZ, VMState: config.VMStateStopped, + Serials: []config.SerialDevice{ + {HostPath: "/dev/cu.usbmodem1101", GuestName: "ttyACM0"}, + }, + })) + + out, err := executeCommand("serial", "remove", "ard", "ttyNOPE") + require.NoError(t, err) + require.Contains(t, out, "nothing matched") + + sb, err := s.Load("ard") + require.NoError(t, err) + require.Len(t, sb.Serials, 1) +} + +func TestSerialListJSONIsNeverNull(t *testing.T) { + s, _ := setupTest(t) + require.NoError(t, s.Save(&config.Sandbox{ + Name: "ard", Provider: config.ProviderVZ, VMState: config.VMStateStopped, + })) + + out, err := executeCommand("serial", "list", "ard", "--json") + require.NoError(t, err) + var devices []config.SerialDevice + require.NoError(t, json.Unmarshal([]byte(out), &devices)) + require.NotNil(t, devices) + require.Empty(t, devices) +} + +func TestSerialListShowsPresence(t *testing.T) { + dir := t.TempDir() + present := filepath.Join(dir, "cu.present") + require.NoError(t, os.WriteFile(present, nil, 0o600)) + + s, _ := setupTest(t) + require.NoError(t, s.Save(&config.Sandbox{ + Name: "ard", Provider: config.ProviderVZ, VMState: config.VMStateStopped, + Serials: []config.SerialDevice{ + {HostPath: present, GuestName: "ttyACM0"}, + {HostPath: filepath.Join(dir, "cu.absent"), GuestName: "ttyACM1"}, + }, + })) + + out, err := executeCommand("serial", "list", "ard") + require.NoError(t, err) + require.Regexp(t, `cu\.present.*/dev/ttyACM0\s+yes`, out) + require.Regexp(t, `cu\.absent.*/dev/ttyACM1\s+no`, out) +} + +// A glob resolves to whichever board is on the bus right now, and that is +// the question the PRESENT column exists to answer. +func TestSerialListResolvesGlob(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "cu.usbmodem14201"), nil, 0o600)) + + s, _ := setupTest(t) + require.NoError(t, s.Save(&config.Sandbox{ + Name: "ard", Provider: config.ProviderVZ, VMState: config.VMStateStopped, + Serials: []config.SerialDevice{ + {HostPath: filepath.Join(dir, "cu.usbmodem*"), GuestName: "ttyACM0"}, + }, + })) + + out, err := executeCommand("serial", "list", "ard") + require.NoError(t, err) + require.Contains(t, out, "cu.usbmodem14201") +} + +func TestParseSerialSpec(t *testing.T) { + tests := []struct { + name string + spec string + want config.SerialDevice + }{ + { + name: "bare path keeps its basename in the guest", + spec: "/dev/cu.usbmodem1101", + want: config.SerialDevice{HostPath: "/dev/cu.usbmodem1101", GuestName: "cu.usbmodem1101"}, + }, + { + name: "explicit guest name", + spec: "/dev/cu.usbmodem1101:ttyACM0", + want: config.SerialDevice{HostPath: "/dev/cu.usbmodem1101", GuestName: "ttyACM0"}, + }, + { + name: "glob with explicit name", + spec: "/dev/cu.usbmodem*:ttyACM0", + want: config.SerialDevice{HostPath: "/dev/cu.usbmodem*", GuestName: "ttyACM0"}, + }, + { + name: "trailing colon falls back to the default name", + spec: "/dev/ttyACM0:", + want: config.SerialDevice{HostPath: "/dev/ttyACM0", GuestName: "ttyACM0"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseSerialSpec(tt.spec) + require.NoError(t, err) + require.Equal(t, tt.want, got) + }) + } +} + +func TestParseSerialSpecRejects(t *testing.T) { + tests := []struct { + name string + spec string + contains string + }{ + {"relative path", "cu.usbmodem1101", "absolute path"}, + {"empty", "", "no host path"}, + {"glob without a name", "/dev/cu.usbmodem*", "needs an explicit guest name"}, + // The guest name becomes /dev/, so anything with a path in it + // has to be refused here rather than inside the VM. + {"guest name with a slash", "/dev/ttyACM0:sub/dir", "path separator"}, + {"guest name escaping /dev", "/dev/ttyACM0:../../etc/passwd", "path separator"}, + {"guest name starting with a dot", "/dev/ttyACM0:.hidden", "must not start with a dot"}, + {"guest name with a space", "/dev/ttyACM0:tty ACM0", "contains"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := parseSerialSpec(tt.spec) + require.Error(t, err) + require.Contains(t, err.Error(), tt.contains) + }) + } +} diff --git a/internal/cli/sessions.go b/internal/cli/sessions.go index f846efd..7b7efb4 100644 --- a/internal/cli/sessions.go +++ b/internal/cli/sessions.go @@ -31,7 +31,7 @@ func sessionHistoryEnabled(sb *config.Sandbox) bool { } // sessionClaudeDir is the host working tree mounted into the guest as -// ~/.claude — the same path PersistentClaudeShares serves. +// ~/.claude — the same path PersistentAgentShares serves. func sessionClaudeDir(sb *config.Sandbox) string { return filepath.Join(store.StateDir(sb.Name), "claude") } diff --git a/internal/cli/setup.go b/internal/cli/setup.go index 0f4237c..7e8d30c 100644 --- a/internal/cli/setup.go +++ b/internal/cli/setup.go @@ -6,6 +6,7 @@ import ( "path/filepath" "strings" + "github.com/clawkwork/clawk/internal/template" "github.com/spf13/cobra" ) @@ -133,6 +134,9 @@ func buildPrereqChecks(root string) []prereqCheck { // have been parsed." func init() { rootCmd.PersistentPreRunE = func(cmd *cobra.Command, _ []string) error { + // --no-global has to land before the first template load, and every + // load happens inside a command's RunE. + template.GlobalDisabled = noGlobalFlag // Walk up the command tree to the top-level child of root // (e.g. `clawk image build` → "image"). The skip list is // keyed on the top-level verb because that's the granularity diff --git a/internal/cli/shares.go b/internal/cli/shares.go index 02bdd2f..b58bd3e 100644 --- a/internal/cli/shares.go +++ b/internal/cli/shares.go @@ -69,14 +69,17 @@ func collectSandboxShares(sb *config.Sandbox) []machine.Share { } } - // Per-sandbox persistent state — whole ~/.claude/, pre-seeded with - // settings.json/CLAUDE.md/.credentials.json by SeedClaudeStateDir - // (called from the provider Create path). Host dir lives outside - // VMDir so destroy cannot touch it; the same sandbox name always - // sees the same state dir across recreate cycles. MUST come before - // DefaultHostShares so the agents/commands sub-mounts land on top - // of the parent rather than being shadowed under it. - for _, sh := range sandbox.PersistentClaudeShares(store.StateDir(sb.Name)) { + // Per-sandbox persistent state — one whole home dir per coding-agent + // runner (~/.claude, ~/.codex, ~/.pi). The vz rootfs is re-cloned from + // the image on every boot, so these mounts are the ONLY thing keeping a + // runner's sessions and login across `clawk down && clawk up`. Claude's + // is additionally pre-seeded with settings.json/CLAUDE.md/.credentials.json + // by SeedClaudeStateDir (called from the provider Create path). Host dirs + // live outside VMDir so destroy cannot touch them; the same sandbox name + // always sees the same state dir across recreate cycles. MUST come before + // DefaultHostShares so the agents/commands/skills sub-mounts land on top + // of their parents rather than being shadowed under them. + for _, sh := range sandbox.PersistentAgentShares(store.StateDir(sb.Name)) { out = append(out, machine.Share{ HostPath: sh.HostPath, Tag: sh.Tag, diff --git a/internal/cli/shares_test.go b/internal/cli/shares_test.go index e29b72b..edd7a70 100644 --- a/internal/cli/shares_test.go +++ b/internal/cli/shares_test.go @@ -1,6 +1,8 @@ package cli import ( + "os" + "path/filepath" "testing" "github.com/clawkwork/clawk/internal/config" @@ -77,3 +79,78 @@ func TestCollectSandboxShares_InPlaceOnly(t *testing.T) { "in-place-only sandbox needs no consolidated parent") require.Contains(t, byTag, "here") } + +// TestCollectSandboxShares_AgentStateDevices pins the host-device side of the +// per-runner state mounts. Every entry in sandbox.AgentStateDirs must get a +// virtio-fs device here, or the guest manifest asks clawk-init to mount a tag +// vz never exposed — the runner then writes to the rootfs, which vz re-clones +// from the image on every boot, and its history disappears on `clawk up`. +func TestCollectSandboxShares_AgentStateDevices(t *testing.T) { + withTempStore(t) + + sb := &config.Sandbox{Name: "box"} + byTag := map[string]machine.Share{} + for _, s := range collectSandboxShares(sb) { + byTag[s.Tag] = s + } + + stateRoot := store.StateDir("box") + for _, d := range sandbox.AgentStateDirs { + sh, ok := byTag[d.Tag] + require.Truef(t, ok, "%s state device (tag %s) missing", d.Agent, d.Tag) + require.Equal(t, filepath.Join(stateRoot, d.Sub), sh.HostPath, + "%s state must come from the sandbox's host state dir, which survives destroy", d.Agent) + require.Falsef(t, sh.ReadOnly, "%s state must be writable", d.Agent) + } +} + +// TestCollectSandboxShares_MatchesGuestManifest is the lock-step check the +// comment on collectSandboxShares demands: the vz device list and the guest +// mount manifest are built by two separate functions, and a tag in one but +// not the other is either a mount of a device that doesn't exist or a device +// nothing ever mounts. Both failures are silent at boot. +func TestCollectSandboxShares_MatchesGuestManifest(t *testing.T) { + withTempStore(t) + + home := t.TempDir() + t.Setenv("HOME", home) + require.NoError(t, os.MkdirAll(filepath.Join(home, ".claude", "agents"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(home, ".codex", "skills"), 0o755)) + + sb := &config.Sandbox{ + Name: "box", + Image: "golang:1.25", + Phases: []config.Phase{ + {Worktree: filepath.Join(store.WorktreeDir("box"), "proj"), Repo: "/code/proj"}, + {Worktree: "/code/here", Repo: "/code/here", InPlace: true}, + }, + Shares: []config.HostShare{ + {HostPath: "/Users/u/.aws", GuestPath: sandbox.GuestHome + "/.aws", ReadOnly: true}, + }, + } + + deviceTags := map[string]bool{} + for _, s := range collectSandboxShares(sb) { + deviceTags[s.Tag] = true + } + + m, err := sandbox.OCIGuestManifest(sb, store.StateDir("box"), store.CacheDir(), store.RootDir()) + require.NoError(t, err) + mountTags := map[string]bool{} + for _, mt := range m.Mounts { + if mt.Block != "" { + continue // block devices carry no virtio-fs tag + } + mountTags[mt.Tag] = true + } + + for tag := range mountTags { + require.Containsf(t, deviceTags, tag, + "guest mounts tag %q but no virtio-fs device exposes it", tag) + } + for tag := range deviceTags { + require.Containsf(t, mountTags, tag, + "virtio-fs device %q is exposed but the guest never mounts it", tag) + } + require.Contains(t, mountTags, "codex_home", "test set up no agent state mounts") +} diff --git a/internal/cli/shell.go b/internal/cli/shell.go index 4c8add8..12ffd1b 100644 --- a/internal/cli/shell.go +++ b/internal/cli/shell.go @@ -89,7 +89,7 @@ func tryVSockShell(sb *config.Sandbox, provider sandbox.Provider) error { Args: []string{"-l"}, Cwd: agentStartDir(provider, sb), User: sandbox.GuestUser, - Env: buildVSockEnv(), + Env: buildVSockEnv(sb), // No ClearScreen: a login shell is line-oriented, so keep the // user's scrollback — the shell should read as a continuation of // the terminal it launched from, not a wiped screen. diff --git a/internal/cli/status.go b/internal/cli/status.go index 0f2867f..649e457 100644 --- a/internal/cli/status.go +++ b/internal/cli/status.go @@ -67,10 +67,16 @@ type statusJSONOutput struct { // v2 additive blocks. Forwards []statusJSONForward `json:"forwards,omitempty"` ReverseForwards []statusJSONForward `json:"reverse_forwards,omitempty"` + Serials []statusJSONSerial `json:"serials,omitempty"` Network *statusJSONNetwork `json:"network,omitempty"` Setup []statusJSONSetup `json:"setup,omitempty"` } +type statusJSONSerial struct { + HostPath string `json:"host_path"` + GuestName string `json:"guest_name"` +} + type statusJSONBranch struct { Index int `json:"index"` Repo string `json:"repo"` @@ -191,6 +197,11 @@ func renderStatusJSON(w io.Writer, sb *config.Sandbox, liveStatus string) error HostPort: f.HostPort, GuestPort: f.GuestPort, }) } + for _, d := range sb.Serials { + out.Serials = append(out.Serials, statusJSONSerial{ + HostPath: d.HostPath, GuestName: d.GuestName, + }) + } out.Network = &statusJSONNetwork{ Use: effectiveUseForLog(sb), Blocks: sb.Network.Blocks, @@ -291,6 +302,15 @@ func renderStatusDashboard(w io.Writer, provider sandbox.Provider, sb *config.Sa } fmt.Fprintf(w, " Reverse %s (guest → host loopback)\n", strings.Join(parts, ", ")) } + // Same direction convention as the rows above: what the guest sees on + // the left, what it reaches on the right. + if len(sb.Serials) > 0 { + parts := make([]string, 0, len(sb.Serials)) + for _, d := range sb.Serials { + parts = append(parts, fmt.Sprintf("/dev/%s → %s", d.GuestName, d.HostPath)) + } + fmt.Fprintf(w, " Serial %s\n", strings.Join(parts, ", ")) + } // One line per policy layer, lowest precedence first: the use chain, // then the sandbox's own blocks with entry counts. Full contents live diff --git a/internal/cli/swap_test.go b/internal/cli/swap_test.go new file mode 100644 index 0000000..d4e4b3f --- /dev/null +++ b/internal/cli/swap_test.go @@ -0,0 +1,37 @@ +package cli + +import ( + "testing" + + "github.com/clawkwork/clawk/internal/template" + "github.com/stretchr/testify/require" +) + +// TestResolveSwap covers the workspace-merge rule: the max size across the +// workspace file and every repo Clawkfile, with zero as "unset" — except +// that an explicit "off" (negative) beats any size, the same asymmetry +// resolveIdleTimeout has and for the same reason. +func TestResolveSwap(t *testing.T) { + ws := func(fileMiB int64, repoMiB ...int64) *template.Workspace { + w := &template.Workspace{File: &template.Template{SwapMiB: fileMiB}} + for _, m := range repoMiB { + w.Repos = append(w.Repos, template.Repo{ + Clawkfile: &template.Template{SwapMiB: m}, + }) + } + return w + } + + require.Equal(t, int64(0), resolveSwap(ws(0)), "unset everywhere stays unset") + require.Equal(t, int64(8192), resolveSwap(ws(8192)), "workspace value") + require.Equal(t, int64(16384), resolveSwap(ws(8192, 16384)), "larger repo wins") + require.Equal(t, int64(8192), resolveSwap(ws(8192, 4096)), "smaller repo ignored") + require.Equal(t, int64(-1), resolveSwap(ws(8192, -1)), "a repo's 'off' wins over a size") + require.Equal(t, int64(-1), resolveSwap(ws(-1, 8192)), "the workspace's 'off' wins too") + require.Equal(t, int64(-1), resolveSwap(ws(0, 4096, -1, 16384)), "off wins wherever it appears") + + // A repo with no Clawkfile must not panic or count. + w := ws(2048) + w.Repos = append(w.Repos, template.Repo{Clawkfile: nil}) + require.Equal(t, int64(2048), resolveSwap(w)) +} diff --git a/internal/cli/up.go b/internal/cli/up.go index 7a7f699..84dd89c 100644 --- a/internal/cli/up.go +++ b/internal/cli/up.go @@ -215,6 +215,21 @@ func ensureRuntimeMounts(provider sandbox.Provider, sb *config.Sandbox) error { } } } + // Per-sandbox agent state (~/.claude, ~/.codex, ~/.pi). Normally the + // boot manifest has these mounted already and every call here is a + // `mountpoint -q` no-op — but clawk-init only LOGS a failed host-share + // mount and carries on booting, and a runner whose home didn't mount + // writes to the rootfs vz re-clones next boot, i.e. loses its history + // with no error anywhere the user looks. Cheap retry, expensive miss. + // + // Mounted BEFORE DefaultHostShares: its capability dirs are sub-mounts + // inside these homes (~/.claude/agents, ~/.codex/skills), and a parent + // mounted afterwards would shadow them. + for _, sh := range sandbox.PersistentAgentShares(store.StateDir(sb.Name)) { + if err := mountIfMissing(sp, sb, sh.Tag, sh.GuestPath); err != nil { + return fmt.Errorf("%s: %w", sh.GuestPath, err) + } + } for _, sh := range sandbox.DefaultHostShares() { if err := mountIfMissing(sp, sb, sh.Tag, sh.GuestPath); err != nil { return fmt.Errorf("%s: %w", sh.GuestPath, err) diff --git a/internal/cli/vshell.go b/internal/cli/vshell.go index 72760c7..4f6675f 100644 --- a/internal/cli/vshell.go +++ b/internal/cli/vshell.go @@ -68,7 +68,7 @@ returns an error rather than falling back.`, Cmd: cmdPath, Args: cmdArgs, User: sandbox.GuestUser, - Env: buildVSockEnv(), + Env: buildVSockEnv(sb), } code, err := vsockclient.Run(context.Background(), cfg) if err != nil { diff --git a/internal/cli/vzd.go b/internal/cli/vzd.go index f1ddc53..e8c5aed 100644 --- a/internal/cli/vzd.go +++ b/internal/cli/vzd.go @@ -105,12 +105,20 @@ func runVzd(_ *cobra.Command, args []string) (retErr error) { rev := newReverseProxy(logger) rev.Set(sb.ReverseForwards) + // Serial proxy: publishes the sandbox's forwarded serial ports to the + // in-guest agent and bridges each PTY the guest opens through to the + // physical device. Built alongside the reverse-forward proxy and for + // the same reason — `clawk serial add` needs somewhere to push while + // the VM is still booting. + ser := newSerialProxy(logger) + ser.Set(sb.Serials) + // Control socket: lets the CLI push network-policy edits into the live // allow list (`clawk network allow` without a down/up cycle), read // the denial ledger (`clawk network denials`), push reverse-forward // edits, and drive the VM lifecycle (`clawk pause/resume/snapshot`). // Best-effort — without it, policy edits apply on the next up, as before. - ctl, err := vzdctl.Start(vzdctl.SocketPath(vmDir), controlHandlers(sb, allow, lc, rev, logger)) + ctl, err := vzdctl.Start(vzdctl.SocketPath(vmDir), controlHandlers(sb, allow, lc, rev, ser, logger)) if err != nil { logger.Printf("control socket: disabled (%v) — network edits apply on next up", err) } else { @@ -215,6 +223,16 @@ func runVzd(_ *cobra.Command, args []string) (retErr error) { defer rev.Stop() } + // Serial devices: same shape as the reverse forwards above — the guest + // agent dials this listener to learn which PTYs to create, and again + // each time a process in the sandbox opens one. Best-effort; a failure + // here means those devices simply don't appear inside the guest. + if err := ser.Start(ctx, m); err != nil { + logger.Printf("serial: disabled (%v)", err) + } else { + defer ser.Stop() + } + // 9p cache servers: one per toolchain cache, each serving its host cache // dir over 9p on the guest vsock port from ToolchainCacheShares. A // 9p-capable clawk-init mounts these instead of the caches' virtio-fs @@ -337,14 +355,25 @@ func buildOCISandboxSpec(sb *config.Sandbox, vmDir string, allow *netfilter.Allo return machine.Spec{}, fmt.Errorf("guest config disk: %w", err) } + swapPath, err := sandbox.EnsureSwapDisk(vmDir, sandbox.SwapDiskMiB(sb)) + if err != nil { + return machine.Spec{}, fmt.Errorf("swap disk: %w", err) + } + spec := buildSandboxSpec(sb, vmDir, allow) spec.Boot = machine.DirectKernel{ Vmlinux: vmlinux, Cmdline: sandbox.OCICmdline, } spec.RootFS = sandbox.OCIRootFS(sb, store.CacheDir(), bins) + // Order is the guest's device order: vdb=guestcfg, vdc=swap + // (sandbox.OCISwapDevice). The manifest names that device by path, so + // anything added here must go after it. spec.Disks = []machine.Disk{ {Path: filepath.Join(vmDir, sandbox.OCIConfigDiskName), ReadOnly: true}, } + if swapPath != "" { + spec.Disks = append(spec.Disks, machine.Disk{Path: swapPath}) + } return spec, nil } diff --git a/internal/cli/workspace_defaults_test.go b/internal/cli/workspace_defaults_test.go new file mode 100644 index 0000000..6185a8f --- /dev/null +++ b/internal/cli/workspace_defaults_test.go @@ -0,0 +1,56 @@ +package cli + +import ( + "os" + "path/filepath" + "testing" + + "github.com/clawkwork/clawk/internal/config" + "github.com/clawkwork/clawk/internal/template" + "github.com/stretchr/testify/require" +) + +// Workspace-position `env` and `agent` blocks used to be dropped on the floor: +// only repo Clawkfiles fed RequiredEnv and the agent docs. That position is +// also where the host-wide clawk.mod lands (see template/global.go), so the +// layer would have been silently half-applied. +func TestApplyWorkspaceLevelDefaults(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "house-rules.md"), + []byte("house rule"), 0o644)) + + ws := &template.Workspace{ + Root: root, + File: &template.Template{ + Env: []string{"GLOBAL_TOKEN"}, + Instructions: []template.AgentDoc{{Path: "./house-rules.md"}}, + Memory: []template.AgentDoc{{Text: "global memory"}}, + }, + } + sb := &config.Sandbox{ + Name: "x", + RequiredEnv: []string{"NS_TOKEN"}, + Instructions: []string{"namespace rule"}, + Memory: "namespace memory", + } + + require.NoError(t, applyWorkspaceLevelDefaults(sb, ws)) + + // Scope-outward ordering: workspace (and the global layer under it) first, + // then the namespace, then the repo — which for env is what decides + // precedence, since the last occurrence of a name wins. + require.Equal(t, []string{"GLOBAL_TOKEN", "NS_TOKEN"}, sb.RequiredEnv) + require.Equal(t, []string{"house rule", "namespace rule"}, sb.Instructions) + require.Equal(t, "global memory\n\nnamespace memory", sb.Memory) +} + +func TestApplyWorkspaceLevelDefaultsMissingDoc(t *testing.T) { + ws := &template.Workspace{ + Root: t.TempDir(), + File: &template.Template{ + Instructions: []template.AgentDoc{{Path: "./absent.md"}}, + }, + } + err := applyWorkspaceLevelDefaults(&config.Sandbox{Name: "x"}, ws) + require.ErrorContains(t, err, "absent.md") +} diff --git a/internal/config/namespace.go b/internal/config/namespace.go index 8841b87..af4ecbe 100644 --- a/internal/config/namespace.go +++ b/internal/config/namespace.go @@ -38,6 +38,12 @@ type Namespace struct { Files []HostFile `json:"files,omitempty"` Shares []HostShare `json:"shares,omitempty"` Env []string `json:"env,omitempty"` + // MCP are MCP servers made available to every sandbox in the namespace. + // This is the natural place for the org-wide set: which namespace a + // sandbox lives in then decides what it can reach, with no per-sandbox + // configuration. Merged with a repo's clawk.mod entries by name in + // applyNamespaceDefaults. See MCPServer. + MCP []MCPServer `json:"mcp,omitempty"` // Instructions and Memory seed every sandbox in the namespace: extra // CLAUDE.md guidance and baseline auto-memory respectively. They merge // with a repo's clawk.mod equivalents — namespace first, as the broader diff --git a/internal/config/types.go b/internal/config/types.go index 7c3cfdd..7402a99 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -2,8 +2,11 @@ package config import ( "fmt" + "path" + "path/filepath" "slices" "strconv" + "strings" "time" ) @@ -122,20 +125,32 @@ type NetworkBlock struct { // silently invert which rules win. const ( BlockOriginNamespace = "namespace" - BlockOriginMod = "mod" - BlockOriginCustom = "custom" + // BlockOriginMCP carries the allow entries derived from declared MCP + // servers (see MCPServer). Deliberately the lowest-precedence stored + // origin above "namespace": it is a convenience — clawk inferring what + // a declared server needs to reach — so an explicit `deny` written in + // clawk.mod ("mod") or via the CLI ("custom") must still win over it. + BlockOriginMCP = "mcp" + BlockOriginMod = "mod" + BlockOriginCustom = "custom" ) // blockOriginRank orders stored blocks; unknown origins sort first so a // record written by a newer clawk never outranks the user's custom block. +// +// The numbers are computed, never persisted — only the origin strings above +// are frozen — so inserting a new origin mid-chain is a matter of +// renumbering here. func blockOriginRank(origin string) int { switch origin { case BlockOriginNamespace: return 1 - case BlockOriginMod: + case BlockOriginMCP: return 2 - case BlockOriginCustom: + case BlockOriginMod: return 3 + case BlockOriginCustom: + return 4 default: return 0 } @@ -224,7 +239,9 @@ var DefaultAllowedDomains = []string{ "claude.ai", "gemini.google.com", "generativelanguage.googleapis.com", - "models.dev", // model catalog fetched by the opencode runner + "models.dev", // model catalog fetched by the opencode runner + "opencode.ai", // auth + update check for the opencode runner + "pi.dev", // model catalog + version check fetched by the pi runner "platform.claude.com", "play.googleapis.com", "statsig.anthropic.com", @@ -386,6 +403,62 @@ func (p PortForward) String() string { return fmt.Sprintf("%d:%d", p.HostPort, p.GuestPort) } +// SerialDevice is a host serial port presented as a PTY inside the guest — +// an Arduino or ESP32 on the Mac's USB, reachable from tooling in the +// sandbox. Bytes and line configuration are tunnelled over vsock (see +// internal/serialfwd); the USB device itself is not passed through, because +// no hypervisor clawk targets can do that. +// +// Like ReverseForwards these apply live to a running sandbox, and for the +// same reason they are vz-only: the guest is the end that dials. +type SerialDevice struct { + // HostPath is the device on the host, e.g. /dev/cu.usbmodem1101 on + // macOS or /dev/ttyACM0 on Linux. It may be a glob, which is resolved + // at open time rather than when it is configured — boards that + // re-enumerate into a bootloader often come back under a neighbouring + // name, and `/dev/cu.usbmodem*` survives that where a literal path + // doesn't. A glob that matches more than one device at open time is an + // error, not a guess. + HostPath string `json:"host_path"` + + // GuestName is the basename the PTY is symlinked to inside the guest, + // at /dev/. Never a path: the guest refuses anything with a + // directory part, and so does the host on the way in. + GuestName string `json:"guest_name"` +} + +// String renders the device in the HOST:GUEST spelling the CLI accepts, +// collapsing to the bare host path when the guest name is the default. +func (s SerialDevice) String() string { + if s.GuestName == DefaultSerialGuestName(s.HostPath) { + return s.HostPath + } + return fmt.Sprintf("%s:%s", s.HostPath, s.GuestName) +} + +// IsGlob reports whether HostPath is a pattern to resolve at open time +// rather than a literal device path. +func (s SerialDevice) IsGlob() bool { return strings.ContainsAny(s.HostPath, "*?[") } + +// DefaultSerialGuestName is the guest name used when a spec doesn't give +// one: the host device's basename, unchanged. +// +// Keeping the name identical on both sides is the same choice reverse +// forwarding makes with ports — you refer to the thing by the name you +// already know it by, and there is no second identifier to keep straight. +// A Mac's /dev/cu.usbmodem1101 is an odd name for a Linux device node, but +// no tool cares, and `arduino-cli -p /dev/cu.usbmodem1101` reading the same +// on both sides is worth more than tidiness. +// +// Returns "" for a glob, which has no meaningful basename; callers must +// require an explicit name in that case. +func DefaultSerialGuestName(hostPath string) string { + if strings.ContainsAny(hostPath, "*?[") { + return "" + } + return path.Base(filepath.ToSlash(hostPath)) +} + // HostFile is a snapshot-on-up file pushed from host into guest. Sourced // from clawk.mod `files (...)` and refreshed on every `clawk up` — edits // on the host are NOT live (use HostShare for that). HostPath is tilde- @@ -409,6 +482,55 @@ type HostShare struct { ReadOnly bool `json:"read_only"` } +// MCP transport identifiers. Persisted in sandbox records — frozen (see +// VMState). These are the wire names Claude Code's MCP config uses, so a +// rendered server config can pass them straight through. +const ( + MCPTransportHTTP = "http" + MCPTransportSSE = "sse" + MCPTransportStdio = "stdio" +) + +// MCPServer is one entry of a clawk.mod `mcp ( … )` block: an MCP server +// made available to the agent inside the sandbox. Sourced from the +// namespace and clawk.mod, snapshotted onto the record at create like +// every other template value, and rendered into the guest by +// sandbox.SeedClaudeMCP. +// +// Credentials are deliberately NOT modeled here. A `${VAR}` reference in +// Headers or Env is stored and rendered verbatim; the runner expands it +// against its own process environment at connect time (confirmed for +// headers, env and args). That keeps this record — and the rendered guest +// config, which lives on host disk under the sandbox state dir — free of +// secret values, exactly like RequiredEnv. The value itself travels +// separately: declare the variable in `env ( … )` and it reaches the +// runner's environment via the vsock handshake (see sandbox.ResolveEnv). +// +// Only static-credential transports are supported. Interactive OAuth +// (`claude mcp login`) stores its grant inside the guest and is out of +// scope: it can't be arranged at create time, which is the whole point of +// declaring servers here. +type MCPServer struct { + // Name is the server's identifier, as the agent sees it. Unique + // within a sandbox. + Name string `json:"name"` + // Transport is one of the MCPTransport* constants above. + Transport string `json:"transport"` + // URL is the endpoint for the http and sse transports; empty for stdio. + URL string `json:"url,omitempty"` + // Command is the argv of a stdio server, already split into words. + // Empty for http and sse. + Command []string `json:"command,omitempty"` + // Headers are extra HTTP request headers in "Name: value" form, for + // http and sse. This is where a PAT goes: + // "Authorization: Bearer ${SENTRY_TOKEN}". + Headers []string `json:"headers,omitempty"` + // Env names the environment variables handed to a stdio server. Each + // is rendered as NAME=${NAME}, so the value comes from the runner's + // environment rather than from this record. + Env []string `json:"env,omitempty"` +} + // DefaultNamespace is the grouping a sandbox belongs to when none is set. const DefaultNamespace = "default" @@ -451,6 +573,10 @@ type Sandbox struct { // by the daemon, so edits apply to a running sandbox. See // internal/revfwd. ReverseForwards []PortForward `json:"reverse_forwards,omitempty"` + // Serials are host serial ports presented as PTYs in the guest — see + // SerialDevice and internal/serialfwd. Like ReverseForwards these are + // tunnelled over vsock and apply to a running sandbox. + Serials []SerialDevice `json:"serials,omitempty"` // Files is the list of host->guest file copies refreshed on every // `clawk up`. See HostFile. Empty = no snapshots. Files []HostFile `json:"files,omitempty"` @@ -466,6 +592,13 @@ type Sandbox struct { // block read on every boot — the place for "always ask before X" or // project conventions that must survive a throwaway VM. Instructions []string `json:"instructions,omitempty"` + // MCP is the set of MCP servers made available to the agent inside the + // sandbox, sourced from the namespace and clawk.mod's `mcp ( … )` + // block. Rendered into the guest before first boot, and each http/sse + // entry's host is folded into the network policy's "mcp" block so a + // declared server is reachable without a separate `network allow`. + // See MCPServer. + MCP []MCPServer `json:"mcp,omitempty"` // Memory is seed content for the agent's auto-memory MEMORY.md, sourced // from the namespace and clawk.mod. Written into the memory dir once on // first boot and never afterward (see sandbox.SeedClaudeMemory), so a @@ -534,6 +667,12 @@ type Sandbox struct { // the next rootfs (re)build. DiskMiB uint64 `json:"disk_mib,omitempty"` + // SwapMiB is the sparse swap device's capacity in mebibytes, from + // clawk.mod `vm ( swap )`. Zero = sandbox.DefaultSwapSizeMiB; + // negative = no swap device at all. Unlike DiskMiB this is not baked into + // the rootfs — the device is a separate file, resized on the next boot. + SwapMiB int64 `json:"swap_mib,omitempty"` + // IdleTimeoutSec is how long the sandbox may sit idle (no attached // session, quiescent guest) before its VM daemon stops it to reclaim // host memory. Zero = the built-in default; negative = never stop. diff --git a/internal/config/types_test.go b/internal/config/types_test.go index 71a3e0c..427b17b 100644 --- a/internal/config/types_test.go +++ b/internal/config/types_test.go @@ -40,3 +40,44 @@ func TestSandboxNamespaceName(t *testing.T) { }) } } + +func TestDefaultSerialGuestName(t *testing.T) { + tests := []struct { + name string + hostPath string + want string + }{ + {"macOS callout device", "/dev/cu.usbmodem1101", "cu.usbmodem1101"}, + {"linux acm device", "/dev/ttyACM0", "ttyACM0"}, + {"bare name", "ttyUSB0", "ttyUSB0"}, + // A glob has no basename worth defaulting to — the caller must + // insist on an explicit guest name rather than inventing one. + {"glob has no default", "/dev/cu.usbmodem*", ""}, + {"character class glob", "/dev/ttyUSB[01]", ""}, + {"single-char glob", "/dev/ttyUSB?", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.want, DefaultSerialGuestName(tt.hostPath)) + }) + } +} + +func TestSerialDeviceString(t *testing.T) { + // The default guest name collapses back to the bare host path, so a + // round-tripped spec reads the way the user typed it. + require.Equal(t, "/dev/cu.usbmodem1101", + SerialDevice{HostPath: "/dev/cu.usbmodem1101", GuestName: "cu.usbmodem1101"}.String()) + require.Equal(t, "/dev/cu.usbmodem1101:ttyACM0", + SerialDevice{HostPath: "/dev/cu.usbmodem1101", GuestName: "ttyACM0"}.String()) + // A glob never has a default name, so it always shows both halves. + require.Equal(t, "/dev/cu.usbmodem*:ttyACM0", + SerialDevice{HostPath: "/dev/cu.usbmodem*", GuestName: "ttyACM0"}.String()) +} + +func TestSerialDeviceIsGlob(t *testing.T) { + require.False(t, SerialDevice{HostPath: "/dev/cu.usbmodem1101"}.IsGlob()) + require.True(t, SerialDevice{HostPath: "/dev/cu.usbmodem*"}.IsGlob()) + require.True(t, SerialDevice{HostPath: "/dev/ttyUSB[01]"}.IsGlob()) + require.True(t, SerialDevice{HostPath: "/dev/ttyUSB?"}.IsGlob()) +} diff --git a/internal/guestcfg/manifest.go b/internal/guestcfg/manifest.go index bc59f27..a5b0875 100644 --- a/internal/guestcfg/manifest.go +++ b/internal/guestcfg/manifest.go @@ -47,11 +47,43 @@ type Manifest struct { Hostname string `json:"hostname,omitempty"` Network *Network `json:"network,omitempty"` User *User `json:"user,omitempty"` + Swap *Swap `json:"swap,omitempty"` Mounts []Mount `json:"mounts,omitempty"` Files []File `json:"files,omitempty"` Services []Service `json:"services,omitempty"` } +// Swap is the guest's swap device: a sparse virtio-blk disk the host +// attaches and clawk-init formats and enables at boot. Nil means the +// sandbox runs without swap. +// +// A dedicated device rather than a swapfile on the rootfs, because +// swapon(2) rejects a file with holes: a 4 GiB swapfile would cost 4 GiB +// of real host bytes at first boot, per sandbox. A block device has no +// such rule, so the backing file stays sparse and only materializes the +// pages actually swapped. +// +// Why swap at all: the balloon controller (machine/vz) reclaims guest RAM +// under host memory pressure, against guest demand — see mergedBalloonTarget. +// With nowhere to put anonymous pages the guest answers that with direct +// reclaim stalls and, at the limit, its OOM killer. Multi-second stalls in +// the agent process are what turn a marginal network link into dropped API +// streams, since a stalled process stops draining its socket. +// +// Additive like Mount.Block — an older clawk-init ignores the field and +// boots without swap, so this needs no Version bump. +type Swap struct { + // Device is the in-guest block device path (e.g. "/dev/vdc"). The + // provider owns the ordering that produces it: disks are attached + // rootfs-first, then Spec.Disks in order. + Device string `json:"device"` + + // Swappiness sets vm.swappiness when non-zero. Zero leaves the kernel + // default (60) alone — it is not a way to say "never swap", which is + // what a literal swappiness of 0 would mean. + Swappiness int `json:"swappiness,omitempty"` +} + // Network is the static interface configuration. gvproxy assigns a fixed // DHCP lease to the sandbox MAC, but arbitrary images have no DHCP // client — clawk-init configures the same values statically instead. diff --git a/internal/sandbox/agentstate_test.go b/internal/sandbox/agentstate_test.go new file mode 100644 index 0000000..30d076f --- /dev/null +++ b/internal/sandbox/agentstate_test.go @@ -0,0 +1,162 @@ +package sandbox + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/clawkwork/clawk/internal/config" + "github.com/clawkwork/clawk/internal/guestcfg" + "github.com/stretchr/testify/require" +) + +// TestPersistentAgentSharesCoverEveryRunnerHome is the regression guard for +// the bug this file exists for: only claude's home was mounted from the host, +// so codex's sessions and login lived on the vz rootfs — which is re-cloned +// from the image on EVERY boot. `clawk down && clawk up` silently wiped codex +// history even though the docs promised it persisted. Each runner clawk +// claims to persist needs its own share here. +func TestPersistentAgentSharesCoverEveryRunnerHome(t *testing.T) { + stateRoot := t.TempDir() + + byGuest := map[string]HostShare{} + for _, sh := range PersistentAgentShares(stateRoot) { + byGuest[sh.GuestPath] = sh + } + + want := map[string]struct{ sub, tag string }{ + GuestHome + "/.claude": {"claude", "claude_home"}, + GuestHome + "/.codex": {"codex", "codex_home"}, + GuestHome + "/.pi": {"pi", "pi_home"}, + // opencode follows the XDG split, so it needs two: the data dir + // (auth.json, mcp-auth.json, opencode.db) and the config dir. + GuestHome + "/.local/share/opencode": {"opencode-data", "opencode_data"}, + GuestHome + "/.config/opencode": {"opencode-config", "opencode_config"}, + } + require.Len(t, byGuest, len(want), "one share per persisted runner home") + for guest, w := range want { + sh, ok := byGuest[guest] + require.Truef(t, ok, "no persistent share mounted at %s", guest) + require.Equal(t, filepath.Join(stateRoot, w.sub), sh.HostPath) + require.Equal(t, w.tag, sh.Tag) + require.Falsef(t, sh.ReadOnly, "%s must be writable — the runner writes its sessions there", guest) + + // virtiofs refuses a missing source path, so the host dir has to + // exist by the time the share is handed to the provider. + info, err := os.Stat(sh.HostPath) + require.NoErrorf(t, err, "stat %s", sh.HostPath) + require.Truef(t, info.IsDir(), "%s exists but is not a directory", sh.HostPath) + } +} + +// TestPersistentAgentSharesExcludeVolatileDirs pins the two opencode XDG +// dirs we deliberately leave on the disposable rootfs. ~/.local/state holds +// only locks/, and a lock that outlives the VM that took it is worse than +// none — a hard stop would strand one for the next boot. ~/.cache is a +// cache. Both look like "state opencode writes", so without this guard the +// obvious-seeming fix is to add them. +func TestPersistentAgentSharesExcludeVolatileDirs(t *testing.T) { + for _, sh := range PersistentAgentShares(t.TempDir()) { + require.NotContains(t, sh.GuestPath, "/.local/state/", + "%s persists a lock directory across boots", sh.Tag) + require.NotContains(t, sh.GuestPath, "/.cache/", + "%s persists a cache directory, costing a PCIe device for no correctness gain", sh.Tag) + } +} + +func TestPersistentAgentSharesEmptyStateRoot(t *testing.T) { + require.Nil(t, PersistentAgentShares(""), "empty state root opts out entirely") +} + +// TestPersistentAgentSharesIdempotent pins that a second call (every boot +// rewrites the manifest) returns the identical list — a churned tag means the +// guest tries to mount a device vz never exposed. +func TestPersistentAgentSharesIdempotent(t *testing.T) { + stateRoot := t.TempDir() + require.Equal(t, PersistentAgentShares(stateRoot), PersistentAgentShares(stateRoot)) +} + +// TestAgentStateDirsUniqueTagsAndPaths guards the two uniqueness constraints +// the list carries: virtio-fs tags identify a device per VM, and two entries +// mounting the same guest path would have the second shadow the first. +func TestAgentStateDirsUniqueTagsAndPaths(t *testing.T) { + tags, guests, subs := map[string]bool{}, map[string]bool{}, map[string]bool{} + for _, d := range AgentStateDirs { + require.NotEmpty(t, d.Agent, "every entry names its runner") + require.Falsef(t, tags[d.Tag], "duplicate virtio-fs tag %q", d.Tag) + require.Falsef(t, guests[d.GuestPath], "duplicate guest path %q", d.GuestPath) + require.Falsef(t, subs[d.Sub], "duplicate state subdir %q", d.Sub) + require.Truef(t, strings.HasPrefix(d.GuestPath, GuestHome+"/"), + "%s must live under the agent's home so clawk-init chowns it", d.GuestPath) + tags[d.Tag], guests[d.GuestPath], subs[d.Sub] = true, true, true + } +} + +// TestPersistentAgentSharesPrecedeCapabilitySubmounts pins the ordering rule +// both share assemblers depend on. DefaultHostShares mounts capability dirs +// (~/.claude/agents, ~/.codex/skills) that live INSIDE the persisted homes; +// clawk-init mounts in list order, so a home mounted afterwards would shadow +// the capability dir already mounted under it. +func TestPersistentAgentSharesPrecedeCapabilitySubmounts(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + for _, sub := range []string{".claude/agents", ".claude/commands", ".codex/skills"} { + require.NoError(t, os.MkdirAll(filepath.Join(home, sub), 0o755)) + } + + // Same assembly order as OCIGuestManifest and collectSandboxShares. + assembled := append([]HostShare{}, PersistentAgentShares(t.TempDir())...) + assembled = append(assembled, DefaultHostShares()...) + + idx := map[string]int{} + for i, sh := range assembled { + idx[sh.GuestPath] = i + } + var checked int + for _, child := range assembled { + for _, parent := range assembled { + if child.GuestPath == parent.GuestPath || + !strings.HasPrefix(child.GuestPath, parent.GuestPath+"/") { + continue + } + checked++ + require.Lessf(t, idx[parent.GuestPath], idx[child.GuestPath], + "parent %s must be mounted before its sub-mount %s", + parent.GuestPath, child.GuestPath) + } + } + require.Positivef(t, checked, "test set up no nested mounts — it would pass vacuously") +} + +// TestOCIGuestManifestMountsEveryAgentHome ties the guest side to the list: +// a runner added to AgentStateDirs but missing from the manifest would write +// to the disposable rootfs instead, which is exactly how codex lost its +// history. +func TestOCIGuestManifestMountsEveryAgentHome(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + require.NoError(t, os.MkdirAll(filepath.Join(home, ".codex", "skills"), 0o755)) + + sb := &config.Sandbox{Name: "box", Image: "golang:1.25"} + m, err := OCIGuestManifest(sb, t.TempDir(), "", t.TempDir()) + require.NoError(t, err) + + idx := map[string]int{} + mounts := map[string]guestcfg.Mount{} + for i, mt := range m.Mounts { + idx[mt.Tag] = i + mounts[mt.Tag] = mt + } + for _, d := range AgentStateDirs { + mt, ok := mounts[d.Tag] + require.Truef(t, ok, "%s state (tag %s) missing from the guest manifest", d.Agent, d.Tag) + require.Equal(t, d.GuestPath, mt.Path) + require.Falsef(t, mt.ReadOnly, "%s state must be writable", d.Agent) + } + // The one nested pair the default shares produce, spelled out so a + // reordering of the assembly in OCIGuestManifest fails loudly here. + require.Contains(t, idx, "codex_skills") + require.Less(t, idx["codex_home"], idx["codex_skills"], + "~/.codex must mount before ~/.codex/skills or the skills mount is shadowed") +} diff --git a/internal/sandbox/firecracker_linux.go b/internal/sandbox/firecracker_linux.go index 3d1770e..8516031 100644 --- a/internal/sandbox/firecracker_linux.go +++ b/internal/sandbox/firecracker_linux.go @@ -63,6 +63,11 @@ const ( // are attached in spec order after the rootfs (machine/firecracker puts // rootfs first, then Spec.Disks), so vda=rootfs, vdb=guestcfg, vdc=this. fcWorktreeDevice = "/dev/vdc" + // fcSwapDevice is the swap disk, attached after the worktree — so + // vda=rootfs, vdb=guestcfg, vdc=worktree, vdd=this. Firecracker's + // virtio-blk advertises no discard, so freed swap pages stay allocated in + // the backing file (see sandbox.DefaultSwapSizeMiB). + fcSwapDevice = "/dev/vdd" // worktreeDiskSlack is the free space added to a worktree disk beyond the // tree itself, for whatever the agent builds in there. Sparse, so unused // capacity costs nothing on the host. @@ -151,6 +156,9 @@ func (f *FirecrackerProvider) Create(sb *config.Sandbox) error { if err := guestcfg.WriteDisk(f.manifest(sb), filepath.Join(vmDir, "guestcfg.img")); err != nil { return fmt.Errorf("guest config disk: %w", err) } + if _, err := EnsureSwapDisk(vmDir, SwapDiskMiB(sb)); err != nil { + return fmt.Errorf("swap disk: %w", err) + } sb.GuestIP = guestIP return nil } @@ -171,7 +179,11 @@ func (f *FirecrackerProvider) hasRestorableState(sb *config.Sandbox, vmDir, root if !machine.SuspendStateExists(filepath.Join(vmDir, "suspend")) { return false } - for _, disk := range []string{rootfs, f.worktreeDiskPath(sb), filepath.Join(vmDir, "guestcfg.img")} { + disks := []string{rootfs, f.worktreeDiskPath(sb), filepath.Join(vmDir, "guestcfg.img")} + if SwapDiskMiB(sb) > 0 { + disks = append(disks, SwapDiskPath(vmDir)) + } + for _, disk := range disks { if _, err := os.Stat(disk); err != nil { return false } @@ -195,6 +207,7 @@ func (f *FirecrackerProvider) manifest(sb *config.Sandbox) guestcfg.Manifest { DNS: []string{gvproxyGateway}, MTU: gvproxyMTU, }, + Swap: fcSwap(sb), Mounts: []guestcfg.Mount{{ Path: f.guestWorktreePath(sb), Block: fcWorktreeDevice, @@ -204,6 +217,16 @@ func (f *FirecrackerProvider) manifest(sb *config.Sandbox) guestcfg.Manifest { } } +// fcSwap is the manifest's swap entry, or nil when the sandbox opted out. +// Keyed off the same SwapDiskMiB as buildSpec's disk list so the manifest +// never names a device firecracker wasn't asked to attach. +func fcSwap(sb *config.Sandbox) *guestcfg.Swap { + if SwapDiskMiB(sb) == 0 { + return nil + } + return &guestcfg.Swap{Device: fcSwapDevice, Swappiness: GuestSwappiness} +} + // guestWorktreePath is where the worktree disk is mounted in the guest — // the same /workspace/ layout the bake produced, so nothing downstream // (sessions, runners, cwd inference) sees a change. @@ -456,6 +479,13 @@ func (f *FirecrackerProvider) DaemonSpec(sb *config.Sandbox, allow *netfilter.Al cleanup() return machine.Spec{}, nil, fmt.Errorf("kernel: %w", err) } + // Ensured on every boot, not just at create, so an edited + // `vm ( swap )` takes effect on the next start — buildSpec attaches + // whatever SwapDiskMiB says, and the device has to exist by then. + if _, err := EnsureSwapDisk(f.vmDir(sb), SwapDiskMiB(sb)); err != nil { + cleanup() + return machine.Spec{}, nil, fmt.Errorf("swap disk: %w", err) + } spec := f.buildSpec(sb, kernelPath) if ns != nil { // Handed over here and nowhere earlier: machine.UserMode documents that @@ -508,6 +538,13 @@ func (f *FirecrackerProvider) buildSpec(sb *config.Sandbox, kernelPath string) m if sb.MemoryMaxMiB > memMaxMiB { memMaxMiB = sb.MemoryMaxMiB } + disks := []machine.Disk{ + {Path: filepath.Join(f.vmDir(sb), "guestcfg.img"), ReadOnly: true}, + {Path: f.worktreeDiskPath(sb)}, + } + if SwapDiskMiB(sb) > 0 { + disks = append(disks, machine.Disk{Path: SwapDiskPath(f.vmDir(sb))}) + } return machine.Spec{ ID: sb.Name, VCPU: vcpu, @@ -523,12 +560,9 @@ func (f *FirecrackerProvider) buildSpec(sb *config.Sandbox, kernelPath string) m }, RootFS: machine.RawDisk{Path: filepath.Join(f.vmDir(sb), "rootfs.raw")}, // Order is the guest's device order: vdb=guestcfg, vdc=worktree - // (fcWorktreeDevice). Appending anything here shifts later devices, - // so keep new disks after these two. - Disks: []machine.Disk{ - {Path: filepath.Join(f.vmDir(sb), "guestcfg.img"), ReadOnly: true}, - {Path: f.worktreeDiskPath(sb)}, - }, + // (fcWorktreeDevice), vdd=swap (fcSwapDevice). Appending anything here + // shifts later devices, so keep new disks after these. + Disks: disks, VSockCID: fcVSockCID, Serial: machine.Serial{LogPath: filepath.Join(f.vmDir(sb), "console.log")}, } diff --git a/internal/sandbox/firecracker_linux_test.go b/internal/sandbox/firecracker_linux_test.go index ada44bc..eeee8fb 100644 --- a/internal/sandbox/firecracker_linux_test.go +++ b/internal/sandbox/firecracker_linux_test.go @@ -39,12 +39,39 @@ func TestHasRestorableState(t *testing.T) { "a state saved before the worktree had its own disk must not be trusted") touch(f.worktreeDiskPath(sb)) + require.False(t, f.hasRestorableState(sb, vmDir, rootfs), + "a state saved before sandboxes had a swap disk must not be trusted either") + + touch(SwapDiskPath(vmDir)) require.True(t, f.hasRestorableState(sb, vmDir, rootfs), "the full disk set is present") require.NoError(t, os.Remove(filepath.Join(vmDir, "guestcfg.img"))) require.False(t, f.hasRestorableState(sb, vmDir, rootfs), "guestcfg.img is attached too") } +// A sandbox with swap disabled attaches no swap disk, so its absence must +// not hold the restore back — the check has to track buildSpec's disk list, +// not a fixed set. +func TestHasRestorableStateSwapOff(t *testing.T) { + root := t.TempDir() + f := &FirecrackerProvider{store: config.NewStoreAt(root)} + sb := &config.Sandbox{Name: "proj", SwapMiB: -1} + vmDir := f.vmDir(sb) + rootfs := filepath.Join(vmDir, "rootfs.raw") + + for _, path := range []string{ + filepath.Join(vmDir, "suspend", "snapshot.state"), + rootfs, + filepath.Join(vmDir, "guestcfg.img"), + f.worktreeDiskPath(sb), + } { + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte("x"), 0o644)) + } + require.True(t, f.hasRestorableState(sb, vmDir, rootfs), + "no swap disk is expected when swap is off") +} + func TestWorktreeDiskSize(t *testing.T) { dir := t.TempDir() require.NoError(t, os.MkdirAll(filepath.Join(dir, "sub"), 0o755)) diff --git a/internal/sandbox/mcp.go b/internal/sandbox/mcp.go new file mode 100644 index 0000000..aa37c68 --- /dev/null +++ b/internal/sandbox/mcp.go @@ -0,0 +1,153 @@ +package sandbox + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/clawkwork/clawk/internal/config" + "github.com/google/renameio/v2" +) + +// GuestMCPConfigPath is where clawk renders the sandbox's declared MCP +// servers inside the guest, and the path passed to the runner's +// --mcp-config flag (see cli.mcpConfigArgs). +// +// It deliberately sits inside ~/.claude/ rather than at either of the +// places the runner would find on its own: +// +// - ~/.claude.json (user scope) is clawk's onboarding marker and carries +// a documented concurrent-write race (anthropics/claude-code#28847). +// - .mcp.json in a project root would land inside a git worktree mounted +// from the host, i.e. clawk would be writing config into the user's +// repo. +// +// Living under ~/.claude/ also means it arrives through the per-sandbox +// PersistentAgentShares mount with the rest of the seeded state, and the +// sessions history repo ignores it by construction — that gitignore denies +// everything (`/*`) and only re-admits transcripts and memory. +const GuestMCPConfigPath = GuestHome + "/.claude/mcp/clawk.json" + +// mcpConfig is the on-disk shape the runner reads. Field names are the +// runner's, not clawk's. +type mcpConfig struct { + MCPServers map[string]mcpServerConfig `json:"mcpServers"` +} + +type mcpServerConfig struct { + // Type is emitted for remote transports only. A stdio server is + // identified by having a command, and omitting the key there matches + // the shape the runner documents for `claude mcp add`. + Type string `json:"type,omitempty"` + URL string `json:"url,omitempty"` + Headers map[string]string `json:"headers,omitempty"` + Command string `json:"command,omitempty"` + Args []string `json:"args,omitempty"` + Env map[string]string `json:"env,omitempty"` +} + +// RenderMCPConfig builds the guest MCP config for a sandbox's declared +// servers. Returns ok=false when there is nothing to declare. +// +// `${VAR}` references in headers and env values are passed through +// verbatim: the runner expands them against its own process environment +// when it connects, so no credential value is ever written here. That +// matters because this file is created on the HOST, inside the sandbox +// state dir — the same reason config.Sandbox.RequiredEnv stores names +// rather than values. The values reach the runner's environment through +// the vsock handshake instead (ResolveEnv → cli.buildVSockEnv). +// +// A stdio server's env is rendered as NAME=${NAME} for the same reason: +// clawk names the variable, the runner supplies the value. +func RenderMCPConfig(servers []config.MCPServer) ([]byte, bool, error) { + if len(servers) == 0 { + return nil, false, nil + } + cfg := mcpConfig{MCPServers: make(map[string]mcpServerConfig, len(servers))} + for _, s := range servers { + entry := mcpServerConfig{} + switch s.Transport { + case config.MCPTransportStdio: + if len(s.Command) == 0 { + return nil, false, fmt.Errorf("mcp server %q: stdio transport with no command", s.Name) + } + entry.Command = s.Command[0] + entry.Args = s.Command[1:] + if len(s.Env) > 0 { + entry.Env = make(map[string]string, len(s.Env)) + for _, name := range s.Env { + entry.Env[name] = "${" + name + "}" + } + } + case config.MCPTransportHTTP, config.MCPTransportSSE: + if s.URL == "" { + return nil, false, fmt.Errorf("mcp server %q: %s transport with no URL", s.Name, s.Transport) + } + entry.Type = s.Transport + entry.URL = s.URL + if len(s.Headers) > 0 { + entry.Headers = make(map[string]string, len(s.Headers)) + for _, h := range s.Headers { + name, val, ok := strings.Cut(h, ":") + if !ok { + return nil, false, fmt.Errorf("mcp server %q: malformed header %q", s.Name, h) + } + entry.Headers[strings.TrimSpace(name)] = strings.TrimSpace(val) + } + } + default: + return nil, false, fmt.Errorf("mcp server %q: unknown transport %q", s.Name, s.Transport) + } + cfg.MCPServers[s.Name] = entry + } + // MarshalIndent sorts map keys, so the bytes are stable for a given + // server list — this file is rewritten on every boot into a virtio-fs + // mount, and a stable rendering keeps that a no-op write. + content, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return nil, false, err + } + return append(content, '\n'), true, nil +} + +// SeedClaudeMCP writes (or clears) the guest MCP config in the per-sandbox +// state dir that PersistentAgentShares mounts at ~/.claude/. Run by the +// provider during sandbox preparation, alongside SeedClaudeStateDir and +// before the VM boots — so the servers are in place for the runner's first +// connection attempt, with no post-boot step and no `on create` hook. +// +// Rewritten on every call, like settings.json: a clawk.mod edit propagates +// on the next `up`. An empty server list removes the file rather than +// leaving a stale one behind, so deleting an `mcp ( … )` entry actually +// retires the server. +// +// Note this only arranges the config. Whether a server then authenticates +// is up to the credential its headers/env reference — see +// config.MCPServer. +func SeedClaudeMCP(stateRoot string, servers []config.MCPServer) error { + if stateRoot == "" { + return nil + } + path := filepath.Join(stateRoot, "claude", "mcp", "clawk.json") + content, ok, err := RenderMCPConfig(servers) + if err != nil { + return err + } + if !ok { + if err := os.Remove(path); err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("clearing mcp config: %w", err) + } + return nil + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return fmt.Errorf("creating mcp config dir: %w", err) + } + if err := renameio.WriteFile(path, content, 0o644); err != nil { + return fmt.Errorf("seeding mcp config: %w", err) + } + return nil +} diff --git a/internal/sandbox/mcp_test.go b/internal/sandbox/mcp_test.go new file mode 100644 index 0000000..8149730 --- /dev/null +++ b/internal/sandbox/mcp_test.go @@ -0,0 +1,152 @@ +package sandbox + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/clawkwork/clawk/internal/config" + "github.com/stretchr/testify/require" +) + +func TestRenderMCPConfigRemote(t *testing.T) { + content, ok, err := RenderMCPConfig([]config.MCPServer{{ + Name: "linear", + Transport: config.MCPTransportHTTP, + URL: "https://mcp.linear.app/mcp", + Headers: []string{"Authorization: Bearer ${LINEAR_TOKEN}", "X-Env: prod"}, + }}) + require.NoError(t, err) + require.True(t, ok) + + var got struct { + MCPServers map[string]struct { + Type string `json:"type"` + URL string `json:"url"` + Headers map[string]string `json:"headers"` + } `json:"mcpServers"` + } + require.NoError(t, json.Unmarshal(content, &got)) + srv := got.MCPServers["linear"] + require.Equal(t, "http", srv.Type) + require.Equal(t, "https://mcp.linear.app/mcp", srv.URL) + require.Equal(t, "Bearer ${LINEAR_TOKEN}", srv.Headers["Authorization"], + "the ${VAR} reference must survive verbatim — the runner expands it in-guest") + require.Equal(t, "prod", srv.Headers["X-Env"]) +} + +func TestRenderMCPConfigStdio(t *testing.T) { + content, ok, err := RenderMCPConfig([]config.MCPServer{{ + Name: "github", + Transport: config.MCPTransportStdio, + Command: []string{"npx", "-y", "@modelcontextprotocol/server-github"}, + Env: []string{"GITHUB_TOKEN"}, + }}) + require.NoError(t, err) + require.True(t, ok) + + var got struct { + MCPServers map[string]struct { + Type string `json:"type"` + Command string `json:"command"` + Args []string `json:"args"` + Env map[string]string `json:"env"` + } `json:"mcpServers"` + } + require.NoError(t, json.Unmarshal(content, &got)) + srv := got.MCPServers["github"] + require.Equal(t, "npx", srv.Command) + require.Equal(t, []string{"-y", "@modelcontextprotocol/server-github"}, srv.Args) + require.Equal(t, map[string]string{"GITHUB_TOKEN": "${GITHUB_TOKEN}"}, srv.Env, + "clawk names the variable; the runner's environment supplies the value") + require.Empty(t, srv.Type, "stdio is implied by having a command") +} + +// TestRenderMCPConfigCarriesNoSecrets is the invariant that lets this file +// live on host disk inside the sandbox state dir: it holds references, never +// values, exactly like config.Sandbox.RequiredEnv. +func TestRenderMCPConfigCarriesNoSecrets(t *testing.T) { + t.Setenv("LINEAR_TOKEN", "super-secret-value") + content, _, err := RenderMCPConfig([]config.MCPServer{{ + Name: "linear", + Transport: config.MCPTransportHTTP, + URL: "https://mcp.linear.app/mcp", + Headers: []string{"Authorization: Bearer ${LINEAR_TOKEN}"}, + }}) + require.NoError(t, err) + require.NotContains(t, string(content), "super-secret-value") + require.Contains(t, string(content), "${LINEAR_TOKEN}") +} + +// TestRenderMCPConfigStable: the file is rewritten on every boot into a +// virtio-fs mount, so identical input must produce identical bytes. +func TestRenderMCPConfigStable(t *testing.T) { + servers := []config.MCPServer{ + {Name: "b", Transport: config.MCPTransportHTTP, URL: "https://b/mcp"}, + {Name: "a", Transport: config.MCPTransportHTTP, URL: "https://a/mcp"}, + } + first, _, err := RenderMCPConfig(servers) + require.NoError(t, err) + second, _, err := RenderMCPConfig(servers) + require.NoError(t, err) + require.Equal(t, string(first), string(second)) + require.Less(t, strings.Index(string(first), `"a"`), strings.Index(string(first), `"b"`), + "keys are sorted, so declaration order can't churn the file") +} + +func TestRenderMCPConfigRejectsIncomplete(t *testing.T) { + _, _, err := RenderMCPConfig([]config.MCPServer{{Name: "x", Transport: config.MCPTransportStdio}}) + require.ErrorContains(t, err, "no command") + + _, _, err = RenderMCPConfig([]config.MCPServer{{Name: "x", Transport: config.MCPTransportHTTP}}) + require.ErrorContains(t, err, "no URL") + + _, _, err = RenderMCPConfig([]config.MCPServer{{Name: "x", Transport: "carrier-pigeon"}}) + require.ErrorContains(t, err, "unknown transport") +} + +func TestRenderMCPConfigEmpty(t *testing.T) { + content, ok, err := RenderMCPConfig(nil) + require.NoError(t, err) + require.False(t, ok) + require.Nil(t, content) +} + +// TestSeedClaudeMCP covers the full lifecycle at the path the guest sees +// through the ~/.claude mount: written before boot, refreshed on the next +// one, and removed when the declaration goes away. +func TestSeedClaudeMCP(t *testing.T) { + stateRoot := t.TempDir() + path := filepath.Join(stateRoot, "claude", "mcp", "clawk.json") + + require.NoError(t, SeedClaudeMCP(stateRoot, []config.MCPServer{{ + Name: "linear", Transport: config.MCPTransportHTTP, URL: "https://mcp.linear.app/mcp", + }})) + first, err := os.ReadFile(path) + require.NoError(t, err, "seeded before the VM boots, so the dir is created on demand") + require.Contains(t, string(first), "mcp.linear.app") + + // A clawk.mod edit propagates on the next up. + require.NoError(t, SeedClaudeMCP(stateRoot, []config.MCPServer{{ + Name: "notion", Transport: config.MCPTransportHTTP, URL: "https://mcp.notion.com/mcp", + }})) + second, err := os.ReadFile(path) + require.NoError(t, err) + require.Contains(t, string(second), "mcp.notion.com") + require.NotContains(t, string(second), "mcp.linear.app", "rewritten, not merged") + + // Dropping the block retires the servers rather than leaving a stale file. + require.NoError(t, SeedClaudeMCP(stateRoot, nil)) + _, err = os.Stat(path) + require.True(t, os.IsNotExist(err), "stale config must not survive an emptied mcp block") + + // Clearing an already-absent file is not an error (every boot calls this). + require.NoError(t, SeedClaudeMCP(stateRoot, nil)) +} + +func TestSeedClaudeMCPNoStateRoot(t *testing.T) { + require.NoError(t, SeedClaudeMCP("", []config.MCPServer{{Name: "x"}}), + "an opted-out state root is a no-op, matching the other seed functions") +} diff --git a/internal/sandbox/oci_sandbox.go b/internal/sandbox/oci_sandbox.go index 1617103..ab5ec9f 100644 --- a/internal/sandbox/oci_sandbox.go +++ b/internal/sandbox/oci_sandbox.go @@ -106,6 +106,16 @@ func OCIGuestManifest(sb *config.Sandbox, stateDir, cacheDir, rootDir string) (g }, } + // Swap rides on its own virtio-blk disk (EnsureSwapDisk), attached right + // after the config disk. Both sides key off SwapDiskMiB, so a sandbox + // with swap off gets neither the device nor the manifest entry. + if SwapDiskMiB(sb) > 0 { + m.Swap = &guestcfg.Swap{ + Device: OCISwapDevice, + Swappiness: GuestSwappiness, + } + } + // Consolidated worktree mount: every managed (non-in-place) worktree for // this sandbox lives under one host parent (store.WorktreeDir), which // collectSandboxShares exposes as a single virtio-fs device. Mount that @@ -150,10 +160,11 @@ func OCIGuestManifest(sb *config.Sandbox, stateDir, cacheDir, rootDir string) (g }) } - // Host shares, in parent-before-child order: the whole ~/.claude - // state mount must precede the skills/agents/commands sub-mounts or - // the latter end up shadowed. - shares := append([]HostShare{}, PersistentClaudeShares(stateDir)...) + // Host shares, in parent-before-child order: the per-runner state + // mounts (~/.claude, ~/.codex, ~/.pi) must precede the + // skills/agents/commands sub-mounts that land inside them, or the + // latter end up shadowed. + shares := append([]HostShare{}, PersistentAgentShares(stateDir)...) shares = append(shares, DefaultHostShares()...) shares = append(shares, ToolchainCacheShares(cacheDir)...) shares = append(shares, UserHostShares(sb.Shares)...) diff --git a/internal/sandbox/oci_sandbox_test.go b/internal/sandbox/oci_sandbox_test.go index 35754e2..85437b5 100644 --- a/internal/sandbox/oci_sandbox_test.go +++ b/internal/sandbox/oci_sandbox_test.go @@ -81,7 +81,7 @@ func TestOCIGuestManifest(t *testing.T) { t.Errorf("workspace parent (idx %d) must be mounted before in-place sub-mount (idx %d)", wsIdx, hereIdx) } if mt, ok := mounts["claude_home"]; !ok || mt.Path != GuestHome+"/.claude" { - // PersistentClaudeShares' tag; if its tag ever changes this test + // PersistentAgentShares' tag; if its tag ever changes this test // must change together with collectSandboxShares. t.Errorf("claude state mount missing/wrong: %+v", mounts) } diff --git a/internal/sandbox/shares.go b/internal/sandbox/shares.go index 3129b37..dd94029 100644 --- a/internal/sandbox/shares.go +++ b/internal/sandbox/shares.go @@ -6,6 +6,7 @@ import ( _ "embed" "encoding/hex" "encoding/json" + "errors" "fmt" "os" "os/exec" @@ -60,61 +61,124 @@ type HostFile struct { Owner string // "user:group" or "" for root } -// PersistentClaudeShares returns the per-sandbox host share that -// carries Claude Code's entire ~/.claude/ across destroy/recreate -// cycles. Call sites already resolved a Store so they pass the state -// root directly rather than re-deriving the path. -// -// The host directory is created idempotently so virtiofs never -// encounters a missing source. The guest-side mount point is -// ~/.claude/ — per-sandbox storage that does NOT suffer the shared- -// .claude.json races documented below; two sandboxes can't touch -// the same path because each sandbox name maps to a distinct host -// dir. -// -// We mount the whole dir rather than a curated subdir list because: -// -// - The "ephemeral" subdirs (cache/, paste-cache/, shell-snapshots/, -// telemetry/) measure in hundreds of KB total — exclusion isn't -// worth the bookkeeping. -// - settings.json and CLAUDE.md, formerly snapshot HostFiles, are -// now seeded via SeedClaudeStateDir straight into the synced -// dir before the mount happens. That removes the snapshot-file -// vs share-mount layering issue. -// - .credentials.json (non-token path) lives at its canonical place -// inside the synced dir, so claude's atomic write-rename refresh -// persists naturally — no more copy-in/copy-out trick via -// auth/credentials.json. -// -// Mount ordering: this share must come BEFORE DefaultHostShares in -// the assembled share list. The agents/commands sub-mounts land on -// top of ~/.claude/ at boot, and Linux would shadow them under a -// later parent mount. -// -// Cross-sandbox races (the ones DefaultHostShares is avoiding) don't -// apply here — each sandbox name maps to its own state dir, so two -// sandboxes never write to the same path. +// AgentStateDir maps one coding-agent runner's home directory onto the +// per-sandbox host storage that backs it. See AgentStateDirs. +type AgentStateDir struct { + // Agent is the runner name in the CLI's agent registry, purely for + // documentation and error messages. + Agent string + // Sub is the subdirectory under the sandbox's state root. + Sub string + // Tag is the virtio-fs tag; unique across every share on one VM. + Tag string + // GuestPath is where the runner looks for its state inside the VM. + GuestPath string +} + +// AgentStateDirs lists the runner home directories clawk persists per +// sandbox. Every entry is one virtio-fs device, so the list is the +// contract both the device side (cli.collectSandboxShares) and the guest +// manifest (OCIGuestManifest) iterate. +// +// Why each runner needs one: the vz rootfs is re-cloned from the image on +// EVERY boot (see suspendBootRootFS — a per-boot-disposable disk is the +// design), so anything a runner writes under $HOME on the rootfs is gone +// after a plain `clawk down && clawk up`, never mind a destroy. Claude +// survived that because its ~/.claude was mounted from the host; codex's +// sessions, history, and login did not, so every restart looked like a +// fresh install. Persisting each runner's home dir is what makes the +// documented "conversation memory persists across destroys" promise true +// for more than one runner. +// +// Guest paths, all whole-home mounts rather than curated subdir lists — +// the ephemeral parts (caches, logs) measure in hundreds of KB and +// excluding them isn't worth the bookkeeping: +// +// claude ~/.claude projects/, memory/, settings.json, .credentials.json +// codex ~/.codex sessions/, history.jsonl, auth.json, config.toml +// pi ~/.pi agent/{sessions,settings.json,auth.json,trust.json} +// +// opencode is the one runner that needs two, because it follows the XDG +// split rather than keeping a single home (verified with `opencode debug +// paths`, which is the authority for its layout): +// +// opencode ~/.local/share/opencode auth.json, mcp-auth.json, opencode.db, repos/ +// opencode ~/.config/opencode opencode.jsonc +// +// Its other two XDG dirs are deliberately left on the disposable rootfs. +// ~/.local/state/opencode holds only locks/, and a lock that outlives the +// VM it was taken in is worse than no lock at all — a hard stop would +// leave one behind for the next boot to trip over. ~/.cache/opencode is a +// cache by name and contract; the only real cost is re-downloading +// cache/bin per sandbox, which is the same trade the toolchain caches make +// (see ToolchainCachesEnabled). +// +// Cost note: every entry here is a PCIe device on every sandbox, against +// the ceiling documented in machine/vz (32, with a field-confirmed failure +// at 34). Five entries plus the workspace, the per-repo aliases, and the +// default capability shares still leaves room for a normal multi-repo +// sandbox, but this list is not free to extend — a sixth runner wanting +// three dirs is where the consolidation trick (one mount plus a +// dir-override env var, e.g. OPENCODE_CONFIG_DIR) starts paying for its +// complexity. +var AgentStateDirs = []AgentStateDir{ + {Agent: "claude", Sub: "claude", Tag: "claude_home", GuestPath: GuestHome + "/.claude"}, + {Agent: "codex", Sub: "codex", Tag: "codex_home", GuestPath: GuestHome + "/.codex"}, + {Agent: "pi", Sub: "pi", Tag: "pi_home", GuestPath: GuestHome + "/.pi"}, + {Agent: "opencode", Sub: "opencode-data", Tag: "opencode_data", + GuestPath: GuestHome + "/.local/share/opencode"}, + {Agent: "opencode", Sub: "opencode-config", Tag: "opencode_config", + GuestPath: GuestHome + "/.config/opencode"}, +} + +// PersistentAgentShares returns the per-sandbox host shares that carry +// each coding agent's home directory across down/up and destroy/recreate +// cycles. Call sites already resolved a Store so they pass the state root +// directly rather than re-deriving the path. +// +// Host directories are created idempotently so virtiofs never encounters +// a missing source; a directory that can't be created drops its share +// rather than failing sandbox creation, matching ToolchainCacheShares. +// The guest mount points are per-sandbox storage, so they do NOT suffer +// the shared-state races documented on DefaultHostShares — two sandboxes +// can't touch the same path because each sandbox name maps to a distinct +// host dir. +// +// For claude specifically, the whole-dir mount is also what lets +// settings.json, CLAUDE.md and .credentials.json be seeded straight into +// the synced dir by SeedClaudeStateDir before the mount happens (no +// snapshot-file vs share-mount layering issue), and lets claude's atomic +// write-rename credential refresh persist naturally. +// +// Mount ordering: these shares must come BEFORE DefaultHostShares in the +// assembled share list. Its sub-mounts land INSIDE these homes +// (~/.claude/agents, ~/.claude/commands, ~/.codex/skills), and Linux +// would shadow them under a later parent mount. // // Opt out by passing an empty stateRoot. -func PersistentClaudeShares(stateRoot string) []HostShare { +func PersistentAgentShares(stateRoot string) []HostShare { if stateRoot == "" { return nil } - hostPath := filepath.Join(stateRoot, "claude") - if err := os.MkdirAll(hostPath, 0o755); err != nil { - return nil + out := make([]HostShare, 0, len(AgentStateDirs)) + for _, d := range AgentStateDirs { + hostPath := filepath.Join(stateRoot, d.Sub) + if err := os.MkdirAll(hostPath, 0o755); err != nil { + continue + } + out = append(out, HostShare{ + HostPath: hostPath, + Tag: d.Tag, + GuestPath: d.GuestPath, + ReadOnly: false, + }) } - return []HostShare{{ - HostPath: hostPath, - Tag: "claude_home", - GuestPath: GuestHome + "/.claude", - ReadOnly: false, - }} + return out } // SeedClaudeStateDir writes the host-snapshot files (settings.json, // CLAUDE.md, .credentials.json) directly into the per-sandbox state dir that -// PersistentClaudeShares mounts at ~/.claude/. Run by the provider +// PersistentAgentShares mounts at ~/.claude/. Run by the provider // during sandbox preparation, BEFORE the VM boots — so by the time // virtio-fs mounts the dir, the files are already in place. // @@ -188,7 +252,7 @@ func SeedClaudeStateDir(stateRoot, clawkRootDir string) error { // claudeCredentialsPath is where claude reads and refreshes its OAuth // credentials, in host terms: inside the per-sandbox state dir that -// PersistentClaudeShares mounts at ~/.claude/. +// PersistentAgentShares mounts at ~/.claude/. func claudeCredentialsPath(stateRoot string) string { return filepath.Join(stateRoot, "claude", ".credentials.json") } @@ -274,7 +338,7 @@ const ToolchainCachesEnabled = false // transparent. Host dirs are created on demand (virtiofs refuses // missing source paths); MkdirAll failures silently drop the offending // share rather than failing sandbox creation, matching the behavior -// of PersistentClaudeShares. +// of PersistentAgentShares. // // Caches included (well-documented concurrent safety, real payoff): // @@ -357,9 +421,15 @@ func ToolchainCacheShares(cacheDir string) []HostShare { // // Sharing Claude agents/commands and Codex skills is safe: user-authored // capability dirs with low write contention. Sharing .claude.json, -// credentials, projects/, file-history/, or the whole ~/.codex state dir -// would risk concurrent writers. Each sandbox authenticates independently — -// one-time cost per sandbox in exchange for no cross-session corruption. +// credentials, projects/, file-history/, or the rest of ~/.codex would risk +// concurrent writers. Each sandbox authenticates independently — one-time +// cost per sandbox in exchange for no cross-session corruption. +// +// "Not shared with the host" is not the same as "not persisted": the +// runners' state dirs still survive down/up and destroy through +// PersistentAgentShares, which gives each sandbox its OWN host directory. +// These sub-mounts land inside those homes, so PersistentAgentShares must +// be assembled first or the parent mount shadows them. // // ~/.claude/skills is deliberately NOT shared. A skill like gstack carries // a large node_modules tree, and virtio-fs caches an inode (and host fd) @@ -444,7 +514,7 @@ func userShareTag(guestPath string) string { // Files that DO live inside ~/.claude/ — settings.json, CLAUDE.md, // .credentials.json — are pre-seeded into the per-sandbox state dir // by SeedClaudeStateDir before the VM boots; they appear at the -// canonical paths through the PersistentClaudeShares mount, no +// canonical paths through the PersistentAgentShares mount, no // snapshot-file involvement needed. // // clawkRootDir resolves the optional long-lived OAuth token @@ -635,52 +705,126 @@ func shellEscapeDoubleQuoted(v string) string { return shellEscapeDoubleQuotedReplacer.Replace(v) } -// EnvFile synthesizes an /etc/profile.d script that exports every -// sandbox-required env var. Entries come from sb.RequiredEnv (declared in -// clawk.mod, in canonical envspec form); values are resolved against the -// host's process env at this call. +// DeclaredEnvNames is the set of guest variable names a sandbox's +// `env ( … )` block declares, independent of whether any of them can be +// resolved right now. +// +// Separate from ResolveEnv because the two answer different questions, and +// conflating them is a security bug: "which names did the user speak for" +// must not depend on the host shell. cli.buildVSockEnv suppresses clawk's +// own injected variables for every declared name, and deriving that set +// from ResolveEnv's output meant an entry that failed to resolve (a +// ${HOST:?msg} whose host var left the shell) silently handed the name back +// to clawk — re-injecting the Anthropic OAuth token into a sandbox whose +// clawk.mod had explicitly disowned it. See config.MCPServer for why that +// token must not reach a third-party endpoint. +// +// Entries that don't parse are skipped: they name nothing usable, and +// ResolveEnv reports them. +func DeclaredEnvNames(sb *config.Sandbox) map[string]bool { + if sb == nil || len(sb.RequiredEnv) == 0 { + return nil + } + names := make(map[string]bool, len(sb.RequiredEnv)) + for _, entry := range sb.RequiredEnv { + spec, err := envspec.Parse(entry) + if err != nil { + continue + } + names[spec.Name] = true + } + return names +} + +// ResolveEnv resolves a sandbox's declared `env ( … )` entries against the +// host process environment, returning them as canonical NAME=value strings +// in declaration order. // // Resolution follows the envspec grammar: a bare passthrough / ${HOST} -// alias exports the host value (empty + a warning when unset); ${HOST:-x} -// / ${HOST-x} fall back to a default; a bare/quoted literal is exported -// verbatim; and ${HOST:?msg} / ${HOST?msg} make a missing variable a hard -// error (returned here, failing sandbox creation with a clear message). -// -// Written into /etc/profile.d/99-clawk-env.sh so every login shell -// (ssh, `claude ...`, interactive bash, phase setup scripts) picks up -// the values without any per-tool configuration. -// -// Returns ok=false if the sandbox has no required env — saves the -// caller from having to filter empty HostFiles. -func EnvFile(sb *config.Sandbox) (HostFile, bool, error) { +// alias takes the host value (empty + a warning when unset); ${HOST:-x} +// / ${HOST-x} fall back to a default; a bare/quoted literal is used +// verbatim; and ${HOST:?msg} / ${HOST?msg} make a missing variable an +// error. +// +// It is the single resolution step behind both delivery paths, so a hard +// failure or an unset-variable warning reads the same either way: +// +// - EnvFile below, which renders /etc/profile.d/99-clawk-env.sh for +// every login shell in the guest. +// - the pty agent's vsock handshake (internal/cli.buildVSockEnv), which +// spawns the runner directly — no login shell, no /etc/profile — and +// so has to carry the values itself. +// +// Entries that fail to resolve are skipped but still reported, so the +// returned slice always holds everything that DID resolve: strict callers +// (sandbox create) treat a non-nil error as fatal, while best-effort +// callers (agent attach, which must not become unusable just because one +// variable left the host shell) can warn and carry on with the rest. +// +// Values are never persisted — they're read from the host env at call +// time, which is why both callers re-resolve on every use. +func ResolveEnv(sb *config.Sandbox) ([]string, error) { if len(sb.RequiredEnv) == 0 { - return HostFile{}, false, nil + return nil, nil } - var b bytes.Buffer - b.WriteString("# Generated by clawk — host env vars requested in clawk.mod\n") + out := make([]string, 0, len(sb.RequiredEnv)) + var errs []error for _, entry := range sb.RequiredEnv { spec, err := envspec.Parse(entry) if err != nil { // The template parser already validated every entry, so this // only fires on a hand-edited sandbox record — still worth a // clear error rather than a malformed export line. - return HostFile{}, false, fmt.Errorf( - "sandbox %q: invalid env entry %q: %w", sb.Name, entry, err) + errs = append(errs, fmt.Errorf( + "sandbox %q: invalid env entry %q: %w", sb.Name, entry, err)) + continue } val, warnUnset, err := spec.Resolve(os.LookupEnv) if err != nil { - return HostFile{}, false, fmt.Errorf("sandbox %q: %w", sb.Name, err) + errs = append(errs, fmt.Errorf("sandbox %q: %w", sb.Name, err)) + continue } if warnUnset { fmt.Fprintf(os.Stderr, "warning: %s required by clawk.mod is unset on host; "+ "exporting empty in sandbox %q\n", spec.Host, sb.Name) } + out = append(out, spec.Name+"="+val) + } + return out, errors.Join(errs...) +} + +// EnvFile synthesizes an /etc/profile.d script that exports every +// sandbox-required env var. Entries come from sb.RequiredEnv (declared in +// clawk.mod, in canonical envspec form); values are resolved by ResolveEnv +// against the host's process env at this call, and a resolution failure +// (e.g. an unset ${HOST:?msg}) is fatal here — it fails sandbox creation +// with a clear message. +// +// Written into /etc/profile.d/99-clawk-env.sh so every login shell +// (ssh, interactive bash, phase setup scripts, the `bash -lc` agent +// fallback) picks up the values without any per-tool configuration. The +// primary agent path does NOT go through a login shell — see ResolveEnv. +// +// Returns ok=false if the sandbox has no required env — saves the +// caller from having to filter empty HostFiles. +func EnvFile(sb *config.Sandbox) (HostFile, bool, error) { + entries, err := ResolveEnv(sb) + if err != nil { + return HostFile{}, false, err + } + if len(entries) == 0 { + return HostFile{}, false, nil + } + var b bytes.Buffer + b.WriteString("# Generated by clawk — host env vars requested in clawk.mod\n") + for _, e := range entries { + name, val, _ := strings.Cut(e, "=") // Double-quoted with $ and " escaped — covers secrets that // contain dollar signs or quotes. Single quotes aren't safe // because bash single-quote strings can't contain single // quotes, and API keys sometimes do. - fmt.Fprintf(&b, "export %s=\"%s\"\n", spec.Name, shellEscapeDoubleQuoted(val)) + fmt.Fprintf(&b, "export %s=\"%s\"\n", name, shellEscapeDoubleQuoted(val)) } return HostFile{ Content: b.Bytes(), diff --git a/internal/sandbox/shares_test.go b/internal/sandbox/shares_test.go index c53a8b1..bae0c39 100644 --- a/internal/sandbox/shares_test.go +++ b/internal/sandbox/shares_test.go @@ -101,7 +101,7 @@ func TestToolchainCacheSharesUniqueTags(t *testing.T) { stateRoot := t.TempDir() all := append([]HostShare{}, DefaultHostShares()...) - all = append(all, PersistentClaudeShares(stateRoot)...) + all = append(all, PersistentAgentShares(stateRoot)...) all = append(all, ToolchainCacheShares(cacheDir)...) seen := make(map[string]string, len(all)) @@ -208,6 +208,62 @@ func TestEnvFileComposeModel(t *testing.T) { } } +// TestResolveEnvSharedByBothDeliveryPaths pins the contract EnvFile and the +// vsock handshake share: canonical NAME=value strings in declaration order, +// values read from the host env at call time (never persisted). +func TestResolveEnvSharedByBothDeliveryPaths(t *testing.T) { + t.Setenv("HOST_GH", "ghp_xyz") + os.Unsetenv("HOST_MISSING") + + got, err := ResolveEnv(&config.Sandbox{ + Name: "sb", + RequiredEnv: []string{ + "GH_TOKEN=${HOST_GH}", + "LOG_LEVEL=${HOST_MISSING:-info}", + "EDITOR=vim", + }, + }) + if err != nil { + t.Fatalf("ResolveEnv: %v", err) + } + want := []string{"GH_TOKEN=ghp_xyz", "LOG_LEVEL=info", "EDITOR=vim"} + if len(got) != len(want) { + t.Fatalf("got %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("entry %d = %q, want %q", i, got[i], want[i]) + } + } + + if entries, err := ResolveEnv(&config.Sandbox{Name: "sb"}); err != nil || entries != nil { + t.Errorf("no declared env: got %v, %v; want nil, nil", entries, err) + } +} + +// TestResolveEnvReturnsPartialResults is what lets attach stay best-effort +// while create stays strict: one unresolvable entry must not take the +// resolvable ones down with it. EnvFile treats the error as fatal; +// cli.buildVSockEnv warns and uses what came back. +func TestResolveEnvReturnsPartialResults(t *testing.T) { + os.Unsetenv("HOST_REQUIRED") + t.Setenv("HOST_OK", "fine") + + got, err := ResolveEnv(&config.Sandbox{ + Name: "sb", + RequiredEnv: []string{ + "API_KEY=${HOST_REQUIRED:?set it in your shell}", + "OK=${HOST_OK}", + }, + }) + if err == nil { + t.Fatal("expected an error for the required-but-missing entry") + } + if len(got) != 1 || got[0] != "OK=fine" { + t.Errorf("got %v, want the one resolvable entry [OK=fine]", got) + } +} + // TestEnvFileRequiredMissingErrors verifies that a ${HOST:?msg} whose // host variable is unset fails EnvFile (and therefore sandbox creation) // with a message that includes the author's note. diff --git a/internal/sandbox/swapdisk.go b/internal/sandbox/swapdisk.go new file mode 100644 index 0000000..cd435fa --- /dev/null +++ b/internal/sandbox/swapdisk.go @@ -0,0 +1,109 @@ +package sandbox + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/clawkwork/clawk/internal/config" +) + +// Swap. Every sandbox gets a swap device unless it opts out with +// `vm ( swap off )`. +// +// The reason is the balloon controller, not the guest's own appetite. Under +// host memory pressure clawk reclaims guest RAM against guest demand +// (machine/vz.mergedBalloonTarget drops the cap to 75% of the ceiling at WARN +// and 50% at CRITICAL), and the guest's only answers without swap are direct +// reclaim stalls and the OOM killer — the balloon's DEFLATE_ON_OOM safety net +// assumes exactly that. A multi-second stall in the agent process is not just +// slow: a process that stops draining its TLS socket lets the connection go +// idle, and on a link whose NAT reaps idle mappings in under a minute that +// ends the streaming response outright. +// +// The cost is bounded by sparseness, not by size — see DefaultSwapSizeMiB. + +// DefaultSwapSizeMiB is the swap device's capacity when the sandbox doesn't +// say otherwise. It is a ceiling on how much the guest may swap, not an +// allocation: the backing file is sparse and materializes host bytes only as +// pages are actually written to it. A sandbox that never swaps carries a +// 2 GiB device that costs a few hundred bytes of directory entry. +// +// It does not shrink again on its own, though. Nothing in the stack punches +// the holes back: swapon(2) is asked for SWAP_FLAG_DISCARD, but neither +// firecracker's virtio-blk nor vz's advertises discard, so the kernel drops +// the flag and freed swap pages stay allocated on the host until the sandbox +// is destroyed. Read the number as a high-water mark — which is the reason +// not to make it larger just because the device is sparse. +const DefaultSwapSizeMiB = 2048 + +// SwapDiskName is the swap device's filename inside a sandbox's VM +// directory. Removed with the rest of the VM dir on destroy. +const SwapDiskName = "swap.img" + +// OCISwapDevice is where the swap disk lands in a vz OCI sandbox. Disks are +// attached in spec order after the rootfs, so vda=rootfs, vdb=guestcfg, and +// swap is the next one. Keep in lock-step with buildOCISandboxSpec's +// Spec.Disks in internal/cli/vzd.go. +const OCISwapDevice = "/dev/vdc" + +// GuestSwappiness is the vm.swappiness clawk-init sets on a swap-enabled +// guest. Above the kernel's default 60 on purpose: the pages we want the +// guest to give up under balloon inflation are cold anonymous ones (an idle +// agent heap), and the page cache we want it to keep is a repo and toolchain +// an active build reads constantly. The default's more even split trades the +// wrong way for this workload. +const GuestSwappiness = 80 + +// SwapDiskMiB is the swap capacity for sb, in MiB, or 0 when the sandbox has +// swap disabled. Mirrors RootDiskSizeMiB's shape: an explicit positive +// override wins, negative means off, and unset takes the default. +func SwapDiskMiB(sb *config.Sandbox) uint64 { + switch { + case sb == nil: + return DefaultSwapSizeMiB + case sb.SwapMiB < 0: + return 0 + case sb.SwapMiB > 0: + return uint64(sb.SwapMiB) + default: + return DefaultSwapSizeMiB + } +} + +// SwapDiskPath is the swap device's host path inside vmDir. +func SwapDiskPath(vmDir string) string { return filepath.Join(vmDir, SwapDiskName) } + +// EnsureSwapDisk makes vmDir hold a sparse swap device of sizeMiB and returns +// its path. A sizeMiB of 0 removes any device a previous configuration left +// behind and returns "" — callers use the empty path as "attach nothing". +// +// Resizing an existing device just truncates it. Swap contents are worthless +// across a boot (the guest re-formats whenever the header doesn't match the +// device), so there is nothing to preserve, and truncation keeps the file +// sparse where a rewrite would not. +func EnsureSwapDisk(vmDir string, sizeMiB uint64) (string, error) { + path := SwapDiskPath(vmDir) + if sizeMiB == 0 { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return "", fmt.Errorf("removing swap disk: %w", err) + } + return "", nil + } + size := int64(sizeMiB) << 20 + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o600) + if err != nil { + return "", fmt.Errorf("creating swap disk: %w", err) + } + defer f.Close() + st, err := f.Stat() + if err != nil { + return "", fmt.Errorf("stat swap disk: %w", err) + } + if st.Size() != size { + if err := f.Truncate(size); err != nil { + return "", fmt.Errorf("sizing swap disk to %d MiB: %w", sizeMiB, err) + } + } + return path, nil +} diff --git a/internal/sandbox/swapdisk_test.go b/internal/sandbox/swapdisk_test.go new file mode 100644 index 0000000..49f3e0d --- /dev/null +++ b/internal/sandbox/swapdisk_test.go @@ -0,0 +1,95 @@ +package sandbox + +import ( + "os" + "path/filepath" + "syscall" + "testing" + + "github.com/clawkwork/clawk/internal/config" + "github.com/stretchr/testify/require" +) + +// allocatedBlocks is the file's real footprint on the host in 512-byte +// blocks — st_blocks, which stays at zero across a hole and grows only +// where something was actually written. st_size would report the whole +// ceiling and tell us nothing about what the sandbox costs. +func allocatedBlocks(t *testing.T, path string) int64 { + t.Helper() + fi, err := os.Stat(path) + require.NoError(t, err) + st, ok := fi.Sys().(*syscall.Stat_t) + require.True(t, ok, "stat is not a syscall.Stat_t") + return int64(st.Blocks) +} + +func TestSwapDiskMiB(t *testing.T) { + tests := []struct { + name string + sb *config.Sandbox + want uint64 + }{ + {name: "nil sandbox takes the default", sb: nil, want: DefaultSwapSizeMiB}, + {name: "unset takes the default", sb: &config.Sandbox{}, want: DefaultSwapSizeMiB}, + {name: "explicit size wins", sb: &config.Sandbox{SwapMiB: 8192}, want: 8192}, + // Negative is the "off" sentinel the parser stores for `swap off`, + // distinct from unset — otherwise disabling swap would be + // indistinguishable from never having configured it. + {name: "negative disables", sb: &config.Sandbox{SwapMiB: -1}, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.want, SwapDiskMiB(tt.sb)) + }) + } +} + +func TestEnsureSwapDisk(t *testing.T) { + vmDir := t.TempDir() + + path, err := EnsureSwapDisk(vmDir, 64) + require.NoError(t, err) + require.Equal(t, filepath.Join(vmDir, SwapDiskName), path) + + fi, err := os.Stat(path) + require.NoError(t, err) + require.Equal(t, int64(64<<20), fi.Size(), "apparent size") + + // The point of the device is that it costs host bytes only as the guest + // swaps into it. A freshly created one must be a hole, not 64 MiB of + // zeros — st_blocks, not st_size, is what the host actually pays. + require.Zero(t, allocatedBlocks(t, path), "a fresh swap disk must be sparse") + + // Re-running is idempotent, and a size change resizes in place. + again, err := EnsureSwapDisk(vmDir, 64) + require.NoError(t, err) + require.Equal(t, path, again) + + _, err = EnsureSwapDisk(vmDir, 128) + require.NoError(t, err) + fi, err = os.Stat(path) + require.NoError(t, err) + require.Equal(t, int64(128<<20), fi.Size(), "resized") + + // Shrinking works too: swap contents don't survive a boot, so there is + // nothing to preserve and truncation keeps the file sparse. + _, err = EnsureSwapDisk(vmDir, 32) + require.NoError(t, err) + fi, err = os.Stat(path) + require.NoError(t, err) + require.Equal(t, int64(32<<20), fi.Size(), "shrunk") + + // Zero means "swap off": the device from a previous configuration has to + // go, or buildSpec would keep finding a file it no longer attaches. + empty, err := EnsureSwapDisk(vmDir, 0) + require.NoError(t, err) + require.Empty(t, empty) + _, err = os.Stat(path) + require.True(t, os.IsNotExist(err), "swap disk removed") + + // Removing an absent device is not an error — the common case is a + // sandbox that never had swap in the first place. + empty, err = EnsureSwapDisk(vmDir, 0) + require.NoError(t, err) + require.Empty(t, empty) +} diff --git a/internal/sandbox/vzprovider_oci_darwin.go b/internal/sandbox/vzprovider_oci_darwin.go index 9d76c12..85324b9 100644 --- a/internal/sandbox/vzprovider_oci_darwin.go +++ b/internal/sandbox/vzprovider_oci_darwin.go @@ -114,6 +114,13 @@ func (v *VZProvider) createOCI(sb *config.Sandbox) error { if err := SeedClaudeStateDir(v.store.StateDir(sb.Name), v.store.RootDir()); err != nil { return fmt.Errorf("seeding claude state dir: %w", err) } + // Declared MCP servers, rendered before boot so the runner's first + // connection attempt already has them. Fatal, unlike the memory seed + // below: a sandbox that silently comes up without the servers its + // clawk.mod asked for is the failure this whole path exists to avoid. + if err := SeedClaudeMCP(v.store.StateDir(sb.Name), sb.MCP); err != nil { + return fmt.Errorf("seeding mcp config: %w", err) + } // Seed baseline auto-memory once (no-op if the sandbox already has memory, // e.g. a re-create whose state dir survived). Best-effort: never block boot. if err := SeedClaudeMemory(v.store.StateDir(sb.Name), sb.Memory); err != nil { diff --git a/internal/serialfwd/serialfwd.go b/internal/serialfwd/serialfwd.go new file mode 100644 index 0000000..6bed51e --- /dev/null +++ b/internal/serialfwd/serialfwd.go @@ -0,0 +1,388 @@ +// Package serialfwd is the wire protocol for serial forwarding: a physical +// serial port on the host presented as a PTY inside the guest. +// +// The motivating case is microcontroller work — an Arduino or ESP32 plugged +// into the Mac, flashed and monitored by tooling running in the sandbox. +// Passing the USB device itself through is not an option: Virtualization. +// framework exposes no physical USB passthrough (its USB controller carries +// virtual mass-storage devices only) and firecracker has no USB at all. But +// none of the tooling actually wants USB — avrdude, esptool and every serial +// monitor want a tty and a baud rate. That is a byte stream plus a little +// out-of-band state, which vsock carries fine. +// +// Shape, per connection (the guest always dials, the host always listens on +// VSockPort — the same asymmetry as internal/revfwd, and the same reason +// this is vz-only): +// +// guest → host one JSON Greeting, newline-terminated +// op=control host replies with a Snapshot line now and another on +// every change to the device set, until the connection is +// closed. This is how the guest learns which PTYs to +// create, so `clawk serial add` applies live. +// op=attach host validates Name against the current set, opens the +// physical port, replies with one AttachReply line, and +// then speaks frames (below) for the rest of the +// connection. +// +// Validation is host-side on purpose: the guest names a *device*, never a +// host path, and a name the user didn't configure is refused. The host +// holds the mapping from name to /dev/cu.usbmodem…, so a process in the +// sandbox can reach exactly the ports the user attached and no others. +// +// # Connection lifetime is the open/close signal +// +// An attach connection exists for exactly as long as some guest process +// holds the PTY open, and the host opens and closes the physical port to +// match. That is not just bookkeeping — it is how auto-reset works. The +// classic Arduino reset circuit pulses RESET from the DTR line, and opening +// a serial port asserts DTR; this is why opening the Arduino IDE's serial +// monitor reboots an Uno. Tying the host-side open to the guest-side open +// reproduces that pulse at the one moment the tooling expects it, without +// the guest ever naming a modem-control line. +// +// It also means the port is free whenever the sandbox isn't using it, so +// the Arduino IDE on the Mac can still have it. +// +// # What a PTY cannot carry +// +// A PTY has no modem-control lines: TIOCMGET/TIOCMSET on either end return +// ENOTTY on Linux. So a guest tool that toggles DTR or RTS explicitly gets +// an error, and no protocol here can fix that — see docs/serial.md for +// which boards that affects and the workarounds. What a PTY *does* carry is +// the termios state (master and slave share it, so the guest agent can read +// back the baud rate its client set) and the open/close edges. Those are +// the two things the FrameMode message and the connection lifetime +// respectively convey, and between them they cover the 1200-baud touch that +// native-USB boards use to enter their bootloader. +// +// # Framing +// +// The handshake is JSON lines like revfwd's, because it's a handful of +// messages and being greppable in a log is worth more than the bytes. What +// follows is *not* a raw byte stream like revfwd's, though: serial data and +// mode changes have to stay in order relative to each other. A tool that +// writes a command, changes baud, then writes another command must have +// those land in that order on the wire — so both travel as frames on the +// one connection rather than data inline and mode on the side. +// +// The guest half is inlined in internal/agentembed/main.go.in (the agent +// builds standalone inside the guest and can't import this package). Any +// change here must be mirrored there — same rule as internal/revfwd, and +// TestSerialProtocolMirroredInAgent enforces it. +package serialfwd + +import ( + "bufio" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "strings" +) + +// VSockPort is the host-side AF_VSOCK port the guest dials for both +// connection kinds. Disjoint from the other fixed ports: 1024 pty-agent, +// 1025 time-sync, 1026 ssh-agent, 1027 mem-report, 1028 reverse-forward, +// 1100+ 9p caches. +const VSockPort uint32 = 1029 + +// ProtoVersion is the current wire version. Bumped only on a breaking +// change; the guest binary is rebuilt from these sources and re-injected on +// every `clawk up`, so host and guest can't drift within a release. +const ProtoVersion = 1 + +// Greeting is the first line of every connection. +type Greeting struct { + // Op is OpControl or OpAttach. + Op string `json:"op"` + + // V is the sender's ProtoVersion. + V int `json:"v"` + + // Name is the guest-visible device name to attach to, without any + // directory part — the same string the host published in a Snapshot. + // Set for OpAttach only. + Name string `json:"name,omitempty"` + + // Mode is the PTY's termios state at the moment the guest attached, so + // the host can configure the port before the first byte moves rather + // than opening at some default and correcting. Set for OpAttach only; + // nil means "whatever the port already had". + Mode *Mode `json:"mode,omitempty"` +} + +// Greeting Op values. +const ( + OpControl = "control" + OpAttach = "attach" +) + +// Snapshot is the host's reply on an OpControl connection: the complete set +// of forwarded serial devices as of now. Every update resends the full set — +// the guest reconciles against it rather than applying deltas, so a +// dropped-and-redialed control connection converges the same way. +type Snapshot struct { + Devices []Device `json:"devices"` +} + +// Device is one host serial port exposed in the guest. +type Device struct { + // Name is the guest-visible name, created at /dev/. The host + // path is deliberately not sent: the guest has no use for it and + // shouldn't learn the shape of the host's /dev. + Name string `json:"name"` +} + +// AttachReply is the host's verdict on an OpAttach greeting. Frames flow +// only after ok=true; on ok=false the host closes the connection. +type AttachReply struct { + OK bool `json:"ok"` + // Error is a short human-readable reason when OK is false — an unknown + // device name, or the port being unplugged or already held by another + // process. The guest logs it; it's the only place an unplugged board + // surfaces. + Error string `json:"error,omitempty"` +} + +// Mode is the line configuration of a serial port. It is the subset of +// termios that describes the wire format, which is all a remote end can +// meaningfully apply — flow control and the special characters belong to +// whichever end is doing the cooking. +type Mode struct { + // Baud is the symbol rate. Both directions are always set to it: + // split-speed serial has no users left and no way to express itself + // through the PTY the guest reads this back from. + Baud int `json:"baud"` + + // Bits is the character size, 5 through 8. + Bits int `json:"bits"` + + // Parity is ParityNone, ParityEven or ParityOdd. + Parity string `json:"parity"` + + // Stop is the number of stop bits, 1 or 2. + Stop int `json:"stop"` +} + +// Parity values. +const ( + ParityNone = "n" + ParityEven = "e" + ParityOdd = "o" +) + +// DefaultMode is what a port is configured to when the guest attaches +// without stating a mode. 9600-8N1 because that is what a tty defaults to +// nearly everywhere, so a guest tool that never calls tcsetattr sees the +// same thing it would on real hardware. +func DefaultMode() Mode { + return Mode{Baud: 9600, Bits: 8, Parity: ParityNone, Stop: 1} +} + +func (m Mode) String() string { + return fmt.Sprintf("%d-%d%s%d", m.Baud, m.Bits, strings.ToUpper(m.Parity), m.Stop) +} + +// Validate reports whether m is a line configuration that can actually be +// applied. It is called on the host before touching the port, because the +// values arrive from the guest: a nonsense character size would otherwise +// become a confusing tcsetattr failure at the far end of the stack. +func (m Mode) Validate() error { + if m.Baud <= 0 { + return fmt.Errorf("serialfwd: baud %d out of range", m.Baud) + } + if m.Bits < 5 || m.Bits > 8 { + return fmt.Errorf("serialfwd: character size %d out of range (5-8)", m.Bits) + } + switch m.Parity { + case ParityNone, ParityEven, ParityOdd: + default: + return fmt.Errorf("serialfwd: unknown parity %q", m.Parity) + } + if m.Stop != 1 && m.Stop != 2 { + return fmt.Errorf("serialfwd: stop bits %d out of range (1-2)", m.Stop) + } + return nil +} + +// ──────────────────────────────────────────────────────────────────────── +// Framing +// ──────────────────────────────────────────────────────────────────────── + +// FrameType tags a post-handshake message. +type FrameType byte + +const ( + // FrameData carries raw serial bytes, in either direction. + FrameData FrameType = 0x01 + + // FrameMode carries a JSON Mode, guest → host, when the guest's PTY + // client changed the line configuration. Ordered with respect to the + // FrameData frames around it, which is the entire reason frames exist + // here — see the package comment. + FrameMode FrameType = 0x02 +) + +func (t FrameType) String() string { + switch t { + case FrameData: + return "data" + case FrameMode: + return "mode" + default: + return fmt.Sprintf("unknown(0x%02x)", byte(t)) + } +} + +// FrameHeaderBytes is the fixed header: one type byte then a big-endian +// uint32 payload length. +const FrameHeaderBytes = 5 + +// MaxFrameBytes caps one frame's payload. Serial is slow — even at 921600 +// baud a port produces ~90 KiB/s — so this is far above any real read, and +// exists only so a peer can't make the reader allocate without bound. +const MaxFrameBytes = 256 * 1024 + +// ErrFrameTooLong reports a frame past MaxFrameBytes. The connection must +// be closed: the reader has no way to resynchronise with the framing. +var ErrFrameTooLong = errors.New("serialfwd: frame exceeds MaxFrameBytes") + +// WriteFrame writes one frame. Callers that interleave frames from several +// goroutines must serialise their own writes — a partially written frame +// desynchronises the stream for good. +func WriteFrame(w io.Writer, t FrameType, payload []byte) error { + if len(payload) > MaxFrameBytes { + return ErrFrameTooLong + } + var hdr [FrameHeaderBytes]byte + hdr[0] = byte(t) + binary.BigEndian.PutUint32(hdr[1:], uint32(len(payload))) + // One Write for the header and one for the body rather than a copy + // into a joined buffer: these go to a vsock conn, and the extra + // allocation per data frame costs more than the extra syscall. + if _, err := w.Write(hdr[:]); err != nil { + return err + } + if len(payload) == 0 { + return nil + } + _, err := w.Write(payload) + return err +} + +// WriteModeFrame encodes m and writes it as a FrameMode. +func WriteModeFrame(w io.Writer, m Mode) error { + b, err := json.Marshal(m) + if err != nil { + return fmt.Errorf("serialfwd: encoding mode: %w", err) + } + return WriteFrame(w, FrameMode, b) +} + +// ReadFrame reads one frame. The returned slice aliases a fresh allocation, +// so the caller may retain it. +// +// It takes a *bufio.Reader for the same reason ReadLine does: the handshake +// and the frames share a connection, and a reader that buffered past the +// greeting's newline holds bytes belonging to the first frame. +func ReadFrame(r *bufio.Reader) (FrameType, []byte, error) { + var hdr [FrameHeaderBytes]byte + if _, err := io.ReadFull(r, hdr[:]); err != nil { + return 0, nil, err + } + n := binary.BigEndian.Uint32(hdr[1:]) + if n > MaxFrameBytes { + return 0, nil, ErrFrameTooLong + } + if n == 0 { + return FrameType(hdr[0]), nil, nil + } + buf := make([]byte, n) + if _, err := io.ReadFull(r, buf); err != nil { + // A truncated payload is a protocol error, not a clean end: report + // it as unexpected EOF so callers don't mistake it for the peer + // hanging up tidily between frames. + if errors.Is(err, io.EOF) { + return 0, nil, io.ErrUnexpectedEOF + } + return 0, nil, err + } + return FrameType(hdr[0]), buf, nil +} + +// ──────────────────────────────────────────────────────────────────────── +// Handshake lines +// ──────────────────────────────────────────────────────────────────────── + +// MaxLineBytes caps one handshake line. Snapshots are a few dozen bytes per +// device; the cap exists so a peer can't make the reader allocate without +// bound. +const MaxLineBytes = 64 * 1024 + +// ErrLineTooLong reports a handshake line past MaxLineBytes. The connection +// must be closed — the reader is out of sync with the framing. +var ErrLineTooLong = errors.New("serialfwd: control line exceeds MaxLineBytes") + +// WriteLine JSON-encodes v and writes it as one newline-terminated line. +func WriteLine(w io.Writer, v any) error { + b, err := json.Marshal(v) + if err != nil { + return fmt.Errorf("serialfwd: encoding %T: %w", v, err) + } + if len(b)+1 > MaxLineBytes { + return ErrLineTooLong + } + _, err = w.Write(append(b, '\n')) + return err +} + +// ReadLine reads one newline-terminated JSON line into v. The reader must +// be the one the caller keeps using — see ReadFrame. +func ReadLine(r *bufio.Reader, v any) error { + line, err := r.ReadSlice('\n') + if errors.Is(err, bufio.ErrBufferFull) { + return ErrLineTooLong + } + if err != nil { + return err + } + if err := json.Unmarshal(line, v); err != nil { + return fmt.Errorf("serialfwd: decoding %T: %w", v, err) + } + return nil +} + +// NewReader wraps r with a reader sized for MaxLineBytes, so ReadLine's +// buffer-full case really does mean "line too long" rather than "buffer too +// small". +func NewReader(r io.Reader) *bufio.Reader { return bufio.NewReaderSize(r, MaxLineBytes) } + +// ValidDeviceName reports whether name is usable as a guest device name. +// +// The guest creates /dev/, so anything with a directory part, a +// relative-path element, or a leading dot is refused — the host validates +// on the way in and the guest validates again on the way out, because this +// is the one field that becomes a filesystem path on the far side. +func ValidDeviceName(name string) error { + switch { + case name == "": + return errors.New("serial: device name is empty") + case len(name) > 64: + return fmt.Errorf("serial: device name %q is longer than 64 characters", name) + case strings.ContainsAny(name, "/\\"): + return fmt.Errorf("serial: device name %q must not contain a path separator", name) + case strings.HasPrefix(name, "."): + return fmt.Errorf("serial: device name %q must not start with a dot", name) + } + for _, r := range name { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + case r == '.' || r == '-' || r == '_': + default: + return fmt.Errorf("serial: device name %q contains %q; "+ + "use letters, digits, dot, dash or underscore", name, r) + } + } + return nil +} diff --git a/internal/serialfwd/serialfwd_test.go b/internal/serialfwd/serialfwd_test.go new file mode 100644 index 0000000..0334662 --- /dev/null +++ b/internal/serialfwd/serialfwd_test.go @@ -0,0 +1,182 @@ +package serialfwd + +import ( + "bytes" + "encoding/binary" + "errors" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestFrameRoundTrip(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, WriteFrame(&buf, FrameData, []byte("hello"))) + require.NoError(t, WriteModeFrame(&buf, Mode{Baud: 115200, Bits: 8, Parity: ParityNone, Stop: 1})) + require.NoError(t, WriteFrame(&buf, FrameData, nil)) + + r := NewReader(&buf) + + typ, payload, err := ReadFrame(r) + require.NoError(t, err) + require.Equal(t, FrameData, typ) + require.Equal(t, []byte("hello"), payload) + + typ, payload, err = ReadFrame(r) + require.NoError(t, err) + require.Equal(t, FrameMode, typ) + require.JSONEq(t, `{"baud":115200,"bits":8,"parity":"n","stop":1}`, string(payload)) + + // An empty data frame is legal and must not be confused with EOF. + typ, payload, err = ReadFrame(r) + require.NoError(t, err) + require.Equal(t, FrameData, typ) + require.Empty(t, payload) + + _, _, err = ReadFrame(r) + require.ErrorIs(t, err, io.EOF) +} + +// Frames and handshake lines share one connection, and the reader that read +// the greeting has already buffered whatever followed it. Reusing that +// reader is a documented requirement of the protocol, so it gets a test: +// the failure mode if it regresses is a first frame that silently vanishes. +func TestHandshakeReaderCarriesBufferedFrames(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, WriteLine(&buf, Greeting{Op: OpAttach, V: ProtoVersion, Name: "ttyACM0"})) + require.NoError(t, WriteFrame(&buf, FrameData, []byte("pipelined"))) + + r := NewReader(&buf) + var g Greeting + require.NoError(t, ReadLine(r, &g)) + require.Equal(t, OpAttach, g.Op) + require.Equal(t, "ttyACM0", g.Name) + + typ, payload, err := ReadFrame(r) + require.NoError(t, err) + require.Equal(t, FrameData, typ) + require.Equal(t, []byte("pipelined"), payload) +} + +func TestGreetingCarriesMode(t *testing.T) { + var buf bytes.Buffer + mode := Mode{Baud: 1200, Bits: 7, Parity: ParityEven, Stop: 2} + require.NoError(t, WriteLine(&buf, Greeting{ + Op: OpAttach, V: ProtoVersion, Name: "ttyACM0", Mode: &mode, + })) + + var got Greeting + require.NoError(t, ReadLine(NewReader(&buf), &got)) + require.NotNil(t, got.Mode) + require.Equal(t, mode, *got.Mode) + + // Absent rather than zero-valued when unset: a guest that doesn't state + // a mode must not be read as asking for 0 baud. + buf.Reset() + require.NoError(t, WriteLine(&buf, Greeting{Op: OpAttach, V: ProtoVersion, Name: "x"})) + require.NotContains(t, buf.String(), "mode") +} + +func TestReadFrameRejectsOversizedLength(t *testing.T) { + // A header claiming more than MaxFrameBytes must be refused before the + // allocation, not after. + var hdr [FrameHeaderBytes]byte + hdr[0] = byte(FrameData) + binary.BigEndian.PutUint32(hdr[1:], MaxFrameBytes+1) + + _, _, err := ReadFrame(NewReader(bytes.NewReader(hdr[:]))) + require.ErrorIs(t, err, ErrFrameTooLong) +} + +func TestWriteFrameRejectsOversizedPayload(t *testing.T) { + err := WriteFrame(io.Discard, FrameData, make([]byte, MaxFrameBytes+1)) + require.ErrorIs(t, err, ErrFrameTooLong) +} + +// A truncated payload is a broken peer, not a tidy hangup. Callers treat +// io.EOF between frames as "the other end went away cleanly", so a short +// read in the middle of one has to be distinguishable. +func TestReadFrameTruncatedPayload(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, WriteFrame(&buf, FrameData, []byte("0123456789"))) + truncated := buf.Bytes()[:FrameHeaderBytes+4] + + _, _, err := ReadFrame(NewReader(bytes.NewReader(truncated))) + require.ErrorIs(t, err, io.ErrUnexpectedEOF) +} + +func TestReadLineRejectsOversizedLine(t *testing.T) { + line := append(bytes.Repeat([]byte("x"), MaxLineBytes+10), '\n') + var g Greeting + err := ReadLine(NewReader(bytes.NewReader(line)), &g) + require.ErrorIs(t, err, ErrLineTooLong) +} + +func TestModeValidate(t *testing.T) { + require.NoError(t, DefaultMode().Validate()) + require.NoError(t, Mode{Baud: 115200, Bits: 8, Parity: ParityNone, Stop: 1}.Validate()) + require.NoError(t, Mode{Baud: 300, Bits: 5, Parity: ParityOdd, Stop: 2}.Validate()) + + for name, m := range map[string]Mode{ + "zero baud": {Baud: 0, Bits: 8, Parity: ParityNone, Stop: 1}, + "negative baud": {Baud: -1, Bits: 8, Parity: ParityNone, Stop: 1}, + "bits too few": {Baud: 9600, Bits: 4, Parity: ParityNone, Stop: 1}, + "bits too many": {Baud: 9600, Bits: 9, Parity: ParityNone, Stop: 1}, + "bad parity": {Baud: 9600, Bits: 8, Parity: "mark", Stop: 1}, + "empty parity": {Baud: 9600, Bits: 8, Parity: "", Stop: 1}, + "bad stop": {Baud: 9600, Bits: 8, Parity: ParityNone, Stop: 3}, + } { + t.Run(name, func(t *testing.T) { + require.Error(t, m.Validate()) + }) + } +} + +func TestModeString(t *testing.T) { + require.Equal(t, "115200-8N1", Mode{Baud: 115200, Bits: 8, Parity: ParityNone, Stop: 1}.String()) + require.Equal(t, "9600-7E2", Mode{Baud: 9600, Bits: 7, Parity: ParityEven, Stop: 2}.String()) +} + +// The guest turns this name into /dev/. Everything that could escape +// that directory, or land somewhere surprising inside it, has to be refused +// on the host before it is ever published. +func TestValidDeviceName(t *testing.T) { + for _, ok := range []string{"ttyACM0", "ttyUSB0", "cu.usbmodem1101", "arduino-uno", "a_b", "x"} { + require.NoError(t, ValidDeviceName(ok), "%q should be accepted", ok) + } + for _, bad := range []string{ + "", + "..", + ".hidden", + "../../etc/passwd", + "sub/dir", + `back\slash`, + "has space", + "null\x00byte", + "emoji✨", + strings.Repeat("a", 65), + } { + require.Error(t, ValidDeviceName(bad), "%q should be refused", bad) + } +} + +func TestFrameTypeString(t *testing.T) { + require.Equal(t, "data", FrameData.String()) + require.Equal(t, "mode", FrameMode.String()) + require.Equal(t, "unknown(0x7f)", FrameType(0x7f).String()) +} + +// WriteFrame emits the header and body as separate writes. A conn that +// reports a short write on the header must not leave the caller thinking +// the frame landed. +func TestWriteFrameSurfacesWriteErrors(t *testing.T) { + want := errors.New("conn closed") + err := WriteFrame(errWriter{want}, FrameData, []byte("x")) + require.ErrorIs(t, err, want) +} + +type errWriter struct{ err error } + +func (w errWriter) Write([]byte) (int, error) { return 0, w.err } diff --git a/internal/serialport/serialport.go b/internal/serialport/serialport.go new file mode 100644 index 0000000..936c342 --- /dev/null +++ b/internal/serialport/serialport.go @@ -0,0 +1,140 @@ +// Package serialport opens and configures a physical serial port on the +// host. It is the host end of `clawk serial` — internal/serialfwd carries +// the bytes, internal/cli/serial_proxy.go brokers them, and this is what +// actually talks to the tty. +// +// Only what forwarding needs is here: open a port (possibly named by a +// glob), put it in raw mode, apply a line configuration the guest asked +// for, and read and write bytes until someone closes it. No enumeration, +// no modem-line control, no flow control — see the serialfwd package +// comment for why the last of those can't be plumbed through a PTY anyway. +package serialport + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/clawkwork/clawk/internal/serialfwd" +) + +// Port is an open serial port. +// +// It embeds no lock: Read and Write may be called concurrently (they are, +// by the two halves of the proxy's pump), which is safe on an *os.File, but +// two concurrent Writes will interleave and two concurrent Reads will race +// for bytes. The proxy runs exactly one of each. +type Port struct { + f *os.File + // path is the resolved device, which for a glob is not what the user + // configured. Every log line uses this rather than the pattern, so a + // board that came back as usbmodem14201 says so. + path string +} + +// ErrNoMatch reports a device path that matched nothing. It is worth +// distinguishing because it is the ordinary "the board is unplugged, or is +// mid-reset" case, and the caller retries on it rather than giving up. +var ErrNoMatch = errors.New("serialport: no device matches") + +// Open resolves pattern, opens the device, and puts it in raw mode with +// mode's line configuration. +// +// pattern may be a literal path or a glob. A glob is resolved here, at open +// time, rather than when the device was configured: a board that reboots +// into its bootloader disappears from /dev and comes back — often under a +// neighbouring name — and re-globbing on each open is what lets a forward +// survive that. A glob matching several devices is an error rather than a +// guess, because picking the wrong board silently is worse than saying so. +func Open(pattern string, mode serialfwd.Mode) (*Port, error) { + if err := mode.Validate(); err != nil { + return nil, err + } + path, err := Resolve(pattern) + if err != nil { + return nil, err + } + + // O_NOCTTY: this is a daemon, and acquiring a controlling terminal + // would hand the board's line disciplines a say in our signal + // handling. + // + // O_NONBLOCK does two jobs. It stops the open itself from blocking on + // carrier detect (which /dev/tty.* on macOS does, unlike the /dev/cu.* + // callout device users should be naming), and it makes the descriptor + // pollable, so os.NewFile hands back a File registered with the Go + // runtime poller. That is what allows Close to unblock a Read parked + // waiting for a byte that may never come — the whole teardown path + // depends on it, so TestCloseUnblocksBlockedRead guards it. + fd, err := openNonblock(path) + if err != nil { + return nil, fmt.Errorf("serialport: opening %s: %w", path, err) + } + p := &Port{f: os.NewFile(uintptr(fd), path), path: path} + if p.f == nil { + _ = closeFD(fd) + return nil, fmt.Errorf("serialport: %s: invalid descriptor", path) + } + if err := p.Configure(mode); err != nil { + _ = p.f.Close() + return nil, err + } + return p, nil +} + +// Resolve turns a device pattern into a single concrete path. A literal +// path is returned once it is confirmed to exist, so an unplugged board +// fails here with ErrNoMatch rather than at open with a bare ENOENT. +func Resolve(pattern string) (string, error) { + if !isGlob(pattern) { + if _, err := os.Stat(pattern); err != nil { + if errors.Is(err, os.ErrNotExist) { + return "", fmt.Errorf("%w %s", ErrNoMatch, pattern) + } + return "", fmt.Errorf("serialport: %s: %w", pattern, err) + } + return pattern, nil + } + matches, err := filepath.Glob(pattern) + if err != nil { + return "", fmt.Errorf("serialport: bad device pattern %q: %w", pattern, err) + } + switch len(matches) { + case 0: + return "", fmt.Errorf("%w %s", ErrNoMatch, pattern) + case 1: + return matches[0], nil + default: + sort.Strings(matches) + return "", fmt.Errorf( + "serialport: pattern %s matches %d devices (%s) — narrow it so it names one", + pattern, len(matches), strings.Join(matches, ", ")) + } +} + +func isGlob(s string) bool { return strings.ContainsAny(s, "*?[") } + +// Configure applies mode to the open port, leaving it in raw mode. +func (p *Port) Configure(mode serialfwd.Mode) error { + if err := mode.Validate(); err != nil { + return err + } + if err := applyMode(int(p.f.Fd()), mode); err != nil { + return fmt.Errorf("serialport: configuring %s to %s: %w", p.path, mode, err) + } + return nil +} + +// Path is the resolved device this port is open on. +func (p *Port) Path() string { return p.path } + +func (p *Port) Read(b []byte) (int, error) { return p.f.Read(b) } +func (p *Port) Write(b []byte) (int, error) { return p.f.Write(b) } + +// Close releases the port. On a tty configured by Open this lowers DTR, +// which on a board wired for auto-reset is half of the reset pulse the next +// Open completes — see the serialfwd package comment. +func (p *Port) Close() error { return p.f.Close() } diff --git a/internal/serialport/serialport_other.go b/internal/serialport/serialport_other.go new file mode 100644 index 0000000..6329d60 --- /dev/null +++ b/internal/serialport/serialport_other.go @@ -0,0 +1,20 @@ +//go:build !darwin && !linux + +package serialport + +import ( + "errors" + + "github.com/clawkwork/clawk/internal/serialfwd" +) + +// errUnsupported keeps the package compiling on platforms clawk doesn't +// host a VM on. Nothing reaches these: serial forwarding is driven by the +// vz daemon (darwin) and its tests run on darwin and linux. +var errUnsupported = errors.New("serialport: serial ports are not supported on this platform") + +func openNonblock(string) (int, error) { return -1, errUnsupported } + +func closeFD(int) error { return errUnsupported } + +func applyMode(int, serialfwd.Mode) error { return errUnsupported } diff --git a/internal/serialport/serialport_test.go b/internal/serialport/serialport_test.go new file mode 100644 index 0000000..e58d35d --- /dev/null +++ b/internal/serialport/serialport_test.go @@ -0,0 +1,188 @@ +//go:build darwin || linux + +package serialport + +import ( + "errors" + "io" + "os" + "path/filepath" + "testing" + "time" + + "github.com/clawkwork/clawk/internal/serialfwd" + "github.com/clawkwork/clawk/internal/serialport/serialporttest" + "github.com/stretchr/testify/require" +) + +func TestResolveLiteralPath(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "ttyFAKE0") + require.NoError(t, os.WriteFile(path, nil, 0o600)) + + got, err := Resolve(path) + require.NoError(t, err) + require.Equal(t, path, got) +} + +// An unplugged board is the common case, not an exceptional one — the proxy +// retries on ErrNoMatch instead of failing the attach outright, so the +// sentinel has to survive Resolve. +func TestResolveMissingLiteralIsErrNoMatch(t *testing.T) { + _, err := Resolve(filepath.Join(t.TempDir(), "nope")) + require.ErrorIs(t, err, ErrNoMatch) +} + +func TestResolveGlob(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "cu.usbmodem1101") + require.NoError(t, os.WriteFile(path, nil, 0o600)) + + got, err := Resolve(filepath.Join(dir, "cu.usbmodem*")) + require.NoError(t, err) + require.Equal(t, path, got) +} + +func TestResolveGlobMatchingNothing(t *testing.T) { + _, err := Resolve(filepath.Join(t.TempDir(), "cu.usbmodem*")) + require.ErrorIs(t, err, ErrNoMatch) +} + +// Two boards behind one pattern must be an error naming both. Silently +// picking the first would flash the wrong device, which is the single most +// expensive way this package could be wrong. +func TestResolveAmbiguousGlob(t *testing.T) { + dir := t.TempDir() + for _, n := range []string{"cu.usbmodem1101", "cu.usbmodem2201"} { + require.NoError(t, os.WriteFile(filepath.Join(dir, n), nil, 0o600)) + } + + _, err := Resolve(filepath.Join(dir, "cu.usbmodem*")) + require.Error(t, err) + require.NotErrorIs(t, err, ErrNoMatch) + require.Contains(t, err.Error(), "cu.usbmodem1101") + require.Contains(t, err.Error(), "cu.usbmodem2201") +} + +func TestOpenAppliesModeAndPumpsBytes(t *testing.T) { + master, slavePath := serialporttest.OpenPTYPair(t) + + port, err := Open(slavePath, serialfwd.Mode{ + Baud: 115200, Bits: 8, Parity: serialfwd.ParityNone, Stop: 1, + }) + require.NoError(t, err) + defer port.Close() + require.Equal(t, slavePath, port.Path()) + + // Device → host. Raw mode has to be in force for this to arrive + // untouched: with OPOST still set the tty layer would rewrite the \n. + _, err = master.WriteString("from board\n") + require.NoError(t, err) + + buf := make([]byte, 64) + n, err := readWithin(t, port, buf, 2*time.Second) + require.NoError(t, err) + require.Equal(t, "from board\n", string(buf[:n])) + + // Host → device. ECHO must be off, or this comes straight back at us + // and the next read sees its own bytes. + _, err = port.Write([]byte("to board\n")) + require.NoError(t, err) + + n, err = readWithin(t, master, buf, 2*time.Second) + require.NoError(t, err) + require.Equal(t, "to board\n", string(buf[:n])) +} + +// The proxy tears a port down by closing it, from a different goroutine +// than the one parked in Read. That only works because Open leaves the +// descriptor non-blocking and therefore registered with the Go poller; if +// that ever regresses the read blocks forever and the daemon leaks a +// goroutine per unplugged board. +func TestCloseUnblocksBlockedRead(t *testing.T) { + _, slavePath := serialporttest.OpenPTYPair(t) + + port, err := Open(slavePath, serialfwd.DefaultMode()) + require.NoError(t, err) + + readErr := make(chan error, 1) + go func() { + _, err := port.Read(make([]byte, 16)) + readErr <- err + }() + + // Give the read time to actually park before pulling the rug. + time.Sleep(100 * time.Millisecond) + require.NoError(t, port.Close()) + + select { + case err := <-readErr: + require.Error(t, err, "Read should fail once the port is closed") + case <-time.After(2 * time.Second): + t.Fatal("Read did not return after Close — the fd is not pollable") + } +} + +// Every mode the guest can legally ask for has to survive the round trip to +// the kernel. The baud rates matter most: 250000 and 460800 have no B +// constant on Linux, which is the whole reason applyMode goes through +// termios2/BOTHER there. +func TestConfigureAcceptsEveryLegalMode(t *testing.T) { + _, slavePath := serialporttest.OpenPTYPair(t) + port, err := Open(slavePath, serialfwd.DefaultMode()) + require.NoError(t, err) + defer port.Close() + + for _, mode := range []serialfwd.Mode{ + {Baud: 1200, Bits: 8, Parity: serialfwd.ParityNone, Stop: 1}, + {Baud: 9600, Bits: 7, Parity: serialfwd.ParityEven, Stop: 2}, + {Baud: 57600, Bits: 8, Parity: serialfwd.ParityOdd, Stop: 1}, + {Baud: 115200, Bits: 8, Parity: serialfwd.ParityNone, Stop: 1}, + {Baud: 250000, Bits: 8, Parity: serialfwd.ParityNone, Stop: 1}, + {Baud: 460800, Bits: 8, Parity: serialfwd.ParityNone, Stop: 1}, + {Baud: 921600, Bits: 8, Parity: serialfwd.ParityNone, Stop: 1}, + {Baud: 5, Bits: 5, Parity: serialfwd.ParityNone, Stop: 1}, + } { + t.Run(mode.String(), func(t *testing.T) { + require.NoError(t, port.Configure(mode)) + }) + } +} + +// A mode arrives from the guest, so a bad one is a wire-level input rather +// than a programming error. It must be refused before any ioctl, and before +// the device is opened at all. +func TestOpenRejectsInvalidModeBeforeTouchingTheDevice(t *testing.T) { + _, err := Open("/definitely/not/a/device", serialfwd.Mode{Baud: 0}) + require.Error(t, err) + require.NotErrorIs(t, err, ErrNoMatch, "should fail on the mode, not the path") +} + +func TestConfigureRejectsInvalidMode(t *testing.T) { + _, slavePath := serialporttest.OpenPTYPair(t) + port, err := Open(slavePath, serialfwd.DefaultMode()) + require.NoError(t, err) + defer port.Close() + + require.Error(t, port.Configure(serialfwd.Mode{Baud: 9600, Bits: 99, Parity: "n", Stop: 1})) +} + +// readWithin fails the test rather than hanging when a read doesn't land. +func readWithin(t *testing.T, r io.Reader, buf []byte, d time.Duration) (int, error) { + t.Helper() + type result struct { + n int + err error + } + ch := make(chan result, 1) + go func() { + n, err := r.Read(buf) + ch <- result{n, err} + }() + select { + case res := <-ch: + return res.n, res.err + case <-time.After(d): + return 0, errors.New("timed out waiting for read") + } +} diff --git a/internal/serialport/serialporttest/pty_darwin.go b/internal/serialport/serialporttest/pty_darwin.go new file mode 100644 index 0000000..816b02d --- /dev/null +++ b/internal/serialport/serialporttest/pty_darwin.go @@ -0,0 +1,50 @@ +package serialporttest + +import ( + "bytes" + "os" + "testing" + "unsafe" + + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +// OpenPTYPair is the darwin half of the Linux function of the same name — +// see there for what it is for. Darwin spells the three steps differently: +// TIOCPTYGRANT and TIOCPTYUNLK for grantpt/unlockpt, and TIOCPTYGNAME to +// read the slave path into a caller-supplied buffer, which x/sys/unix has +// no typed wrapper for. +func OpenPTYPair(t *testing.T) (master *os.File, slavePath string) { + t.Helper() + + // O_NONBLOCK so os.NewFile registers the master with the Go poller, + // which is what makes SetDeadline work on it — tests read from this end + // and need to fail rather than hang when nothing arrives. + fd, err := unix.Open("/dev/ptmx", unix.O_RDWR|unix.O_NOCTTY|unix.O_NONBLOCK, 0) + require.NoError(t, err, "opening /dev/ptmx") + master = os.NewFile(uintptr(fd), "/dev/ptmx") + t.Cleanup(func() { _ = master.Close() }) + + require.NoError(t, unix.IoctlSetInt(fd, unix.TIOCPTYGRANT, 0), "grantpt") + require.NoError(t, unix.IoctlSetInt(fd, unix.TIOCPTYUNLK, 0), "unlockpt") + + // TIOCPTYGNAME writes a NUL-terminated path into a 128-byte buffer. + var buf [128]byte + if _, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(fd), + uintptr(unix.TIOCPTYGNAME), uintptr(unsafe.Pointer(&buf[0]))); errno != 0 { + t.Fatalf("ptsname: %v", errno) + } + name, _, _ := bytes.Cut(buf[:], []byte{0}) + + return master, string(name) +} + +// Speed reads back the line speed currently set on fd. Darwin stores the +// rate numerically in the termios speed fields, so it needs no decoding. +func Speed(t *testing.T, fd int) int { + t.Helper() + tio, err := unix.IoctlGetTermios(fd, unix.TIOCGETA) + require.NoError(t, err) + return int(tio.Ospeed) +} diff --git a/internal/serialport/serialporttest/pty_linux.go b/internal/serialport/serialporttest/pty_linux.go new file mode 100644 index 0000000..7b7c6c1 --- /dev/null +++ b/internal/serialport/serialporttest/pty_linux.go @@ -0,0 +1,49 @@ +package serialporttest + +import ( + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +// OpenPTYPair returns an open PTY master and the path of its slave. +// +// A PTY is not a UART — it has no modem-control lines and does nothing with +// the baud rate — but it is a real tty, so every ioctl the serial code +// issues goes through the kernel's tty layer rather than a mock, and the +// termios state is genuinely shared between the two ends. That last part is +// what lets a test assert that a mode change actually landed. +func OpenPTYPair(t *testing.T) (master *os.File, slavePath string) { + t.Helper() + + // O_NONBLOCK so os.NewFile registers the master with the Go poller, + // which is what makes SetDeadline work on it — tests read from this end + // and need to fail rather than hang when nothing arrives. + fd, err := unix.Open("/dev/ptmx", unix.O_RDWR|unix.O_NOCTTY|unix.O_NONBLOCK, 0) + require.NoError(t, err, "opening /dev/ptmx") + master = os.NewFile(uintptr(fd), "/dev/ptmx") + t.Cleanup(func() { _ = master.Close() }) + + // unlockpt, then ptsname. + require.NoError(t, unix.IoctlSetPointerInt(fd, unix.TIOCSPTLCK, 0), "unlockpt") + n, err := unix.IoctlGetInt(fd, unix.TIOCGPTN) + require.NoError(t, err, "ptsname") + + return master, fmt.Sprintf("/dev/pts/%d", n) +} + +// Speed reads back the line speed currently set on fd, which for a PTY +// master is the speed its slave was configured to. +// +// Linux keeps the real rate in the termios2 speed fields whenever the CBAUD +// field says BOTHER, which is how internal/serialport sets every rate, so +// that is where this reads it from. +func Speed(t *testing.T, fd int) int { + t.Helper() + tio, err := unix.IoctlGetTermios(fd, unix.TCGETS2) + require.NoError(t, err) + return int(tio.Ospeed) +} diff --git a/internal/serialport/termios_darwin.go b/internal/serialport/termios_darwin.go new file mode 100644 index 0000000..6f8c520 --- /dev/null +++ b/internal/serialport/termios_darwin.go @@ -0,0 +1,46 @@ +package serialport + +import ( + "github.com/clawkwork/clawk/internal/serialfwd" + "golang.org/x/sys/unix" +) + +// applyMode configures the port through TIOCGETA/TIOCSETA. +// +// Darwin needs no equivalent of Linux's BOTHER dance: its B constants +// are the numeric rates themselves (B115200 is 115200), and c_ispeed and +// c_ospeed are plain speed_t fields, so assigning the requested baud +// directly is both correct for the standard rates and the way non-standard +// ones are expressed. A rate the driver can't divide down to is rejected by +// the driver at TIOCSETA, which is the error the caller wants anyway. +func applyMode(fd int, mode serialfwd.Mode) error { + t, err := unix.IoctlGetTermios(fd, unix.TIOCGETA) + if err != nil { + return err + } + + cbits, err := cflagBits(mode) + if err != nil { + return err + } + + t.Iflag &^= uint64(rawIflagClear) + t.Oflag &^= uint64(rawOflagClear) + t.Lflag &^= uint64(rawLflagClear) + t.Cflag &^= uint64(rawCflagClear) + t.Cflag |= uint64(rawCflagSet) | cbits + + t.Ispeed = uint64(mode.Baud) + t.Ospeed = uint64(mode.Baud) + + // See the Linux implementation for why VMIN=1/VTIME=0 is the only safe + // pairing here. + t.Cc[unix.VMIN] = 1 + t.Cc[unix.VTIME] = 0 + + // TIOCSETA applies immediately. The draining variants (TIOCSETAW / + // TIOCSETAF) would wait for output to flush, and a mode change arrives + // here precisely when a tool is mid-handshake with a board that may + // have stopped listening — blocking there would wedge the pump. + return unix.IoctlSetTermios(fd, unix.TIOCSETA, t) +} diff --git a/internal/serialport/termios_linux.go b/internal/serialport/termios_linux.go new file mode 100644 index 0000000..adfa48b --- /dev/null +++ b/internal/serialport/termios_linux.go @@ -0,0 +1,54 @@ +package serialport + +import ( + "github.com/clawkwork/clawk/internal/serialfwd" + "golang.org/x/sys/unix" +) + +// applyMode configures the port through the termios2 interface. +// +// TCGETS2/TCSETS2 rather than the plain TCGETS/TCSETS pair, and BOTHER +// rather than a B constant, so any baud rate works. The classic +// interface can only express the rates that have a constant, which leaves +// out everything from an ESP-PROG at 460800 to the 250000 that DMX and some +// 3D-printer firmwares use. With BOTHER set in the speed field the kernel +// reads the rate from c_ispeed/c_ospeed instead, and the driver rounds to +// whatever its hardware can divide down to. +// +// golang.org/x/sys/unix.Termios is already the termios2 layout on Linux — +// it carries Ispeed and Ospeed after Cc, which the kernel's `struct +// termios` does not — so the same struct serves both ioctls. +func applyMode(fd int, mode serialfwd.Mode) error { + t, err := unix.IoctlGetTermios(fd, unix.TCGETS2) + if err != nil { + return err + } + + cbits, err := cflagBits(mode) + if err != nil { + return err + } + + t.Iflag &^= uint32(rawIflagClear) + t.Oflag &^= uint32(rawOflagClear) + t.Lflag &^= uint32(rawLflagClear) + t.Cflag &^= uint32(rawCflagClear) + t.Cflag |= uint32(rawCflagSet) | uint32(cbits) + + // Speed: clear the CBAUD field, select BOTHER, and put the real rate in + // the dedicated fields. + t.Cflag = (t.Cflag &^ uint32(unix.CBAUD)) | uint32(unix.BOTHER) + t.Ispeed = uint32(mode.Baud) + t.Ospeed = uint32(mode.Baud) + + // VMIN=1/VTIME=0: a read blocks until at least one byte arrives and + // never returns early empty. The alternative (VTIME as a poll timeout) + // would surface as a zero-length read, which os.File reports as io.EOF + // — indistinguishable from the port going away. Blocking is safe here + // only because the descriptor is registered with the Go poller, so + // Close can still interrupt it. + t.Cc[unix.VMIN] = 1 + t.Cc[unix.VTIME] = 0 + + return unix.IoctlSetTermios(fd, unix.TCSETS2, t) +} diff --git a/internal/serialport/termios_unix.go b/internal/serialport/termios_unix.go new file mode 100644 index 0000000..4030e49 --- /dev/null +++ b/internal/serialport/termios_unix.go @@ -0,0 +1,77 @@ +//go:build darwin || linux + +package serialport + +import ( + "fmt" + + "github.com/clawkwork/clawk/internal/serialfwd" + "golang.org/x/sys/unix" +) + +// openNonblock opens a tty without acquiring it as a controlling terminal +// and without blocking on carrier. See the comment in Open for why both +// matter. +func openNonblock(path string) (int, error) { + return unix.Open(path, unix.O_RDWR|unix.O_NOCTTY|unix.O_NONBLOCK, 0) +} + +func closeFD(fd int) error { return unix.Close(fd) } + +// Raw-mode masks, as uint64 so one definition serves both platforms — +// darwin's termios flags are 64-bit and Linux's are 32-bit, and each +// applyMode narrows these to its own width. +// +// This is cfmakeraw plus the two bits a forwarded port needs on top of it: +// CLOCAL, because nothing here should care about a modem carrier that +// USB-serial adapters don't really have, and CREAD, because a port we +// can't read from is useless. +const ( + rawIflagClear = uint64(unix.IGNBRK | unix.BRKINT | unix.PARMRK | unix.ISTRIP | + unix.INLCR | unix.IGNCR | unix.ICRNL | unix.IXON) + rawOflagClear = uint64(unix.OPOST) + rawLflagClear = uint64(unix.ECHO | unix.ECHONL | unix.ICANON | unix.ISIG | unix.IEXTEN) + + // CRTSCTS goes off with the rest: hardware flow control on a forwarded + // port would stall on an RTS line the guest has no way to drive. + rawCflagClear = uint64(unix.CSIZE | unix.PARENB | unix.PARODD | unix.CSTOPB | unix.CRTSCTS) + + // HUPCL lowers DTR when the last descriptor closes. That is deliberate + // and load-bearing rather than incidental: it is what makes the next + // open a rising DTR edge, and so what resets an Arduino at exactly the + // moment a guest process opens the PTY. + rawCflagSet = uint64(unix.CLOCAL | unix.CREAD | unix.HUPCL) +) + +// cflagBits returns the character-size, parity and stop-bit c_cflag bits +// for mode. The caller has already validated mode, so the switches here are +// total; the default arms exist to make that a compile-time-checkable claim +// rather than an assumption. +func cflagBits(mode serialfwd.Mode) (uint64, error) { + var bits uint64 + switch mode.Bits { + case 5: + bits |= unix.CS5 + case 6: + bits |= unix.CS6 + case 7: + bits |= unix.CS7 + case 8: + bits |= unix.CS8 + default: + return 0, fmt.Errorf("serialport: unsupported character size %d", mode.Bits) + } + switch mode.Parity { + case serialfwd.ParityNone: + case serialfwd.ParityEven: + bits |= unix.PARENB + case serialfwd.ParityOdd: + bits |= unix.PARENB | unix.PARODD + default: + return 0, fmt.Errorf("serialport: unsupported parity %q", mode.Parity) + } + if mode.Stop == 2 { + bits |= unix.CSTOPB + } + return bits, nil +} diff --git a/internal/template/global.go b/internal/template/global.go new file mode 100644 index 0000000..8c476c3 --- /dev/null +++ b/internal/template/global.go @@ -0,0 +1,407 @@ +package template + +// The host-wide defaults layer: one clawk.mod outside any repo whose sandbox +// block supplies values for every sandbox created on this machine. It answers +// the "same file over and over" problem (clawkwork/clawk#14) — a kernel path, a +// token alias, a personal skill mount and a house rule are properties of the +// HOST, not of the repo they currently sit in. +// +// It is the lowest layer of the precedence chain: +// +// built-in defaults +// < ~/.config/clawk/clawk.mod (this file) +// < namespace +// < repo clawk.mod +// < clawk.mod. +// < flags +// +// Lists union with the global entries first, so a conflict message reads +// scope-outward; scalars are filled only where nothing narrower declared one. +// It is read at sandbox-create time like every other template, so editing it +// never retro-modifies existing sandboxes. + +import ( + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" +) + +// GlobalModEnvVar names the environment variable that overrides the host-wide +// defaults file outright. Pointing it at a file makes a run reproducible +// regardless of what the host happens to have in ~/.config; pointing it at a +// path that does not exist is an error rather than a silent fall-through, so a +// typo in CI surfaces. +// +// Named for the layer it overrides, so it pairs with --no-global and can't be +// misread as "the repo's clawk.mod", and prefixed like every other variable +// clawk reads (CLAWK_DEBUG, CLAWK_NET_MODE, CLAWK_MAX_VZ_DEVICES) so it +// doesn't collide with an unrelated ROOT_* in someone's shell. +const GlobalModEnvVar = "CLAWK_GLOBAL_MOD" + +// GlobalDisabled skips the host-wide layer entirely — wired to `--no-global`. +// A package var rather than a parameter threaded through nine loader +// signatures: it is written once from PersistentPreRunE before any load, and +// tests set and restore it around a call. +var GlobalDisabled bool + +// ErrNoGlobalMod reports that no host-wide defaults file exists. Not a +// failure — the overwhelmingly common case is a host that never wrote one. +var ErrNoGlobalMod = errors.New("no host-wide clawk.mod") + +// ErrGlobalMod marks every OTHER host-wide-layer failure: unreadable, +// unparseable, out-of-scope directive, two candidate locations. Callers that +// try loaders in sequence (see the cli's resolveSource, which walks workspace → +// standalone → bare-git-repo and treats a failure as "not this shape") must +// test for it and surface it instead of moving on: a broken defaults file is +// not a hint to try somewhere else, and degrading to "no defaults" would hand +// back a sandbox quietly missing half its configuration. +var ErrGlobalMod = errors.New("host-wide clawk.mod") + +// GlobalModPath resolves the host-wide defaults file, in order: +// +// $CLAWK_GLOBAL_MOD explicit override (must exist) +// $XDG_CONFIG_HOME/clawk/clawk.mod default ~/.config/clawk/clawk.mod +// ~/.clawk/clawk.mod compatibility fallback +// +// ~/.config is the primary home because this file is the one thing in clawk's +// footprint a user hand-edits, symlinks out of a dotfiles repo and would be +// annoyed to lose: ~/.clawk is disposable machine state (VM disks, an image +// cache, per-sandbox records, a live OAuth token) that people exclude from +// backups and delete to start clean. Config must not be collateral. +// +// Deliberately NOT os.UserConfigDir(): on darwin that is +// ~/Library/Application Support, which is the wrong place for a +// dotfile-managed text file. Honouring $XDG_CONFIG_HOME with a ~/.config +// fallback on every platform is what gh, git and nvim do on macOS, and what +// anyone writing this file expects. +// +// Both non-env locations present is an error, never a silent precedence pick. +func GlobalModPath() (string, error) { + if p := os.Getenv(GlobalModEnvVar); p != "" { + expanded, err := ExpandPath(p) + if err != nil { + return "", err + } + abs, err := filepath.Abs(expanded) + if err != nil { + return "", err + } + if !fileExists(abs) { + return "", fmt.Errorf("%s=%s: no such file", GlobalModEnvVar, p) + } + return abs, nil + } + + xdg := os.Getenv("XDG_CONFIG_HOME") + if xdg == "" { + home, err := os.UserHomeDir() + if err != nil { + return "", ErrNoGlobalMod + } + xdg = filepath.Join(home, ".config") + } + primary := filepath.Join(xdg, "clawk", RepoFileName) + + var legacy string + if home, err := os.UserHomeDir(); err == nil { + legacy = filepath.Join(home, ".clawk", RepoFileName) + } + + switch { + case fileExists(primary) && legacy != "" && fileExists(legacy): + return "", fmt.Errorf( + "two host-wide clawk.mod files: %s and %s — keep one (%s is the documented location) or set %s", + primary, legacy, primary, GlobalModEnvVar) + case fileExists(primary): + return primary, nil + case legacy != "" && fileExists(legacy): + return legacy, nil + } + return "", ErrNoGlobalMod +} + +// Global is a loaded host-wide defaults layer. +type Global struct { + // Path is the file it came from, for the note printed at create. + Path string + // Template is the file's sandbox block with every host-side path made + // absolute against Path's directory (see absolutiseHostPaths), so it can + // be merged under a repo template that resolves paths against its own + // root. + Template *Template + // Policies are `policy ( … )` blocks declared beside it — a + // personal policy library, registered by the create paths exactly like a + // repo's own. + Policies []PolicyDef + // ProfileMatched reports whether a clawk.mod. overlay beside the + // global file existed and was applied, so a profile satisfied only by the + // host-wide layer is not reported as matching nothing. + ProfileMatched bool +} + +// LoadGlobal is LoadGlobalWithProfile with no profile. +func LoadGlobal() (*Global, error) { return LoadGlobalWithProfile("") } + +// LoadGlobalWithProfile loads and validates the host-wide defaults layer, +// applying a clawk.mod. overlay beside it when profile is non-empty. +// Returns ErrNoGlobalMod when there is no such file (or --no-global was +// passed) — callers treat that as "no defaults", not as a failure. +func LoadGlobalWithProfile(profile string) (*Global, error) { + g, err := loadGlobal(profile) + if err != nil && !errors.Is(err, ErrNoGlobalMod) { + // Tagged so a loader ladder can tell "this layer is broken" from + // "this shape doesn't apply here" — see ErrGlobalMod. + return nil, fmt.Errorf("%w: %w", ErrGlobalMod, err) + } + return g, err +} + +// loadGlobal is the body of LoadGlobalWithProfile, split out so every failure +// path gets the ErrGlobalMod tag from one place. +func loadGlobal(profile string) (*Global, error) { + if GlobalDisabled { + return nil, ErrNoGlobalMod + } + path, err := GlobalModPath() + if err != nil { + return nil, err + } + f, err := loadFile(path) + if err != nil { + return nil, err + } + if err := validateGlobalFile(path, f); err != nil { + return nil, err + } + + g := &Global{Path: path, Template: f.Sandbox, Policies: f.Policies} + if g.Template == nil { + // A file carrying only policy blocks is legitimate: a personal policy + // library with no defaults of its own. + g.Template = &Template{} + } + + if profile != "" { + overlayPath := path + "." + profile + overFile, err := maybeParseOverlay(overlayPath) + if err != nil { + return nil, err + } + if overFile != nil { + if err := validateGlobalFile(overlayPath, overFile); err != nil { + return nil, err + } + g.ProfileMatched = true + g.Template.Merge(overFile.Sandbox) + g.Policies = append(g.Policies, overFile.Policies...) + } + } + + // After the overlay merge, so its entries are resolved too — an overlay + // lives beside the file it extends, so one directory covers both. + absolutiseHostPaths(g.Template, filepath.Dir(path)) + return g, nil +} + +// validateGlobalFile enforces the global scope: the file describes defaults +// for ANY sandbox, so anything that identifies a particular one is rejected by +// name rather than silently ignored. Mirrors how parseNamespaceBlock rejects +// sandbox-level directives and rejectLifecycleAtWorkspace rejects unwired +// hooks — a third scope in the same family. +func validateGlobalFile(path string, f *File) error { + if len(f.Namespaces) > 0 { + return fmt.Errorf( + "%s: 'namespace' blocks are not accepted in the host-wide clawk.mod — "+ + "it declares defaults for every sandbox, not named resources, and clawk "+ + "owns the namespace records itself (a hand-edited copy here would be overwritten)", + path) + } + tmpl := f.Sandbox + if tmpl == nil { + return nil + } + switch { + case tmpl.Name != "": + // The header name is the repo/phase label (it lands in Repo.Name and + // names worktrees), not the sandbox's name — so a name here would + // silently relabel every repo on the host. + return fmt.Errorf( + "%s: the host-wide clawk.mod's sandbox block must be anonymous — "+ + "write `sandbox ( … )`; a header name labels one repo's phases, which is meaningless for defaults", + path) + case len(tmpl.Includes) > 0: + return fmt.Errorf( + "%s: 'includes' declares a workspace root and cannot be host-wide — "+ + "it would pull those repos into every sandbox", + path) + case len(tmpl.OnDown) > 0: + return fmt.Errorf("%s: 'on down' is reserved and wired nowhere yet", path) + case len(tmpl.OnEnter) > 0: + return fmt.Errorf("%s: 'on enter' is reserved and wired nowhere yet", path) + } + return nil +} + +// absolutiseHostPaths rewrites every host-side relative path in tmpl to be +// absolute against dir — the global file's own directory. +// +// This is what lets the layer be merged as a plain base template: once +// resolved, a `files ( ./x )` or `agent ( instructions ./house-rules.md )` +// entry no longer needs to remember which file declared it, so the downstream +// compose steps (which resolve relative paths against the repo root, or the +// process CWD) reach the right file with no provenance plumbing. +// +// Guest-side paths, ~ / $HOME prefixes and URLs are left alone; expanding ~ +// here would hide the user's spelling from error messages for no gain. +func absolutiseHostPaths(tmpl *Template, dir string) { + if tmpl == nil { + return + } + for i := range tmpl.Files { + tmpl.Files[i].HostPath = absHostPath(dir, tmpl.Files[i].HostPath) + } + for i := range tmpl.Shares { + tmpl.Shares[i].HostPath = absHostPath(dir, tmpl.Shares[i].HostPath) + } + for i := range tmpl.Serials { + tmpl.Serials[i].HostPath = absHostPath(dir, tmpl.Serials[i].HostPath) + } + for i := range tmpl.Instructions { + tmpl.Instructions[i].Path = absHostPath(dir, tmpl.Instructions[i].Path) + } + for i := range tmpl.Memory { + tmpl.Memory[i].Path = absHostPath(dir, tmpl.Memory[i].Path) + } + // `vm ( kernel … )` takes a path OR an http(s) URL; only the former moves. + if !isURL(tmpl.Kernel) { + tmpl.Kernel = absHostPath(dir, tmpl.Kernel) + } +} + +// absHostPath makes p absolute against dir, leaving empty values, absolute +// paths and home-relative spellings (~, $HOME) untouched. +func absHostPath(dir, p string) string { + if p == "" || filepath.IsAbs(p) { + return p + } + if strings.HasPrefix(p, "~") || strings.HasPrefix(p, "$HOME") { + return p + } + return filepath.Join(dir, p) +} + +// isURL reports whether s is an http(s) URL (a kernel may be either). +func isURL(s string) bool { + u, err := url.Parse(s) + return err == nil && (u.Scheme == "http" || u.Scheme == "https") +} + +// foldGlobalUnderOnlyRepo layers the global defaults under the sole repo's +// clawk.mod, so the repo's scalars win and its list entries follow the global +// ones. Repo.Clawkfile may be nil — a repo with no clawk.mod is exactly the +// case the host-wide layer exists for — and then the layer becomes its whole +// template. +// +// Only correct for a single-repo workspace: folding into N repos would +// duplicate agent instructions and run every global hook once per phase. See +// foldGlobalIntoWorkspace for the multi-repo shape. +func foldGlobalUnderOnlyRepo(ws *Workspace, g *Global) { + if len(ws.Repos) != 1 { + return + } + base := g.Template.Clone() + base.Merge(ws.Repos[0].Clawkfile) + ws.Repos[0].Clawkfile = base + // A repo whose name came from its own block header keeps it; the global + // file is required to be anonymous, so there is nothing to inherit. +} + +// foldGlobalIntoWorkspace layers the global defaults under a workspace root's +// own sandbox block. +// +// The workspace position is the right one for a multi-repo sandbox: its +// `files`/`shares`/`env` compose once for the VM and its `on up` / `on create` +// run at the guest workspace root, whereas folding the layer into each repo +// would duplicate agent instructions and run every hook once per phase. +// +// The catch is precedence. resolveProvider / resolveImage / resolveKernel and +// the resource resolvers consult ws.File FIRST, by design: a workspace root +// exists to settle disagreements between its repos. A default must not inherit +// that authority, so any scalar that arrived from the global layer is dropped +// again when some repo declares its own — leaving the global value to apply +// only where nothing narrower did. +// +// That decision is made against the FULL repo list, before any later +// FilterRepos (`--only`). So a global scalar yielded to a repo that a subsequent +// --only excludes stays yielded, and the provider default applies instead of +// the host's. Deliberate: re-deriving it per selection would make one repo's +// resolved shape depend on which siblings came along. +func foldGlobalIntoWorkspace(ws *Workspace, g *Global) { + own := ws.File // what the workspace file (plus its profile overlay) declared + base := g.Template.Clone() + base.Merge(own) + ws.File = base + + if own.Provider == "" && anyRepo(ws, func(t *Template) bool { return t.Provider != "" }) { + ws.File.Provider = "" + } + if own.Image == "" && anyRepo(ws, func(t *Template) bool { return t.Image != "" }) { + ws.File.Image = "" + } + if own.Kernel == "" && anyRepo(ws, func(t *Template) bool { return t.Kernel != "" }) { + ws.File.Kernel = "" + } + if own.CPU == 0 && anyRepo(ws, func(t *Template) bool { return t.CPU != 0 }) { + ws.File.CPU = 0 + } + if own.MemoryMiB == 0 && anyRepo(ws, func(t *Template) bool { return t.MemoryMiB != 0 }) { + ws.File.MemoryMiB = 0 + } + if own.MemoryMaxMiB == 0 && anyRepo(ws, func(t *Template) bool { return t.MemoryMaxMiB != 0 }) { + ws.File.MemoryMaxMiB = 0 + } + if own.DiskMiB == 0 && anyRepo(ws, func(t *Template) bool { return t.DiskMiB != 0 }) { + ws.File.DiskMiB = 0 + } + if own.SwapMiB == 0 && anyRepo(ws, func(t *Template) bool { return t.SwapMiB != 0 }) { + ws.File.SwapMiB = 0 + } + if own.IdleTimeoutSec == 0 && anyRepo(ws, func(t *Template) bool { return t.IdleTimeoutSec != 0 }) { + ws.File.IdleTimeoutSec = 0 + } + // Nested is deliberately left as the union: it is opt-in-only everywhere + // (there is no `nested false`), so a host that asks for it gets it. +} + +// anyRepo reports whether some repo's clawk.mod satisfies pred. +func anyRepo(ws *Workspace, pred func(*Template) bool) bool { + for _, r := range ws.Repos { + if r.Clawkfile != nil && pred(r.Clawkfile) { + return true + } + } + return false +} + +// attachGlobal loads the host-wide layer and hands it to fold, recording the +// path on the workspace for the create-time note. A missing file is not an +// error; anything else (unreadable, unparseable, out-of-scope directive) is — +// silently ignoring a broken defaults file would leave the user staring at a +// sandbox that quietly lacks half its configuration. +func attachGlobal(ws *Workspace, profile string, fold func(*Workspace, *Global)) error { + g, err := LoadGlobalWithProfile(profile) + if errors.Is(err, ErrNoGlobalMod) { + return nil + } + if err != nil { + return err + } + fold(ws, g) + ws.GlobalPath = g.Path + ws.GlobalProfileMatched = g.ProfileMatched + ws.Policies = append(g.Policies, ws.Policies...) + return nil +} diff --git a/internal/template/global_test.go b/internal/template/global_test.go new file mode 100644 index 0000000..7defd6f --- /dev/null +++ b/internal/template/global_test.go @@ -0,0 +1,371 @@ +package template + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// emptySandbox is the smallest valid global file: a sandbox block declaring +// nothing. Used by the path-resolution tests, which only care about location. +const emptySandbox = "sandbox (\n)\n" + +// withGlobalMod writes src as the host-wide clawk.mod inside a fresh +// XDG_CONFIG_HOME and enables the layer for the duration of the test. Returns +// the file's path. HOME is redirected too, so the ~/.clawk fallback can never +// resolve to the developer's real one. +func withGlobalMod(t *testing.T, src string) string { + t.Helper() + home := t.TempDir() + xdg := filepath.Join(home, ".config") + require.NoError(t, os.MkdirAll(filepath.Join(xdg, "clawk"), 0o755)) + path := filepath.Join(xdg, "clawk", RepoFileName) + require.NoError(t, os.WriteFile(path, []byte(src), 0o644)) + + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", xdg) + t.Setenv(GlobalModEnvVar, "") + + prev := GlobalDisabled + GlobalDisabled = false + t.Cleanup(func() { GlobalDisabled = prev }) + return path +} + +// newRepo creates an initialised git repo with an optional clawk.mod. +func newRepo(t *testing.T, name, clawkmod string) string { + t.Helper() + repo := filepath.Join(t.TempDir(), name) + require.NoError(t, os.MkdirAll(repo, 0o755)) + initRepo(t, repo) + if clawkmod != "" { + require.NoError(t, os.WriteFile( + filepath.Join(repo, RepoFileName), []byte(clawkmod), 0o644)) + } + return repo +} + +func TestGlobalModPathPrecedence(t *testing.T) { + t.Run("env var wins", func(t *testing.T) { + xdgPath := withGlobalMod(t, emptySandbox) + explicit := filepath.Join(t.TempDir(), "custom.mod") + require.NoError(t, os.WriteFile(explicit, []byte(emptySandbox), 0o644)) + t.Setenv(GlobalModEnvVar, explicit) + + got, err := GlobalModPath() + require.NoError(t, err) + require.Equal(t, explicit, got) + require.NotEqual(t, xdgPath, got) + }) + + t.Run("env var pointing nowhere is an error", func(t *testing.T) { + withGlobalMod(t, emptySandbox) + t.Setenv(GlobalModEnvVar, filepath.Join(t.TempDir(), "absent.mod")) + + _, err := GlobalModPath() + require.ErrorContains(t, err, GlobalModEnvVar) + require.ErrorContains(t, err, "no such file") + }) + + t.Run("xdg location", func(t *testing.T) { + path := withGlobalMod(t, emptySandbox) + got, err := GlobalModPath() + require.NoError(t, err) + require.Equal(t, path, got) + }) + + t.Run("legacy ~/.clawk fallback", func(t *testing.T) { + xdgPath := withGlobalMod(t, emptySandbox) + require.NoError(t, os.Remove(xdgPath)) + + legacy := filepath.Join(os.Getenv("HOME"), ".clawk", RepoFileName) + require.NoError(t, os.MkdirAll(filepath.Dir(legacy), 0o755)) + require.NoError(t, os.WriteFile(legacy, []byte(emptySandbox), 0o644)) + + got, err := GlobalModPath() + require.NoError(t, err) + require.Equal(t, legacy, got) + }) + + t.Run("both locations is an error, not a silent pick", func(t *testing.T) { + withGlobalMod(t, emptySandbox) + legacy := filepath.Join(os.Getenv("HOME"), ".clawk", RepoFileName) + require.NoError(t, os.MkdirAll(filepath.Dir(legacy), 0o755)) + require.NoError(t, os.WriteFile(legacy, []byte(emptySandbox), 0o644)) + + _, err := GlobalModPath() + require.ErrorContains(t, err, "two host-wide clawk.mod files") + }) + + t.Run("no file at all", func(t *testing.T) { + p := withGlobalMod(t, emptySandbox) + require.NoError(t, os.Remove(p)) + + _, err := GlobalModPath() + require.ErrorIs(t, err, ErrNoGlobalMod) + }) +} + +func TestLoadGlobalScopeRejections(t *testing.T) { + cases := []struct { + name, src, want string + }{ + { + name: "named sandbox block", + src: "sandbox house (\n vm (\n cpu 2\n )\n)\n", + want: "must be anonymous", + }, + { + name: "includes", + src: "sandbox (\n includes (\n ~/code/a\n )\n)\n", + want: "cannot be host-wide", + }, + { + name: "namespace block", + src: "namespace work (\n env (\n TOKEN\n )\n)\n", + want: "not accepted in the host-wide clawk.mod", + }, + { + name: "unwired lifecycle hook", + src: "sandbox (\n on down (\n \"echo bye\"\n )\n)\n", + want: "reserved", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + withGlobalMod(t, tc.src) + _, err := LoadGlobal() + require.ErrorContains(t, err, tc.want) + }) + } +} + +func TestLoadGlobalAbsolutisesHostPaths(t *testing.T) { + path := withGlobalMod(t, `sandbox ( + vm ( + kernel ./kernels/vmlinux + ) + files ( + ./house.netrc /home/agent/.netrc + ) + shares ( + ~/.aws + ) + agent ( + instructions ./house-rules.md + ) +) +`) + dir := filepath.Dir(path) + + g, err := LoadGlobal() + require.NoError(t, err) + require.Equal(t, filepath.Join(dir, "kernels", "vmlinux"), g.Template.Kernel) + require.Equal(t, filepath.Join(dir, "house.netrc"), g.Template.Files[0].HostPath) + require.Equal(t, filepath.Join(dir, "house-rules.md"), g.Template.Instructions[0].Path) + // A home-relative spelling is the user's own and stays verbatim, so error + // messages echo what they wrote. + require.Equal(t, "~/.aws", g.Template.Shares[0].HostPath) +} + +func TestLoadGlobalLeavesKernelURLAlone(t *testing.T) { + withGlobalMod(t, "sandbox (\n vm (\n kernel https://example.com/vmlinux\n )\n)\n") + g, err := LoadGlobal() + require.NoError(t, err) + require.Equal(t, "https://example.com/vmlinux", g.Template.Kernel) +} + +func TestLoadGlobalPolicyOnlyFile(t *testing.T) { + withGlobalMod(t, "policy house (\n allow example.com\n)\n") + g, err := LoadGlobal() + require.NoError(t, err) + require.NotNil(t, g.Template) + require.Len(t, g.Policies, 1) + require.Equal(t, "house", g.Policies[0].Name) +} + +func TestGlobalDisabledSkipsTheLayer(t *testing.T) { + withGlobalMod(t, "sandbox (\n vm (\n cpu 8\n )\n)\n") + GlobalDisabled = true + + _, err := LoadGlobal() + require.ErrorIs(t, err, ErrNoGlobalMod) +} + +// The single-repo shape: the layer folds under the repo's own clawk.mod, so the +// repo wins scalars and its list entries follow the global ones. +func TestGlobalUnderStandaloneRepo(t *testing.T) { + globalPath := withGlobalMod(t, `sandbox ( + vm ( + cpu 2 + memory_max 8GiB + provider vz + ) + network ( + allow global.example.com + ) + env ( + GITHUB_TOKEN = ${HOST_TOKEN} + ) + shares ( + ~/.claude/skills/idiomatic-go + ) +) +`) + repo := newRepo(t, "proj", `sandbox ( + vm ( + cpu 4 + ) + network ( + allow repo.example.com + ) +) +`) + + ws, err := LoadStandaloneClawkfile(repo) + require.NoError(t, err) + require.Equal(t, globalPath, ws.GlobalPath) + + tmpl := ws.Repos[0].Clawkfile + require.EqualValues(t, 4, tmpl.CPU, "repo cpu must beat the global default") + require.EqualValues(t, 8192, tmpl.MemoryMaxMiB, "global memory applies where the repo is silent") + require.Equal(t, "vz", tmpl.Provider) + require.Equal(t, []string{"global.example.com", "repo.example.com"}, tmpl.Domains, + "lists union with the global entries first") + require.Equal(t, []string{"GITHUB_TOKEN=${HOST_TOKEN}"}, tmpl.Env) + require.Len(t, tmpl.Shares, 1) + // The repo's own name is untouched by the (necessarily anonymous) layer. + require.Equal(t, "proj", ws.Repos[0].Name) +} + +func TestGlobalIsWholeTemplateForRepoWithoutClawkMod(t *testing.T) { + withGlobalMod(t, `sandbox ( + vm ( + cpu 3 + ) + network ( + allow global.example.com + ) +) +`) + repo := newRepo(t, "bare", "") + + ws, err := WorkspaceFromGitRepo(repo) + require.NoError(t, err) + require.NotNil(t, ws.Repos[0].Clawkfile) + require.EqualValues(t, 3, ws.Repos[0].Clawkfile.CPU) + require.Equal(t, []string{"global.example.com"}, ws.Repos[0].Clawkfile.Domains) +} + +// In a multi-repo workspace the layer sits at the workspace position, but must +// not inherit the workspace file's authority to settle repo disagreements. +func TestGlobalScalarsDemotedBehindRepos(t *testing.T) { + withGlobalMod(t, `sandbox ( + vm ( + image golang:1.25 + cpu 2 + memory_max 8GiB + ) +) +`) + + root := t.TempDir() + repo := filepath.Join(root, "svc") + require.NoError(t, os.MkdirAll(repo, 0o755)) + initRepo(t, repo) + require.NoError(t, os.WriteFile(filepath.Join(repo, RepoFileName), []byte(`sandbox ( + vm ( + image node:22 + cpu 6 + ) +) +`), 0o644)) + + wsPath := filepath.Join(root, RepoFileName) + require.NoError(t, os.WriteFile(wsPath, + []byte("sandbox (\n includes (\n ./svc\n )\n)\n"), 0o644)) + + ws, err := LoadWorkspace(wsPath) + require.NoError(t, err) + require.Empty(t, ws.File.Image, "a global image must not outrank the repo's own") + require.Zero(t, ws.File.CPU, "a global cpu must not outrank the repo's own") + require.EqualValues(t, 8192, ws.File.MemoryMaxMiB, + "but it still applies where no repo declared one") +} + +func TestGlobalAtWorkspacePositionUnderTheWorkspaceFile(t *testing.T) { + withGlobalMod(t, `sandbox ( + vm ( + memory_max 8GiB + ) + network ( + allow global.example.com + ) + env ( + GLOBAL_TOKEN + ) + on up ( + "global-hook" + ) +) +`) + root := t.TempDir() + repo := filepath.Join(root, "svc") + require.NoError(t, os.MkdirAll(repo, 0o755)) + initRepo(t, repo) + + wsPath := filepath.Join(root, RepoFileName) + require.NoError(t, os.WriteFile(wsPath, []byte(`sandbox ( + includes ( + ./svc + ) + vm ( + memory_max 16GiB + ) + on up ( + "workspace-hook" + ) +) +`), 0o644)) + + ws, err := LoadWorkspace(wsPath) + require.NoError(t, err) + require.EqualValues(t, 16384, ws.File.MemoryMaxMiB, "the workspace file wins") + require.Equal(t, []string{"global.example.com"}, ws.File.Domains) + require.Equal(t, []string{"GLOBAL_TOKEN"}, ws.File.Env) + require.Equal(t, []string{"global-hook", "workspace-hook"}, ws.File.OnUp, + "the broader scope's hooks run first") +} + +func TestGlobalProfileOverlay(t *testing.T) { + path := withGlobalMod(t, "sandbox (\n network (\n allow base.example.com\n )\n)\n") + require.NoError(t, os.WriteFile(path+".investigation", + []byte("sandbox (\n network (\n allow deep.example.com\n )\n)\n"), 0o644)) + + repo := newRepo(t, "proj", emptySandbox) + + // The repo has no overlay of its own: the profile is satisfied entirely by + // the host-wide layer, which must count as a match rather than erroring. + ws, err := LoadStandaloneClawkfileWithProfile(repo, "investigation") + require.NoError(t, err) + require.Equal(t, []string{"base.example.com", "deep.example.com"}, + ws.Repos[0].Clawkfile.Domains) + + // An unknown profile still fails loudly. + _, err = LoadStandaloneClawkfileWithProfile(repo, "nope") + require.ErrorContains(t, err, "nope") +} + +func TestGlobalParseErrorSurfaces(t *testing.T) { + withGlobalMod(t, "sandbox (\n vm (\n cpu\n )\n)\n") + repo := newRepo(t, "proj", emptySandbox) + + // A broken defaults file must not degrade to "no defaults" — that would + // leave the user with a sandbox quietly missing half its configuration. + _, err := LoadStandaloneClawkfile(repo) + require.Error(t, err) + require.True(t, strings.Contains(err.Error(), "cpu"), "got %v", err) +} diff --git a/internal/template/main_test.go b/internal/template/main_test.go new file mode 100644 index 0000000..75ee7ce --- /dev/null +++ b/internal/template/main_test.go @@ -0,0 +1,16 @@ +package template + +import ( + "os" + "testing" +) + +// TestMain disables the host-wide clawk.mod for the whole package. Without +// this, every loader test would silently pick up whatever the developer (or +// CI runner) has in ~/.config/clawk/clawk.mod and assert against a moving +// target. Tests that mean to exercise the layer re-enable it through +// withGlobalMod (see global_test.go). +func TestMain(m *testing.M) { + GlobalDisabled = true + os.Exit(m.Run()) +} diff --git a/internal/template/mcp_test.go b/internal/template/mcp_test.go new file mode 100644 index 0000000..bd94f60 --- /dev/null +++ b/internal/template/mcp_test.go @@ -0,0 +1,155 @@ +package template + +import ( + "testing" + + "github.com/clawkwork/clawk/internal/config" + "github.com/stretchr/testify/require" +) + +// TestParseMCPBlock covers the three line shapes an `mcp ( … )` entry can +// take, plus the repeatable modifiers. The bare-URL form is the one users +// write most, so its http default is pinned explicitly. +func TestParseMCPBlock(t *testing.T) { + src := `mcp ( + linear https://mcp.linear.app/mcp header "Authorization: Bearer ${LINEAR_TOKEN}" + sentry sse https://mcp.sentry.dev/sse + corridor http https://app.corridor.dev/api/mcp header "X-Api-Key: ${CORRIDOR_KEY}" header "X-Env: prod" + github stdio "npx -y @modelcontextprotocol/server-github" env GITHUB_TOKEN +) +` + tmpl, err := parseBody(src) + require.NoError(t, err) + require.Len(t, tmpl.MCP, 4) + + linear := tmpl.MCP[0] + require.Equal(t, "linear", linear.Name) + require.Equal(t, config.MCPTransportHTTP, linear.Transport, "a bare URL defaults to http") + require.Equal(t, "https://mcp.linear.app/mcp", linear.URL) + require.Equal(t, []string{"Authorization: Bearer ${LINEAR_TOKEN}"}, linear.Headers) + + require.Equal(t, config.MCPTransportSSE, tmpl.MCP[1].Transport) + require.Equal(t, "https://mcp.sentry.dev/sse", tmpl.MCP[1].URL) + + corridor := tmpl.MCP[2] + require.Equal(t, config.MCPTransportHTTP, corridor.Transport) + require.Equal(t, []string{"X-Api-Key: ${CORRIDOR_KEY}", "X-Env: prod"}, corridor.Headers, + "header is repeatable and order-preserving") + + gh := tmpl.MCP[3] + require.Equal(t, config.MCPTransportStdio, gh.Transport) + require.Equal(t, []string{"npx", "-y", "@modelcontextprotocol/server-github"}, gh.Command) + require.Equal(t, []string{"GITHUB_TOKEN"}, gh.Env) + require.Empty(t, gh.URL) +} + +// TestParseMCPKeepsCredentialReferencesLiteral is the invariant that keeps +// secrets off host disk: a ${VAR} in a header is stored as written, never +// resolved at parse time. The runner expands it inside the guest. +func TestParseMCPKeepsCredentialReferencesLiteral(t *testing.T) { + t.Setenv("LINEAR_TOKEN", "should-not-be-read") + tmpl, err := parseBody(`mcp ( + linear https://mcp.linear.app/mcp header "Authorization: Bearer ${LINEAR_TOKEN}" +) +`) + require.NoError(t, err) + require.Equal(t, []string{"Authorization: Bearer ${LINEAR_TOKEN}"}, tmpl.MCP[0].Headers) +} + +func TestParseMCPErrors(t *testing.T) { + cases := []struct { + name string + src string + want string + }{ + { + name: "no target", + src: "mcp (\n linear\n)\n", + want: "expected a URL or a transport", + }, + { + name: "non-http scheme", + src: "mcp (\n linear ftp://mcp.linear.app/mcp\n)\n", + want: "must use http or https", + }, + { + name: "url without host", + src: "mcp (\n linear https:///mcp\n)\n", + want: "has no host", + }, + { + name: "stdio without quoted command", + src: "mcp (\n github stdio npx\n)\n", + want: "expected a quoted command", + }, + { + name: "header without colon", + src: "mcp (\n linear https://mcp.linear.app/mcp header \"Authorization Bearer x\"\n)\n", + want: "want \"Name: value\"", + }, + { + name: "header on stdio server", + src: "mcp (\n github stdio \"npx server\" header \"X: y\"\n)\n", + want: "'header' applies to http/sse servers", + }, + { + name: "env on remote server", + src: "mcp (\n linear https://mcp.linear.app/mcp env LINEAR_TOKEN\n)\n", + want: "'env' applies to stdio servers", + }, + { + name: "invalid env name", + src: "mcp (\n github stdio \"npx server\" env 9BAD\n)\n", + want: "9BAD", + }, + { + name: "invalid server name", + src: "mcp (\n my:server https://mcp.linear.app/mcp\n)\n", + want: "invalid mcp server name", + }, + { + name: "trailing junk", + src: "mcp (\n linear https://mcp.linear.app/mcp wat\n)\n", + want: "unexpected token after mcp entry", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := parseBody(tc.src) + require.Error(t, err) + require.Contains(t, err.Error(), tc.want) + }) + } +} + +// TestParseMCPInNamespaceBlock guards the namespace scope: the org-wide +// server set is the whole reason per-sandbox MCP setup can be zero-touch, +// so `mcp ( … )` has to be accepted there too. +func TestParseMCPInNamespaceBlock(t *testing.T) { + f, err := ParseFileString(`namespace acme ( + mcp ( + linear https://mcp.linear.app/mcp header "Authorization: Bearer ${LINEAR_TOKEN}" + ) +) +`) + require.NoError(t, err) + require.Len(t, f.Namespaces, 1) + require.Len(t, f.Namespaces[0].Template.MCP, 1) + require.Equal(t, "linear", f.Namespaces[0].Template.MCP[0].Name) +} + +// A URL with userinfo is the one way a secret VALUE could reach the sandbox +// record and the rendered guest config, both of which live unencrypted on +// host disk — the invariant the whole `${VAR}`-reference design exists to +// hold. Refuse it and name the supported spelling. +func TestMCPURLRejectsEmbeddedCredentials(t *testing.T) { + _, err := parseBody(`mcp ( + sentry https://user:s3cr3t@mcp.sentry.dev/mcp +) +`) + require.Error(t, err) + require.ErrorContains(t, err, "embeds credentials") + require.ErrorContains(t, err, "Authorization: Bearer") + require.NotContains(t, err.Error(), "s3cr3t", + "the error must not echo the secret it is refusing") +} diff --git a/internal/template/parse.go b/internal/template/parse.go index 4e2b83c..eb17dcc 100644 --- a/internal/template/parse.go +++ b/internal/template/parse.go @@ -5,11 +5,13 @@ import ( "fmt" "io/fs" "net/netip" + "net/url" "slices" "strconv" "strings" "time" + "github.com/clawkwork/clawk/internal/config" "github.com/clawkwork/clawk/internal/envspec" ) @@ -35,6 +37,23 @@ type FileSpec struct { Line, Col int } +// SerialSpec is one `serial (...)` entry: a host serial port presented as a +// device inside the guest. +// +// - HostPath: the device on this machine, e.g. /dev/cu.usbmodem1101. May +// be a glob, resolved when the port is opened rather than now. +// - GuestName: the bare name the device appears under in the guest's +// /dev. Empty defaults to the host device's basename — except for a +// glob, which has no basename and must name one explicitly. +type SerialSpec struct { + HostPath string + GuestName string + + // Line/Col record where the entry appeared so duplicate-name conflicts + // can be reported back to the right line. + Line, Col int +} + // ShareSpec is one entry in a `shares (...)` block: a host directory // live-mounted into the guest via virtio-fs. Edits on the host are // visible inside the guest without clawk involvement — the host owns @@ -50,6 +69,32 @@ type ShareSpec struct { Line, Col int } +// MCPSpec is one entry in an `mcp (...)` block: an MCP server the project +// wants available to the agent inside the sandbox. Written as +// +// http transport (the default) +// http|sse explicit remote transport +// stdio " " a local server process +// +// followed by any number of `header "Name: value"` and `env NAME` +// modifiers on the same line. +// +// Credential values never appear here — a `${VAR}` inside a header (or the +// implied `NAME=${NAME}` of an env entry) is kept verbatim and expanded by +// the runner inside the guest. See config.MCPServer. +type MCPSpec struct { + Name string + Transport string // config.MCPTransport* value + URL string + Command []string + Headers []string + Env []string + + // Line/Col record where the entry appeared so a conflict between two + // repos declaring the same server name is reported at the right line. + Line, Col int +} + // AgentDoc is one entry in an `agent (...)` block — a unit of persistent // instructions or memory seed. Exactly one field is set: Text is inline // markdown from a quoted string (one line); Path is a markdown file, relative @@ -129,6 +174,14 @@ type Template struct { // See ShareSpec for semantics. Shares []ShareSpec + // Serials is the forwarded serial-port list (`serial (...)`). + // See SerialSpec for semantics. + Serials []SerialSpec + + // MCP is the declared MCP server list (`mcp (...)`). + // See MCPSpec for semantics. + MCP []MCPSpec + // Instructions and Memory come from the `agent ( ... )` block: extra // persistent CLAUDE.md guidance and a baseline auto-memory seed. Each is // an ordered list of AgentDocs (inline text or a markdown file) resolved @@ -165,6 +218,13 @@ type Template struct { // on the rootfs. DiskMiB uint64 + // SwapMiB is the guest swap device's capacity in mebibytes, declared as + // `swap ` inside the `vm ( ... )` block. Zero = unset (the + // built-in sandbox.DefaultSwapSizeMiB applies); negative = "off", no swap + // device. The device is sparse, so the size bounds how much the guest may + // swap rather than reserving anything up front. + SwapMiB int64 + // IdleTimeoutSec is the sandbox's idle-stop timeout in seconds: how // long the VM may sit with no attached session and a quiescent guest // before the daemon stops it to reclaim host memory. Declared as @@ -224,6 +284,8 @@ func (t *Template) Merge(over *Template) { t.Skills = mergeSkills(t.Skills, over.Skills) t.Files = append(t.Files, over.Files...) t.Shares = append(t.Shares, over.Shares...) + t.Serials = append(t.Serials, over.Serials...) + t.MCP = append(t.MCP, over.MCP...) t.Instructions = append(t.Instructions, over.Instructions...) t.Memory = append(t.Memory, over.Memory...) // Bool fields: OR so a profile can only enable, never disable. @@ -245,11 +307,47 @@ func (t *Template) Merge(over *Template) { if over.DiskMiB != 0 { t.DiskMiB = over.DiskMiB } + if over.SwapMiB != 0 { + t.SwapMiB = over.SwapMiB + } if over.IdleTimeoutSec != 0 { t.IdleTimeoutSec = over.IdleTimeoutSec } } +// Clone returns a deep-enough copy of t for Merge to write into without +// touching the original's backing arrays. Every field is either a scalar or a +// slice of values, so copying the slices is sufficient — used by the host-wide +// defaults layer, which is loaded once and merged under several templates. +func (t *Template) Clone() *Template { + if t == nil { + return &Template{} + } + c := *t + c.Includes = slices.Clone(t.Includes) + c.Domains = slices.Clone(t.Domains) + c.IPs = slices.Clone(t.IPs) + c.Use = slices.Clone(t.Use) + c.DenyDomains = slices.Clone(t.DenyDomains) + c.DenyIPs = slices.Clone(t.DenyIPs) + c.DenySources = slices.Clone(t.DenySources) + c.Forwards = slices.Clone(t.Forwards) + c.ReverseForwards = slices.Clone(t.ReverseForwards) + c.Env = slices.Clone(t.Env) + c.OnCreate = slices.Clone(t.OnCreate) + c.OnUp = slices.Clone(t.OnUp) + c.OnDown = slices.Clone(t.OnDown) + c.OnEnter = slices.Clone(t.OnEnter) + c.Skills = slices.Clone(t.Skills) + c.Files = slices.Clone(t.Files) + c.Shares = slices.Clone(t.Shares) + c.Serials = slices.Clone(t.Serials) + c.MCP = slices.Clone(t.MCP) + c.Instructions = slices.Clone(t.Instructions) + c.Memory = slices.Clone(t.Memory) + return &c +} + type parser struct { toks []Token i int @@ -309,6 +407,10 @@ func (p *parser) parseTemplateDirective(tmpl *Template, t Token) error { return p.parseFilesBlock(tmpl) case "shares": return p.parseSharesBlock(tmpl) + case "serial": + return p.parseSerialBlock(tmpl) + case "mcp": + return p.parseMCPBlock(tmpl) case "agent": return p.parseAgentBlock(tmpl) case "env": @@ -336,8 +438,8 @@ func (p *parser) parseTemplateDirective(tmpl *Template, t Token) error { return p.errorAt(t, "unknown directive %q (want %s)", t.Val, describeFirst([]string{ - "vm", "network", "forwards", "files", "shares", - "agent", "skills", "on", "includes", "env", + "vm", "network", "forwards", "files", "shares", "serial", + "mcp", "agent", "skills", "on", "includes", "env", })) } } @@ -699,6 +801,44 @@ func (p *parser) parseIdleTimeout(tmpl *Template) error { return p.expectNewlineOrEOF() } +// parseSwap handles `swap ` inside the vm block — the capacity of +// the guest's swap device. Sizes use the same units as `memory` and `disk`; +// "off" and "0" disable swap entirely (stored as -1, so "explicitly off" +// stays distinguishable from "unset", exactly as idle_timeout does). +// +// A floor of 64 MiB rejects the unit typo `swap 512M` meant as 512 GiB less +// harshly than it rejects `swap 4` — anything smaller is too little to +// absorb a balloon inflation and is almost certainly a mistake. +func (p *parser) parseSwap(tmpl *Template) error { + p.advance() // consume "swap" + val := p.peek() + if val.Kind != TokIdent { + return p.errorAt(val, "expected size or 'off' after 'swap', got %s", val) + } + if tmpl.SwapMiB != 0 { + return p.errorAt(val, "duplicate 'swap' directive") + } + switch val.Val { + case "off", "0": + tmpl.SwapMiB = -1 + default: + mib, err := parseMiB(val.Val) + if err != nil { + return p.errorAt(val, "%q: %v", "swap", err) + } + if mib < minSwapMiB { + return p.errorAt(val, + "swap %q too small: minimum %d MiB (use 'off' to disable)", val.Val, minSwapMiB) + } + tmpl.SwapMiB = int64(mib) + } + p.advance() + return p.expectNewlineOrEOF() +} + +// minSwapMiB is the floor for an explicit `vm ( swap )`. +const minSwapMiB = 64 + // parseNested handles the bare `nested` directive. It takes no value — // setting Template.Nested true and requiring the line to end immediately // keeps the syntax unambiguous and matches how Go's `go 1.x` line works @@ -751,6 +891,10 @@ func (p *parser) parseVMBlock(tmpl *Template) error { if err := p.parseMemory(&tmpl.DiskMiB, "disk"); err != nil { return err } + case "swap": + if err := p.parseSwap(tmpl); err != nil { + return err + } case "nested": if err := p.parseNested(tmpl); err != nil { return err @@ -771,7 +915,7 @@ func (p *parser) parseVMBlock(tmpl *Template) error { return p.errorAt(t, "unknown 'vm' directive %q (want %s)", t.Val, describeFirst([]string{ - "provider", "cpu", "memory", "memory_max", "disk", "nested", "idle_timeout", "image", "kernel", + "provider", "cpu", "memory", "memory_max", "disk", "swap", "nested", "idle_timeout", "image", "kernel", })) } } @@ -1064,6 +1208,53 @@ func (p *parser) parseFilesBlock(tmpl *Template) error { } } +// parseSerialBlock handles `serial ( [] ... )`. +// Each entry is one or two whitespace-separated tokens on a line: +// +// serial ( +// /dev/cu.usbmodem1101 # same name inside the guest +// /dev/cu.usbserial-A50285BI ttyUSB0 +// /dev/cu.usbmodem* ttyACM0 # resolved when the port is opened +// ) +// +// Space-separated rather than the CLI's HOST:GUEST spelling, matching +// `files` and `shares`: these are paths, and a colon inside one would be +// ambiguous in a way a port number never is. +// +// Nothing is validated here beyond the shape — internal/cli turns these +// into config.SerialDevice and checks the names, the same division of +// labour as the other resource blocks. +func (p *parser) parseSerialBlock(tmpl *Template) error { + p.advance() // consume "serial" + t := p.peek() + if t.Kind != TokLParen { + return p.errorAt(t, "expected '(' after 'serial', got %s", t) + } + p.advance() + for { + p.skipNewlines() + t := p.peek() + if t.Kind == TokRParen { + p.advance() + return p.expectNewlineOrEOF() + } + if t.Kind != TokIdent { + return p.errorAt(t, "expected serial device path or ')', got %s", t) + } + spec := SerialSpec{HostPath: t.Val, Line: t.Line, Col: t.Col} + p.advance() + if nx := p.peek(); nx.Kind == TokIdent { + spec.GuestName = nx.Val + p.advance() + } + if next := p.peek(); next.Kind != TokNewline && next.Kind != TokRParen && next.Kind != TokEOF { + return p.errorAt(next, + "unexpected token after serial entry; one host device and optional guest name per line") + } + tmpl.Serials = append(tmpl.Serials, spec) + } +} + // parseSharesBlock handles `shares ( [] [ro|rw] ... )`. // Mount points default to ReadOnly = true: the host owns rotation for // the use cases we built this for (`~/.aws`), and an accidental in-VM @@ -1125,6 +1316,212 @@ func (p *parser) parseSharesBlock(tmpl *Template) error { } } +// parseMCPBlock handles `mcp ( [http|sse|stdio] [modifiers] ... )`. +// See MCPSpec for the accepted line shapes. +// +// The transport keyword is optional in the common case: a target that looks +// like an http(s) URL implies the http transport, which keeps the frequent +// one-line remote declaration short. stdio always needs its keyword, since a +// bare command word would otherwise be ambiguous with a server name. +func (p *parser) parseMCPBlock(tmpl *Template) error { + p.advance() // consume "mcp" + t := p.peek() + if t.Kind != TokLParen { + return p.errorAt(t, "expected '(' after 'mcp', got %s", t) + } + p.advance() + for { + p.skipNewlines() + t := p.peek() + if t.Kind == TokRParen { + p.advance() + return p.expectNewlineOrEOF() + } + if t.Kind != TokIdent { + return p.errorAt(t, "expected MCP server name or ')', got %s", t) + } + spec := MCPSpec{Name: t.Val, Line: t.Line, Col: t.Col} + if err := validateMCPName(spec.Name); err != nil { + return p.errorAt(t, "%v", err) + } + p.advance() + + if err := p.parseMCPTarget(&spec); err != nil { + return err + } + if err := p.parseMCPModifiers(&spec); err != nil { + return err + } + if next := p.peek(); next.Kind != TokNewline && next.Kind != TokRParen && next.Kind != TokEOF { + return p.errorAt(next, "unexpected token after mcp entry %q", spec.Name) + } + tmpl.MCP = append(tmpl.MCP, spec) + } +} + +// parseMCPTarget reads the transport and endpoint that follow a server name: +// an explicit `http|sse ` / `stdio ""`, or a bare URL taking +// the http default. +func (p *parser) parseMCPTarget(spec *MCPSpec) error { + t := p.peek() + if t.Kind != TokIdent { + return p.errorAt(t, + "expected a URL or a transport (http, sse, stdio) after mcp server %q, got %s", + spec.Name, t) + } + switch t.Val { + case config.MCPTransportHTTP, config.MCPTransportSSE: + spec.Transport = t.Val + p.advance() + return p.parseMCPURL(spec) + case config.MCPTransportStdio: + spec.Transport = config.MCPTransportStdio + p.advance() + cmd := p.peek() + if cmd.Kind != TokString { + return p.errorAt(cmd, + "expected a quoted command after 'stdio' for mcp server %q, got %s", + spec.Name, cmd) + } + argv := strings.Fields(cmd.Val) + if len(argv) == 0 { + return p.errorAt(cmd, "empty stdio command for mcp server %q", spec.Name) + } + spec.Command = argv + p.advance() + return nil + default: + // No transport keyword: the token must be the URL itself. + spec.Transport = config.MCPTransportHTTP + return p.parseMCPURL(spec) + } +} + +// parseMCPURL consumes and validates the endpoint of an http/sse server. +func (p *parser) parseMCPURL(spec *MCPSpec) error { + t := p.peek() + if t.Kind != TokIdent { + return p.errorAt(t, "expected a URL for mcp server %q, got %s", spec.Name, t) + } + if err := validateMCPURL(t.Val); err != nil { + return p.errorAt(t, "mcp server %q: %v", spec.Name, err) + } + spec.URL = t.Val + p.advance() + return nil +} + +// parseMCPModifiers reads the repeatable `header "..."` / `env NAME` trailers +// of one mcp entry, stopping at the end of the line. +func (p *parser) parseMCPModifiers(spec *MCPSpec) error { + for { + t := p.peek() + if t.Kind != TokIdent { + return nil + } + switch t.Val { + case "header": + p.advance() + v := p.peek() + if v.Kind != TokString { + return p.errorAt(v, + "expected a quoted \"Name: value\" after 'header' for mcp server %q, got %s", + spec.Name, v) + } + if err := validateMCPHeader(v.Val); err != nil { + return p.errorAt(v, "mcp server %q: %v", spec.Name, err) + } + if spec.Transport == config.MCPTransportStdio { + return p.errorAt(t, "mcp server %q: 'header' applies to http/sse servers, not stdio", spec.Name) + } + spec.Headers = append(spec.Headers, v.Val) + p.advance() + case "env": + p.advance() + v := p.peek() + if v.Kind != TokIdent { + return p.errorAt(v, + "expected a variable name after 'env' for mcp server %q, got %s", spec.Name, v) + } + if err := envspec.ValidateName(v.Val); err != nil { + return p.errorAt(v, "mcp server %q: %v", spec.Name, err) + } + if spec.Transport != config.MCPTransportStdio { + return p.errorAt(t, + "mcp server %q: 'env' applies to stdio servers; pass a credential to an "+ + "http/sse server with header \"Authorization: Bearer ${VAR}\"", spec.Name) + } + spec.Env = append(spec.Env, v.Val) + p.advance() + default: + // Not a modifier — leave it for the caller's end-of-entry check, + // which reports it against the entry as a whole. + return nil + } + } +} + +// validateMCPName rejects server names that would not survive the round trip +// into the guest's MCP config or the agent's tool namespace. Claude Code +// derives tool names from the server name, so keep it to the conservative +// set every runner tolerates. +func validateMCPName(name string) error { + if name == "" { + return errors.New("empty mcp server name") + } + for _, r := range name { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + case r == '-', r == '_', r == '.': + default: + return fmt.Errorf("invalid mcp server name %q: use letters, digits, '-', '_' or '.'", name) + } + } + return nil +} + +// validateMCPURL requires an absolute http(s) URL with a host and no +// credentials in it. Catching this at parse time means the +// network-derivation step downstream can assume it has a host to allow. +// +// Userinfo is refused rather than accepted-and-ignored because it is the one +// way to smuggle a secret VALUE past the design invariant that clawk stores +// only references: the URL is persisted verbatim onto the sandbox record and +// rendered into the guest MCP config, both of which sit unencrypted on host +// disk. `header "Authorization: Bearer ${VAR}"` is the supported spelling, +// and it keeps the value in the runner's process environment. See +// config.MCPServer. +func validateMCPURL(raw string) error { + u, err := url.Parse(raw) + if err != nil { + return fmt.Errorf("invalid URL %q: %w", raw, err) + } + if u.Scheme != "http" && u.Scheme != "https" { + return fmt.Errorf("URL %q must use http or https", raw) + } + if u.Hostname() == "" { + return fmt.Errorf("URL %q has no host", raw) + } + if u.User != nil { + return fmt.Errorf( + "URL %q embeds credentials — clawk would store them on disk; "+ + "drop the user:password@ and pass the credential with "+ + "header \"Authorization: Bearer ${VAR}\" instead", + u.Redacted()) + } + return nil +} + +// validateMCPHeader requires the "Name: value" spelling, so the rendered +// config never carries a header the runner will silently drop. +func validateMCPHeader(h string) error { + name, _, ok := strings.Cut(h, ":") + if !ok || strings.TrimSpace(name) == "" { + return fmt.Errorf("invalid header %q: want \"Name: value\"", h) + } + return nil +} + // parseAgentBlock handles `agent ( instructions ... | memory "..." )` — the // persistent agent context seeded into a sandbox. `instructions` accumulates // markdown guidance blocks (inline string or a parenthesised list); `memory` diff --git a/internal/template/parse_test.go b/internal/template/parse_test.go index 25ff50a..ea1c747 100644 --- a/internal/template/parse_test.go +++ b/internal/template/parse_test.go @@ -299,6 +299,12 @@ func TestParseResourceErrors(t *testing.T) { {"duplicate cpu", "cpu 2\ncpu 4\n", "duplicate"}, {"duplicate memory", "memory 1G\nmemory 2G\n", "duplicate"}, {"duplicate memory_max", "memory_max 1G\nmemory_max 2G\n", "duplicate"}, + {"duplicate swap", "swap 1G\nswap 2G\n", "duplicate"}, + {"bare swap number", "swap 4096", "unit suffix"}, + {"empty swap", "swap\n", "expected size or 'off'"}, + // The unit typo this floor exists for: `swap 512M` written meaning + // half a gigabyte is fine, but a bare small number is not. + {"swap below the floor", "swap 32M", "too small"}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -314,6 +320,33 @@ func TestParseResourceErrors(t *testing.T) { } } +func TestParseSwap(t *testing.T) { + cases := []struct { + src string + wantMiB int64 + }{ + {"swap 8GiB", 8192}, + {"swap 512M", 512}, + // "off" and "0" store -1, not 0: the zero value has to keep meaning + // "unset" so the built-in default still applies to everyone else. + {"swap off", -1}, + {"swap 0", -1}, + } + for _, c := range cases { + t.Run(c.src, func(t *testing.T) { + tmpl, err := parseBody("vm (\n" + c.src + "\n)\n") + require.NoError(t, err) + require.Equal(t, c.wantMiB, tmpl.SwapMiB) + }) + } + + t.Run("unset stays zero", func(t *testing.T) { + tmpl, err := parseBody("vm (\n cpu 2\n)\n") + require.NoError(t, err) + require.Zero(t, tmpl.SwapMiB) + }) +} + func TestParseIdleTimeout(t *testing.T) { cases := []struct { src string diff --git a/internal/template/resources.go b/internal/template/resources.go index 3ed1420..16addfb 100644 --- a/internal/template/resources.go +++ b/internal/template/resources.go @@ -57,7 +57,8 @@ type PolicyDef struct { } // NamespaceDef is one `namespace ( ... )` block: a named overlay of -// the per-namespace template subset (network / files / shares / env / agent). +// the per-namespace template subset (network / files / shares / mcp / env / +// agent). // VM shape, includes and lifecycle hooks are sandbox-level concerns and are // rejected inside a namespace body. type NamespaceDef struct { @@ -373,6 +374,8 @@ func (p *parser) parseNamespaceBlock() (NamespaceDef, error) { err = p.parseFilesBlock(def.Template) case "shares": err = p.parseSharesBlock(def.Template) + case "mcp": + err = p.parseMCPBlock(def.Template) case "env": err = p.parseEnvBlock(&def.Template.Env) case "agent": @@ -380,11 +383,11 @@ func (p *parser) parseNamespaceBlock() (NamespaceDef, error) { case "vm", "includes", "on": err = p.errorAt(t, "%q is a sandbox-level directive, not allowed in a namespace (want %s)", - t.Val, describeFirst([]string{"network", "files", "shares", "env", "agent"})) + t.Val, describeFirst([]string{"network", "files", "shares", "mcp", "env", "agent"})) default: err = p.errorAt(t, "unknown 'namespace' directive %q (want %s)", t.Val, - describeFirst([]string{"network", "files", "shares", "env", "agent"})) + describeFirst([]string{"network", "files", "shares", "mcp", "env", "agent"})) } if err != nil { return def, err diff --git a/internal/template/serial_test.go b/internal/template/serial_test.go new file mode 100644 index 0000000..ca2d4e7 --- /dev/null +++ b/internal/template/serial_test.go @@ -0,0 +1,51 @@ +package template + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParseSerialBlock(t *testing.T) { + tmpl, err := parseBody(`serial ( + /dev/cu.usbmodem1101 + /dev/cu.usbserial-A50285BI ttyUSB0 + /dev/cu.usbmodem* ttyACM0 +) +`) + require.NoError(t, err) + require.Len(t, tmpl.Serials, 3) + + require.Equal(t, "/dev/cu.usbmodem1101", tmpl.Serials[0].HostPath) + require.Empty(t, tmpl.Serials[0].GuestName, "a bare entry defaults its name downstream") + + require.Equal(t, "/dev/cu.usbserial-A50285BI", tmpl.Serials[1].HostPath) + require.Equal(t, "ttyUSB0", tmpl.Serials[1].GuestName) + + require.Equal(t, "/dev/cu.usbmodem*", tmpl.Serials[2].HostPath) + require.Equal(t, "ttyACM0", tmpl.Serials[2].GuestName) + + // Positions are recorded so a downstream name clash can point back at + // the line that caused it. + require.NotZero(t, tmpl.Serials[0].Line) +} + +func TestParseSerialBlockRejectsThirdToken(t *testing.T) { + _, err := parseBody("serial (\n /dev/cu.usbmodem1101 ttyACM0 extra\n)\n") + require.Error(t, err) + require.Contains(t, err.Error(), "one host device and optional guest name per line") +} + +func TestParseSerialBlockRequiresParen(t *testing.T) { + _, err := parseBody("serial /dev/cu.usbmodem1101\n") + require.Error(t, err) + require.Contains(t, err.Error(), "expected '(' after 'serial'") +} + +// The directive has to be listed in the unknown-directive hint, or a typo +// sends the user looking for a feature the error says doesn't exist. +func TestUnknownDirectiveMentionsSerial(t *testing.T) { + _, err := parseBody("serialz (\n /dev/cu.usbmodem1101\n)\n") + require.Error(t, err) + require.Contains(t, err.Error(), `"serial"`) +} diff --git a/internal/template/workspace.go b/internal/template/workspace.go index 1bebd45..451b184 100644 --- a/internal/template/workspace.go +++ b/internal/template/workspace.go @@ -18,11 +18,22 @@ type Workspace struct { Repos []Repo // every repo included by the workspace, in declaration order // Policies collects every `policy ( ... )` block declared across - // the loaded files — workspace file and its overlay first (broader - // scope), then each repo's clawk.mod and overlay. The create paths - // register them into the host policy store, so later (nearer) blocks - // win a name collision. + // the loaded files — the host-wide clawk.mod first, then the workspace + // file and its overlay (broader scope), then each repo's clawk.mod and + // overlay. The create paths register them into the host policy store, so + // later (nearer) blocks win a name collision. Policies []PolicyDef + + // GlobalPath is the host-wide clawk.mod folded in as the lowest layer, + // or "" when there is none. Reporting only — its directives have already + // been merged into File or a Repo's Clawkfile by the time a caller sees + // this. See global.go. + GlobalPath string + + // GlobalProfileMatched records that a clawk.mod. overlay beside + // the host-wide file was applied, so a profile satisfied only by the + // host-wide layer isn't reported as matching nothing. + GlobalProfileMatched bool } // Repo is one git repository brought into a sandbox by a workspace. The @@ -193,6 +204,14 @@ func loadWorkspaceParsed(abs string, f *File, profile string) (*Workspace, error if err := checkNameCollisions(ws.Repos); err != nil { return nil, err } + // The host-wide layer lands last, once the repos are known: fold needs + // them to decide which of its scalars a repo has already spoken for. + if err := attachGlobal(ws, profile, foldGlobalIntoWorkspace); err != nil { + return nil, err + } + if ws.GlobalProfileMatched { + profileMatched = true + } if profile != "" && !profileMatched { return nil, fmt.Errorf( "profile %q matched no overlay file (looked for %s.%s beside the workspace file and in each repo)", @@ -257,16 +276,23 @@ func LoadStandaloneClawkfileWithProfile(dir, profile string) (*Workspace, error) if err != nil { return nil, err } - if profile != "" && !matched { - return nil, fmt.Errorf("profile %q has no matching overlay in %s", - profile, abs) - } - return &Workspace{ + ws := &Workspace{ Root: repo.RepoPath, File: &Template{}, Repos: []Repo{repo}, Policies: policies, - }, nil + } + // One repo, so the host-wide layer folds straight under its clawk.mod: + // the repo's scalars win, its list entries follow the global ones, and its + // `on` hooks keep the per-phase position they have today. + if err := attachGlobal(ws, profile, foldGlobalUnderOnlyRepo); err != nil { + return nil, err + } + if profile != "" && !matched && !ws.GlobalProfileMatched { + return nil, fmt.Errorf("profile %q has no matching overlay in %s", + profile, abs) + } + return ws, nil } // LoadClawkfilePathWithProfile loads an explicitly-given clawk.mod path, @@ -302,7 +328,7 @@ func WorkspaceFromGitRepo(dir string) (*Workspace, error) { if err != nil { return nil, err } - return &Workspace{ + ws := &Workspace{ Root: repoRoot, File: &Template{}, Repos: []Repo{{ @@ -310,7 +336,13 @@ func WorkspaceFromGitRepo(dir string) (*Workspace, error) { Path: repoRoot, RepoPath: repoRoot, }}, - }, nil + } + // The repo has no clawk.mod at all — the case the host-wide layer exists + // for. It becomes the repo's entire template. + if err := attachGlobal(ws, "", foldGlobalUnderOnlyRepo); err != nil { + return nil, err + } + return ws, nil } // resolveRepoWithProfile resolves a workspace repo entry, optionally @@ -466,7 +498,16 @@ func (w *Workspace) FilterRepos(only []string) (*Workspace, error) { return nil, fmt.Errorf("unknown repo(s): %s (known: %s)", strings.Join(missing, ", "), strings.Join(known, ", ")) } - return &Workspace{Root: w.Root, File: w.File, Repos: kept, Policies: w.Policies}, nil + return &Workspace{ + Root: w.Root, + File: w.File, + Repos: kept, + Policies: w.Policies, + // Carried so the create-time note still names the host-wide file. Its + // directives already live in File, which is shared with the original. + GlobalPath: w.GlobalPath, + GlobalProfileMatched: w.GlobalProfileMatched, + }, nil } // rejectLifecycleAtWorkspace flags the `on ` lists that stay repo-local diff --git a/internal/vsockclient/client.go b/internal/vsockclient/client.go index 1afec7b..2eed7c1 100644 --- a/internal/vsockclient/client.go +++ b/internal/vsockclient/client.go @@ -65,7 +65,7 @@ type Config struct { User string // ClearScreen asks Run to clear the terminal before relaying, so a - // full-screen TUI child (claude, codex, opencode) starts on a clean + // full-screen TUI child (claude, codex, pi, opencode) starts on a clean // canvas instead of overdrawing whatever the CLI printed first (boot // progress, hints): such TUIs position with absolute cursor moves and // don't erase the cells they skip, so stale text shows through. diff --git a/internal/vzdctl/vzdctl.go b/internal/vzdctl/vzdctl.go index 1df7868..a17af93 100644 --- a/internal/vzdctl/vzdctl.go +++ b/internal/vzdctl/vzdctl.go @@ -45,6 +45,12 @@ type Handlers struct { // ErrReverseForwardsUnsupported. ReloadForwards func() error + // ReloadSerials, if non-nil, re-reads the sandbox's serial devices from + // the store and pushes them to the in-guest agent. Nil on backends with + // no vsock listener (firecracker), where the endpoint reports 404 and + // the client maps it to ErrSerialUnsupported. + ReloadSerials func() error + // Gate, if non-nil, powers the interactive allow/deny endpoints // (/v1/events, /v1/decide, /v1/pending). When nil those endpoints // report 404 and the daemon serves only the denial ledger + reload. @@ -149,6 +155,21 @@ func Start(path string, h Handlers) (*Server, error) { } w.WriteHeader(http.StatusNoContent) }) + mux.HandleFunc("POST /v1/reload-serials", func(w http.ResponseWriter, _ *http.Request) { + // Its own endpoint for the same reason reload-forwards is: a daemon + // that predates serial forwarding would answer a shared endpoint + // happily and the CLI would report a live apply that never + // happened. A 404 here is the honest answer. + if h.ReloadSerials == nil { + http.Error(w, "serial forwarding not supported", http.StatusNotFound) + return + } + if err := h.ReloadSerials(); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusNoContent) + }) mux.HandleFunc("GET /v1/events", func(w http.ResponseWriter, r *http.Request) { serveEvents(w, r, h.Gate) }) @@ -410,6 +431,11 @@ var ErrLifecycleUnsupported = errors.New("daemon does not support lifecycle cont // errors.Is. var ErrReverseForwardsUnsupported = errors.New("daemon does not support reverse port forwarding") +// ErrSerialUnsupported reports that the daemon answered but has no +// serial endpoint — either it predates the feature or its backend has no +// host-side vsock listener (firecracker). Callers check with errors.Is. +var ErrSerialUnsupported = errors.New("daemon does not support serial forwarding") + // Lifecycle fetches the VM's live lifecycle snapshot. func (c *Client) Lifecycle(ctx context.Context) (LifecycleState, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://vzd/v1/lifecycle", nil) @@ -528,6 +554,28 @@ func (c *Client) ReloadForwards(ctx context.Context) error { } } +// ReloadSerials asks the daemon to re-read the sandbox's serial devices +// from the store and push them to the in-guest agent. +func (c *Client) ReloadSerials(ctx context.Context) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://vzd/v1/reload-serials", nil) + if err != nil { + return fmt.Errorf("building reload-serials request: %w", err) + } + resp, err := c.do(req) + if err != nil { + return err + } + defer resp.Body.Close() + switch resp.StatusCode { + case http.StatusNoContent: + return nil + case http.StatusNotFound: + return fmt.Errorf("%w: %s", ErrSerialUnsupported, responseError(resp)) + default: + return fmt.Errorf("reload-serials: %s", responseError(resp)) + } +} + // Pending fetches the daemon's outstanding interactive holds. func (c *Client) Pending(ctx context.Context) ([]netfilter.Pending, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://vzd/v1/pending", nil) diff --git a/internal/vzdctl/vzdctl_test.go b/internal/vzdctl/vzdctl_test.go index 853e77c..aedb1f4 100644 --- a/internal/vzdctl/vzdctl_test.go +++ b/internal/vzdctl/vzdctl_test.go @@ -329,3 +329,67 @@ func TestReloadForwardsErrorSurfaces(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "record vanished") } + +func TestReloadSerialsRoundTrip(t *testing.T) { + sock := testSocket(t) + var reloaded int + srv, err := Start(sock, Handlers{ + Denials: func() []netfilter.Denial { return nil }, + Reload: func() error { return nil }, + ReloadSerials: func() error { reloaded++; return nil }, + }) + require.NoError(t, err) + t.Cleanup(func() { srv.Close() }) + + require.NoError(t, NewClient(sock).ReloadSerials(context.Background())) + require.Equal(t, 1, reloaded) +} + +// A daemon with no serial sink (firecracker, or one predating the feature) +// must say so rather than let the CLI report a live apply that never +// happened. +func TestReloadSerialsUnsupported(t *testing.T) { + sock := testSocket(t) + srv, err := Start(sock, Handlers{ + Denials: func() []netfilter.Denial { return nil }, + Reload: func() error { return nil }, + }) + require.NoError(t, err) + t.Cleanup(func() { srv.Close() }) + + err = NewClient(sock).ReloadSerials(context.Background()) + require.ErrorIs(t, err, ErrSerialUnsupported) +} + +// The two reload endpoints must stay independent: a daemon that supports +// reverse forwards but predates serial forwarding has to report exactly +// that, not blanket success. +func TestReloadSerialsIndependentOfReloadForwards(t *testing.T) { + sock := testSocket(t) + srv, err := Start(sock, Handlers{ + Denials: func() []netfilter.Denial { return nil }, + Reload: func() error { return nil }, + ReloadForwards: func() error { return nil }, + }) + require.NoError(t, err) + t.Cleanup(func() { srv.Close() }) + + c := NewClient(sock) + require.NoError(t, c.ReloadForwards(context.Background())) + require.ErrorIs(t, c.ReloadSerials(context.Background()), ErrSerialUnsupported) +} + +func TestReloadSerialsErrorSurfaces(t *testing.T) { + sock := testSocket(t) + srv, err := Start(sock, Handlers{ + Denials: func() []netfilter.Denial { return nil }, + Reload: func() error { return nil }, + ReloadSerials: func() error { return errors.New("record vanished") }, + }) + require.NoError(t, err) + t.Cleanup(func() { srv.Close() }) + + err = NewClient(sock).ReloadSerials(context.Background()) + require.Error(t, err) + require.Contains(t, err.Error(), "record vanished") +} diff --git a/machine/vz/balloon.go b/machine/vz/balloon.go index 3090ca9..022d63f 100644 --- a/machine/vz/balloon.go +++ b/machine/vz/balloon.go @@ -92,8 +92,81 @@ const ( // "roomy" and idle memory can be reclaimed: 2/5 = 40% available. The gap // between low and high is the hysteresis band in which we hold steady. highSlackNum, highSlackDen = 2, 5 + + // swapUsedFloorKiB is how much swap a guest must be holding before we + // stop believing that its free memory means it has memory to spare. + // + // Sandboxes ship with a swap device, and that changes what the two + // signals above mean. Swapping out cold anonymous pages frees them — + // MemAvailable rises — and removes the reclaim stalls PSI measures, so a + // guest that answered a squeeze by paging out reads as roomy on both + // counts. Reclaiming another step there is how a guest ends up living at + // its baseline, swapping, instead of growing to the ceiling it was given. + // + // 64 MiB is above the incidental few megabytes a long-lived guest pages + // out and never touches again, and far below anything that indicates real + // pressure. + // + // Occupancy alone is not enough to act on, though — see swapTrend. + swapUsedFloorKiB = 64 * 1024 + + // swapQuietReportsBeforeReclaim is how many consecutive guest reports may + // show swap at or above the floor WITHOUT growing before the controller + // stops reading it as pressure. + // + // Occupancy latches: a swap slot is released only when its page is faulted + // back in or its owner exits, so a guest that paged out cold anonymous + // memory once reports the same number forever. With GuestSwappiness at 80 + // that is an ordinary thing for a long-lived sandbox to do — one big link + // step early on — and holding on it indefinitely would retire the reclaim + // path for the rest of that sandbox's life on a signal that never decays. + // + // Growth is the part that means "being squeezed right now". 24 reports at + // memPollInterval (5s) is two minutes of quiet: long enough that a guest + // still working through a squeeze keeps the hold, short enough that a spike + // from an hour ago stops speaking for a sandbox that has been idle since. + // + // Guests that are actively thrashing are not this signal's job. Paging in + // and out at the same rate holds occupancy flat, so the trend reads quiet — + // but that case is exactly what elevated PSI describes, and it grows the + // guest to the ceiling two branches earlier in guestDesiredTarget. + swapQuietReportsBeforeReclaim = 24 ) +// swapTrend tracks swap occupancy across guest reports, so the controller can +// tell a guest paging out right now from one that has old cold pages parked in +// swap. One per VM, owned by runBalloonController's goroutine — no lock. +type swapTrend struct { + prevUsedKiB uint64 + quiet int +} + +// observe folds one guest report in and reports whether swap use should still +// count as evidence of memory pressure. +// +// Call it once per REPORT, not once per balloon re-evaluation. Growth is only +// observable between two distinct reports, so re-running it on the same report +// (as the reeval ticker would) counts as quiet and would decay the hold in +// seconds instead of minutes. The controller keeps the verdict between reports, +// exactly as it already keeps the report itself. +func (s *swapTrend) observe(r memReport) bool { + used := r.SwapUsedKiB() + // Below the floor is the same "no signal" as a guest with no swap device, + // or one whose agent is too old to report any — reset, so a later rise + // starts a fresh hold rather than inheriting a stale quiet count. + if used < swapUsedFloorKiB { + s.prevUsedKiB, s.quiet = used, 0 + return false + } + if used > s.prevUsedKiB { + s.quiet = 0 // still paging out + } else { + s.quiet++ + } + s.prevUsedKiB = used + return s.quiet < swapQuietReportsBeforeReclaim +} + // reclaimStepBytes is how much we inflate per reclaim step when the guest is // idle. Gentle by design — a step at a time avoids reclaiming a large chunk // the instant a guest goes briefly quiet, only to deflate it again moments @@ -109,7 +182,11 @@ const reclaimStepBytes = 128 * 1024 * 1024 // A zero report (TotalKiB == 0) means "no fresh guest data" — the caller is // responsible for not constraining the guest in that case; here we simply hold // cur clamped into range. -func guestDesiredTarget(cur, baseline, ceiling uint64, r memReport) uint64 { +// swapPressure is swapTrend.observe's latest verdict for this guest: swap is +// in use AND recently growing. It is a parameter rather than something derived +// from r because the judgement needs history across reports, which this +// deliberately-pure function has none of. +func guestDesiredTarget(cur, baseline, ceiling uint64, r memReport, swapPressure bool) uint64 { if ceiling <= baseline || r.TotalKiB == 0 { return clampRange(cur, baseline, ceiling) } @@ -119,6 +196,14 @@ func guestDesiredTarget(cur, baseline, ceiling uint64, r memReport) uint64 { r.AvailableKiB*uint64(lowSlackDen) < r.TotalKiB*uint64(lowSlackNum) { return ceiling } + // A guest that is paging out is not idle, whatever its slack says: it is + // being squeezed hard enough to evict, and its headroom is the product of + // that, not evidence it was never needed. Hold instead of reclaiming — + // growth stays available through the branch above, which fires as soon as + // it stalls or runs genuinely tight. + if swapPressure { + return clampRange(cur, baseline, ceiling) + } // Reclaim a step toward the baseline when the guest is roomy: high slack // means the guest genuinely isn't using the memory, so handing it back to // the host is exactly right. @@ -138,8 +223,12 @@ func guestDesiredTarget(cur, baseline, ceiling uint64, r memReport) uint64 { // CRITICAL the cap drops to balloonTarget's fractions, reclaiming RAM for the // host even against guest demand (the guest's DEFLATE_ON_OOM is its safety // net). The result is always min(guest-desired, host-allowed). -func mergedBalloonTarget(level pressureLevel, cur, baseline, ceiling uint64, r memReport) uint64 { - desired := guestDesiredTarget(cur, baseline, ceiling, r) +// Note that swapPressure only ever suppresses the guest's own voluntary +// reclaim; it is not a veto over the host. A guest holding swap still gets +// clipped to 3/4 (WARN) or 1/2 (CRITICAL) of its ceiling, because the host +// needing RAM outranks any read of what the guest would prefer. +func mergedBalloonTarget(level pressureLevel, cur, baseline, ceiling uint64, r memReport, swapPressure bool) uint64 { + desired := guestDesiredTarget(cur, baseline, ceiling, r, swapPressure) allowed := balloonTarget(level, ceiling) if desired > allowed { return allowed diff --git a/machine/vz/balloon_test.go b/machine/vz/balloon_test.go index 9906f8d..9974acf 100644 --- a/machine/vz/balloon_test.go +++ b/machine/vz/balloon_test.go @@ -85,7 +85,11 @@ func TestGuestDesiredTarget(t *testing.T) { name string cur uint64 r memReport - want uint64 + // swapPressure is swapTrend.observe's verdict — swap in use and + // recently growing. Its own logic is covered by TestSwapTrend; here it + // is an input, so these cases pin what the target does with it. + swapPressure bool + want uint64 }{ { name: "no report holds current", @@ -129,6 +133,55 @@ func TestGuestDesiredTarget(t *testing.T) { r: memReport{TotalKiB: baseline / 1024, AvailableKiB: baseline / 1024 / 2}, want: baseline, // ceiling == baseline here }, + // The slack a paging guest shows is manufactured: it evicted to produce + // it. Reclaiming on the strength of it is the feedback loop that parks + // a working guest at its baseline, swapping, forever. + { + name: "high slack with swap pressure holds instead of reclaiming", + cur: ceiling, + r: memReport{ + TotalKiB: totalKiB, AvailableKiB: totalKiB / 2, // 50% available > 40% + SwapTotalKiB: 4 << 20, SwapFreeKiB: 4<<20 - swapUsedFloorKiB, + }, + swapPressure: true, + want: ceiling, + }, + // Occupancy without pressure is history, not demand: once the trend has + // gone quiet the headroom is real and the guest gives memory back. This + // is what stops one early spike retiring reclaim for the sandbox's life. + { + name: "swap held but no longer growing reclaims again", + cur: ceiling, + r: memReport{ + TotalKiB: totalKiB, AvailableKiB: totalKiB / 2, + SwapTotalKiB: 4 << 20, SwapFreeKiB: 4<<20 - 2*swapUsedFloorKiB, + }, + swapPressure: false, + want: ceiling - reclaimStepBytes, + }, + // Swap pressure suppresses reclaim, never growth: a guest that is both + // paging and stalling still needs the ceiling. + { + name: "swap pressure does not block growth on demand", + cur: baseline, + r: memReport{ + TotalKiB: totalKiB, AvailableKiB: totalKiB / 2, PSIMemSomeCenti: psiDemandCenti, + SwapTotalKiB: 4 << 20, SwapFreeKiB: 0, + }, + swapPressure: true, + want: ceiling, + }, + // An untouched swap device says nothing at all — a guest that never + // swapped is as reclaimable as one with no swap device. + { + name: "swap present but unused reclaims normally", + cur: ceiling, + r: memReport{ + TotalKiB: totalKiB, AvailableKiB: totalKiB / 2, + SwapTotalKiB: 4 << 20, SwapFreeKiB: 4 << 20, + }, + want: ceiling - reclaimStepBytes, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -136,12 +189,90 @@ func TestGuestDesiredTarget(t *testing.T) { if tt.name == "no burst headroom collapses to baseline" { c = baseline } - got := guestDesiredTarget(tt.cur, baseline, c, tt.r) + got := guestDesiredTarget(tt.cur, baseline, c, tt.r, tt.swapPressure) require.Equal(t, tt.want, got) }) } } +// TestSwapTrend covers the signal that decides swapPressure. The property that +// matters is that it DECAYS: swap occupancy latches (a slot is freed only when +// its page is faulted back in or its owner exits), so reading the raw level as +// pressure pinned a guest that spiked once and then idled, and retired the +// reclaim path for the rest of its life. +func TestSwapTrend(t *testing.T) { + const total = 4 << 20 // 4 GiB of swap, in KiB + + used := func(kib uint64) memReport { + return memReport{TotalKiB: 4 << 20, AvailableKiB: 2 << 20, + SwapTotalKiB: total, SwapFreeKiB: total - kib} + } + + t.Run("below the floor is no signal", func(t *testing.T) { + var s swapTrend + for i := 0; i < 3; i++ { + require.False(t, s.observe(used(swapUsedFloorKiB-1))) + } + }) + + t.Run("no swap device at all is no signal", func(t *testing.T) { + var s swapTrend + require.False(t, s.observe(memReport{TotalKiB: 4 << 20, AvailableKiB: 2 << 20})) + }) + + t.Run("crossing the floor starts a hold", func(t *testing.T) { + var s swapTrend + require.True(t, s.observe(used(swapUsedFloorKiB))) + }) + + t.Run("hold decays once growth stops", func(t *testing.T) { + var s swapTrend + require.True(t, s.observe(used(2*swapUsedFloorKiB)), "first sighting holds") + // Same occupancy, report after report: the guest evicted and moved on. + for i := 1; i < swapQuietReportsBeforeReclaim; i++ { + require.True(t, s.observe(used(2*swapUsedFloorKiB)), + "quiet report %d should still hold", i) + } + require.False(t, s.observe(used(2*swapUsedFloorKiB)), + "past the quiet window the occupancy is history, not pressure") + require.False(t, s.observe(used(2*swapUsedFloorKiB)), "and stays decayed") + }) + + t.Run("renewed growth restarts the hold", func(t *testing.T) { + var s swapTrend + s.observe(used(2 * swapUsedFloorKiB)) + for i := 0; i < swapQuietReportsBeforeReclaim+5; i++ { + s.observe(used(2 * swapUsedFloorKiB)) + } + require.False(t, s.observe(used(2*swapUsedFloorKiB)), "decayed first") + require.True(t, s.observe(used(3*swapUsedFloorKiB)), + "growing again means the guest is being squeezed now") + }) + + t.Run("dropping below the floor resets the quiet count", func(t *testing.T) { + var s swapTrend + s.observe(used(2 * swapUsedFloorKiB)) + for i := 0; i < swapQuietReportsBeforeReclaim+5; i++ { + s.observe(used(2 * swapUsedFloorKiB)) + } + require.False(t, s.observe(used(2*swapUsedFloorKiB)), "decayed") + // Pages came back in and the guest fell under the floor; a later rise + // must not inherit the stale quiet count. + require.False(t, s.observe(used(0))) + require.True(t, s.observe(used(2*swapUsedFloorKiB))) + }) + + t.Run("shrinking swap counts as quiet", func(t *testing.T) { + var s swapTrend + require.True(t, s.observe(used(10*swapUsedFloorKiB))) + // Faulting pages back in is not eviction pressure. + for i := 1; i < swapQuietReportsBeforeReclaim; i++ { + require.True(t, s.observe(used(uint64(10-i/4)*swapUsedFloorKiB))) + } + require.False(t, s.observe(used(5*swapUsedFloorKiB))) + }) +} + func TestGuestDesiredTargetStaysInRange(t *testing.T) { const ( gib = 1024 * 1024 * 1024 @@ -156,7 +287,7 @@ func TestGuestDesiredTargetStaysInRange(t *testing.T) { } for _, cur := range []uint64{baseline, 2 * gib, ceiling} { for _, r := range reports { - got := guestDesiredTarget(cur, baseline, ceiling, r) + got := guestDesiredTarget(cur, baseline, ceiling, r, false) require.GreaterOrEqual(t, got, uint64(baseline)) require.LessOrEqual(t, got, uint64(ceiling)) } @@ -174,18 +305,18 @@ func TestMergedBalloonTargetHostPressureWins(t *testing.T) { // Under normal pressure the guest gets what it asks for. require.Equal(t, uint64(ceiling), - mergedBalloonTarget(pressureNormal, baseline, baseline, ceiling, starved)) + mergedBalloonTarget(pressureNormal, baseline, baseline, ceiling, starved, false)) // Under WARN the host caps growth at balloonTarget(warn, ceiling) even // though the guest is starving. wantWarn := balloonTarget(pressureWarn, ceiling) require.Equal(t, wantWarn, - mergedBalloonTarget(pressureWarn, baseline, baseline, ceiling, starved)) + mergedBalloonTarget(pressureWarn, baseline, baseline, ceiling, starved, false)) require.Less(t, wantWarn, uint64(ceiling)) // Under CRITICAL it caps even lower. wantCrit := balloonTarget(pressureCritical, ceiling) require.Equal(t, wantCrit, - mergedBalloonTarget(pressureCritical, baseline, baseline, ceiling, starved)) + mergedBalloonTarget(pressureCritical, baseline, baseline, ceiling, starved, false)) require.Less(t, wantCrit, wantWarn) } diff --git a/machine/vz/memreport.go b/machine/vz/memreport.go index af24d8a..629bde0 100644 --- a/machine/vz/memreport.go +++ b/machine/vz/memreport.go @@ -21,12 +21,14 @@ const agentMemPort uint32 = 1027 // memReport wire sizes. The report grew from three to five big-endian // uint64s when the guest agent started sampling activity (load, net I/O) -// alongside memory. Decoding accepts the legacy 24-byte prefix alone so a -// new host reads an old guest's report (and vice versa: an old host's -// fixed 24-byte read simply ignores the tail a new guest appends). +// alongside memory, and to seven when sandboxes gained a swap device. +// Decoding accepts any of the three lengths, so a new host reads an old +// guest's report (and vice versa: an old host's fixed read simply ignores +// the tail a new guest appends). const ( - memReportLegacySize = 24 - memReportSize = 40 + memReportLegacySize = 24 + memReportActivitySize = 40 + memReportSize = 56 ) // memReport is a guest snapshot. The balloon controller polls the memory @@ -63,12 +65,33 @@ type memReport struct { // samples: moving counters mean traffic is flowing. NetIOBytes uint64 + // SwapTotalKiB and SwapFreeKiB mirror /proc/meminfo's SwapTotal and + // SwapFree. Both zero on a guest with no swap device — and equally on a + // guest agent too old to report them, which is why the controller only + // ever reads them as "swap is in use", never as "swap is not in use". + // + // They exist because AvailableKiB alone stops describing demand once a + // guest can swap: pushing cold anonymous pages out raises MemAvailable + // and lowers PSI, so a guest under real pressure can look idle. See + // guestDesiredTarget. + SwapTotalKiB uint64 + SwapFreeKiB uint64 + // HasActivity reports whether the guest agent is new enough to include // the activity fields (Load1Centi, NetIOBytes). When false those fields // are zero because they're absent, not because the guest is quiet. HasActivity bool } +// SwapUsedKiB is the swap the guest has actually written. Zero when the +// guest has no swap, reports none, or has swapped nothing. +func (r memReport) SwapUsedKiB() uint64 { + if r.SwapTotalKiB == 0 || r.SwapFreeKiB > r.SwapTotalKiB { + return 0 + } + return r.SwapTotalKiB - r.SwapFreeKiB +} + // encodeMemReport serializes r into exactly memReportSize bytes. func encodeMemReport(r memReport) []byte { b := make([]byte, memReportSize) @@ -77,6 +100,8 @@ func encodeMemReport(r memReport) []byte { binary.BigEndian.PutUint64(b[16:24], r.PSIMemSomeCenti) binary.BigEndian.PutUint64(b[24:32], r.Load1Centi) binary.BigEndian.PutUint64(b[32:40], r.NetIOBytes) + binary.BigEndian.PutUint64(b[40:48], r.SwapTotalKiB) + binary.BigEndian.PutUint64(b[48:56], r.SwapFreeKiB) return b } @@ -94,11 +119,15 @@ func decodeMemReport(b []byte) (memReport, error) { AvailableKiB: binary.BigEndian.Uint64(b[8:16]), PSIMemSomeCenti: binary.BigEndian.Uint64(b[16:24]), } - if len(b) >= memReportSize { + if len(b) >= memReportActivitySize { r.Load1Centi = binary.BigEndian.Uint64(b[24:32]) r.NetIOBytes = binary.BigEndian.Uint64(b[32:40]) r.HasActivity = true } + if len(b) >= memReportSize { + r.SwapTotalKiB = binary.BigEndian.Uint64(b[40:48]) + r.SwapFreeKiB = binary.BigEndian.Uint64(b[48:56]) + } return r, nil } @@ -111,6 +140,8 @@ type GuestStats struct { PSIMemSomeCenti uint64 Load1Centi uint64 NetIOBytes uint64 + SwapTotalKiB uint64 + SwapFreeKiB uint64 // HasActivity is false when the guest runs a legacy agent without the // activity fields; Load1Centi and NetIOBytes are then meaningless. @@ -151,6 +182,8 @@ func FetchGuestStats(ctx context.Context, m machine.Machine) (GuestStats, error) PSIMemSomeCenti: r.PSIMemSomeCenti, Load1Centi: r.Load1Centi, NetIOBytes: r.NetIOBytes, + SwapTotalKiB: r.SwapTotalKiB, + SwapFreeKiB: r.SwapFreeKiB, HasActivity: r.HasActivity, }, nil } diff --git a/machine/vz/memreport_test.go b/machine/vz/memreport_test.go index 36cc308..4324f50 100644 --- a/machine/vz/memreport_test.go +++ b/machine/vz/memreport_test.go @@ -59,6 +59,48 @@ func TestDecodeMemReportLegacy(t *testing.T) { require.False(t, got.HasActivity) } +// TestDecodeMemReportActivityOnly: a 40-byte report from a guest agent +// that predates the swap fields keeps everything up to NetIOBytes and +// reports no swap. Zero swap is indistinguishable from "no swap device", +// which is why the controller only ever reads these fields as evidence +// that swap IS in use. +func TestDecodeMemReportActivityOnly(t *testing.T) { + full := encodeMemReport(memReport{ + TotalKiB: 4 * 1024 * 1024, AvailableKiB: 2_000_000, PSIMemSomeCenti: 17, + Load1Centi: 250, NetIOBytes: 4242, + SwapTotalKiB: 999, SwapFreeKiB: 999, // must NOT survive the shorter decode + }) + got, err := decodeMemReport(full[:memReportActivitySize]) + require.NoError(t, err) + require.Equal(t, memReport{ + TotalKiB: 4 * 1024 * 1024, AvailableKiB: 2_000_000, PSIMemSomeCenti: 17, + Load1Centi: 250, NetIOBytes: 4242, HasActivity: true, + }, got) + require.Zero(t, got.SwapUsedKiB()) +} + +func TestSwapUsedKiB(t *testing.T) { + tests := []struct { + name string + total uint64 + free uint64 + want uint64 + }{ + {name: "no swap device", total: 0, free: 0, want: 0}, + {name: "swap present but untouched", total: 4 << 20, free: 4 << 20, want: 0}, + {name: "half used", total: 4 << 20, free: 2 << 20, want: 2 << 20}, + // A report where free exceeds total is nonsense from a torn or + // legacy read; it must not underflow into a huge "used". + {name: "free above total is not underflow", total: 1024, free: 4096, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := memReport{SwapTotalKiB: tt.total, SwapFreeKiB: tt.free} + require.Equal(t, tt.want, r.SwapUsedKiB()) + }) + } +} + func TestDecodeMemReportShort(t *testing.T) { _, err := decodeMemReport(make([]byte, memReportLegacySize-1)) require.Error(t, err, "short buffer must error, not zero-pad") diff --git a/machine/vz/pressure_darwin.go b/machine/vz/pressure_darwin.go index a4a3a46..e25b3cb 100644 --- a/machine/vz/pressure_darwin.go +++ b/machine/vz/pressure_darwin.go @@ -199,6 +199,12 @@ func (v *vm) runBalloonController(ctx context.Context, dev *codevz.VirtioTraditi level := pressureNormal var rep memReport var lastReport time.Time + // Swap occupancy needs history to mean anything, so the trend is folded + // once per arriving report and its verdict held until the next one — the + // same lifetime as rep itself. Re-observing on a tick would see the same + // numbers, count them as quiet, and decay the hold in seconds. + var swap swapTrend + var swapPressure bool debug.Log("vz", "balloon controller started", "id", v.spec.ID, "baseline_mib", baseline/(1024*1024), "ceiling_mib", ceiling/(1024*1024), @@ -213,7 +219,7 @@ func (v *vm) runBalloonController(ctx context.Context, dev *codevz.VirtioTraditi if fresh { // Guest reported, so its balloon driver is up: manage between // baseline and ceiling on demand. - target = mergedBalloonTarget(level, cur, baseline, ceiling, rep) + target = mergedBalloonTarget(level, cur, baseline, ceiling, rep, swapPressure) } else { // No fresh report — guest still booting, image has no reporter, or // the agent died. Don't try to inflate: a target set before the @@ -235,7 +241,11 @@ func (v *vm) runBalloonController(ctx context.Context, dev *codevz.VirtioTraditi debug.Log("vz", "balloon apply", "id", v.spec.ID, "trigger", trigger, "level", level.String(), "target_mib", target/(1024*1024), "fresh_report", fresh, "changed", changed, - "guest_avail_mib", rep.AvailableKiB/1024, "guest_psi_centi", rep.PSIMemSomeCenti) + "guest_avail_mib", rep.AvailableKiB/1024, "guest_psi_centi", rep.PSIMemSomeCenti, + // Logged because "why is this guest not shrinking?" is otherwise + // unanswerable from the outside: the hold looks identical to the + // hysteresis band. + "guest_swap_used_mib", rep.SwapUsedKiB()/1024, "swap_pressure", swapPressure) } apply("init") @@ -248,6 +258,7 @@ func (v *vm) runBalloonController(ctx context.Context, dev *codevz.VirtioTraditi apply("pressure") case rep = <-reports: lastReport = time.Now() + swapPressure = swap.observe(rep) apply("report") case <-t.C: apply("tick") From e95e149127cf993f080152faecb70e5d94af24a5 Mon Sep 17 00:00:00 2001 From: Celrenheit Date: Wed, 12 Aug 2026 20:47:50 +0000 Subject: [PATCH 3/3] docs: v0.4.0 changelog --- CHANGELOG.md | 218 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 218 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ae0f22..bfe80c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,224 @@ tagged. ## Unreleased +## v0.4.0 + +### Added + +- **Host-wide defaults: `~/.config/clawk/clawk.mod`.** One file outside any + repo whose anonymous `sandbox ( … )` block supplies settings for every + sandbox on the machine — including repos with no `clawk.mod` at all. It is + the lowest layer of the chain (built-in defaults < host-wide < namespace < + repo `clawk.mod` < `clawk.mod.` < flags): lists union with the + host-wide entries first, scalars apply only where nothing narrower declared + one. Read at create time like every other template, so editing it shapes the + next sandbox rather than retro-modifying existing ones. + ([#14](https://github.com/clawkwork/clawk/issues/14)) + + The point is that a kernel path, a token alias, a personal skill mount and a + house rule about commit messages are properties of the host, not of the repo + they happened to be written in — and committing them puts an absolute + `/Users/you/...` path in a shared file. + + Location resolves `$CLAWK_GLOBAL_MOD` (explicit, must exist) → + `$XDG_CONFIG_HOME/clawk/clawk.mod` → `~/.clawk/clawk.mod` (compatibility). + `~/.config` is the documented home because this is the one file in clawk's + footprint you hand-edit and may symlink out of a dotfiles repo, whereas + `~/.clawk` is disposable machine state — VM disks, an image cache, a live + OAuth token — that people exclude from backups and delete to start clean. + Both present is an error, not a silent pick. + + Scope is enforced: the block must be anonymous (a header name labels one + repo's phases, so a name here would silently relabel every repo), `includes` + and the unwired `on down` / `on enter` are rejected, `policy` blocks are + accepted as a personal policy library, and `namespace` blocks are refused — + the file declares defaults for every sandbox, not named resources, and clawk + owns those records itself. Relative paths resolve against the file's own + directory. `--no-global` drops the layer for a reproducible run. + +- **Serial devices.** `clawk serial add /dev/cu.usbmodem1101` puts a + serial port plugged into your Mac inside the sandbox as `/dev/`, so + `arduino-cli`, `esptool`, `avrdude` and a serial monitor can run in there + against real hardware. Declarable in `clawk.mod` as `serial ( … )`, applied + live like reverse forwards, and vz-only for the same reason. + + The USB device is not passed through, because nothing clawk runs on can do + that: Virtualization.framework's USB controller carries virtual + mass-storage devices only, and firecracker has no USB at all. What crosses + is the byte stream and the line settings — which is all any of those tools + ever wanted from a serial port. The guest side is a PTY; the host side is + the real tty, opened only while a process in the sandbox holds the device, + so the board stays available to the Mac the rest of the time. + + Tying the host's open to the guest's open is also what makes auto-reset + work: opening a serial port asserts DTR, which is the pulse a board's reset + circuit is waiting for. A PTY carries the baud rate too, so the 1200-baud + touch that native-USB boards use to enter their bootloader survives the + trip. What a PTY cannot carry is a modem-control line under direct control + — a plain ESP32 DevKit that needs DTR and RTS driven in sequence still + needs its BOOT button. [docs/serial.md](docs/serial.md) has the full table. + + A device may be named by a glob (`'/dev/cu.usbmodem*:ttyACM0'`), resolved + each time the port is opened rather than when it is configured, so a board + that re-enumerates into its bootloader under a neighbouring name stays + reachable. The guest-side name never changes, so `-p /dev/ttyACM0` keeps + working across the whole flash cycle. + +- **Sandboxes now boot with swap.** Every sandbox gets a swap device — 2 GiB + by default, sized with `vm ( swap )` and disabled with + `vm ( swap off )`. + + The reason is the balloon controller, not the guest's appetite. Under host + memory pressure clawk takes RAM back from a guest against its demand (75% + of the ceiling at WARN, 50% at CRITICAL), and a guest with nowhere to put + cold anonymous pages answers that with direct-reclaim stalls and, at the + limit, its OOM killer. A multi-second stall in the agent process is not + merely slow: a process that stops draining its socket lets the connection + go idle, and on a link whose NAT reaps idle mappings in well under a minute + — a phone hotspot, a hotel network — that ends the streaming API response + outright. Swap turns the stall into paging. + + It rides on its own sparse virtio-blk device rather than a swapfile on the + rootfs, because `swapon(2)` rejects a file with holes: a swapfile would + cost its full size in real host bytes at first boot, per sandbox. The + device costs only the pages actually swapped. Note that nothing punches + those holes back — no virtio-blk in the stack advertises discard yet — so + read the usage as a high-water mark until the sandbox is destroyed. + + The balloon controller learned about swap at the same time, and had to: + paging out cold pages raises `MemAvailable` and lowers PSI, so a guest that + answered a squeeze by swapping reads as *roomy* on both of the signals the + controller uses. Left alone it would have reclaimed another step, and + parked working guests at their baseline, swapping, for good. + +- **`mcp ( … )`: MCP servers ready on first boot.** Declare the servers a + project needs and every sandbox created from that `clawk.mod` comes up with + them configured — no per-sandbox setup, no interactive login inside the VM: + + ```text + mcp ( + linear https://mcp.linear.app/mcp header "Authorization: Bearer ${LINEAR_TOKEN}" + github stdio "npx -y @modelcontextprotocol/server-github" env GITHUB_TOKEN + ) + ``` + + Declaring a server also allows its host in the egress policy, in a derived + `mcp` layer that ranks below anything you wrote yourself — so a remote + server doesn't need a matching `network allow`, and doesn't get to + override your own `deny`. Valid in a repo file, a workspace root and a + `namespace` block, merging by name with the narrowest scope winning, so a + namespace can carry the org-wide set. + + Credential values stay out of clawk entirely: `clawk.mod` and the rendered + guest config hold a `${VAR}` reference, and the value travels from your + shell to the runner's process environment at attach time. Pair it with + `env ( TOKEN = ${TOKEN:?message} )` and a missing PAT fails sandbox + creation with your message instead of surfacing as a 401 mid-task. Static + credentials only — that's what can be in place before the VM boots. See + [docs/mcp.md](docs/mcp.md). + +- **The pi coding agent is a built-in runner.** `clawk run pi` attaches + [pi](https://github.com/earendil-works/pi) the same way `clawk run claude` + and `clawk run codex` do, and the `clawk-dev` image ships it. `pi.dev` (its + model catalog and version check) joins the default network allow-list, and + its `~/.pi/` is persisted per sandbox like the other runners'. + + The image also gains `fd-find`. pi's search tools probe PATH for `rg` and + `fd`/`fdfind` and download GitHub release tarballs when they're missing — + the image had ripgrep but not fd, so every fresh sandbox printed "fd not + found. Downloading..." on pi's first run. Debian ships the binary as + `fdfind`, one of the names pi probes for, so the package alone settles it. + +- **opencode is a fully wired runner.** It was in the registry but nothing + else: not installed in `clawk-dev`, and not persisted. Both are fixed, so + `clawk run opencode` works out of the box and its sessions and login + survive `down`/`up` and `destroy` like every other runner's. + + It launches with `--auto` (approve anything not explicitly denied); a deny + rule in your `opencode.jsonc` still wins, and `--safe` drops the flag. + `opencode.ai` joins the default network allow-list, alongside the + `models.dev` entry it already had. + + opencode is the one runner needing two state mounts: it follows the XDG + split rather than keeping a single home, so `~/.local/share/opencode` + (auth.json, mcp-auth.json, opencode.db, repos/) and `~/.config/opencode` + are mounted separately. Its `~/.local/state/opencode` and + `~/.cache/opencode` deliberately stay on the disposable rootfs — the + former holds only lock files, and a lock outliving the VM that took it is + worse than none. + + Note the image grows by roughly 180 MB: `opencode-ai` resolves a prebuilt + platform binary rather than shipping JS. Drop it from the Dockerfile's npm + line for a slimmer variant. + + pi launches with `--approve`. It has no approval prompts to bypass — it + ships no built-in sandbox at all, by design, on the grounds that real + isolation has to come from a VM — but it does gate project-local + `.pi/settings.json` and `.pi/extensions` behind an interactive trust + prompt, and answering that inside a sandbox whose whole premise is that + the project already has the machine is theatre. `--safe` drops the flag + like it does for the other runners. A sandbox declaring `mcp ( … )` gets + no config flag for pi: pi loads MCP through an extension rather than a + config file. + +### Fixed + +- Workspace-position `env ( … )` and `agent ( … )` blocks were silently + dropped: only a repo's own `clawk.mod` fed the sandbox's required-env set and + agent instructions, so a workspace root declaring `env ( GITHUB_TOKEN )` got + nothing. They now apply, ordered scope-outward (workspace, then namespace, + then repo). + +- **An `env ( … )` declaration now overrides clawk's own variables.** The + vsock handshake emitted clawk's vars — `CLAUDE_CODE_OAUTH_TOKEN`, + `COLORTERM`, the terminal-identification set — *after* the sandbox's + declared env, so they won on the last-write-wins fold in the guest agent + and a `clawk.mod` declaration of the same name was silently discarded. + + The two other delivery paths already disagreed with it: `/etc/profile.d` + sources `99-clawk-env.sh` after `98-clawk-claude-oauth.sh`, so login + shells and the `bash -lc` exec fallback have always let the declaration + win. Only the default path — the one `clawk run claude` uses — did not. + + The case that needs it: a sandbox with `ANTHROPIC_BASE_URL` pointed at a + gateway that needs no credential of its own. Claude Code falls back to + `CLAUDE_CODE_OAUTH_TOKEN` when `ANTHROPIC_AUTH_TOKEN` is unset, so clawk's + `sk-ant-oat-…` was sent to that third-party endpoint as an + `Authorization: Bearer` header — a token leak, and one no clawk.mod could + prevent. `env ( CLAUDE_CODE_OAUTH_TOKEN = "" )` now expresses it per + sandbox; previously the only lever was `clawk auth clear`, which disarms + every sandbox on the host. clawk still supplies all of these when the + sandbox says nothing about them. + +- **Codex no longer loses its history on `clawk down && clawk up`.** Only + `~/.claude` was mounted from the host; `~/.codex` lived on the VM rootfs, + which vz re-clones from the image master on *every* boot. So codex's + sessions, `history.jsonl` and login were discarded by a plain stop/start — + not just by `clawk destroy` — even though the docs promised + `state//codex/` was mounted at `~/.codex/`. Each runner's home + directory is now a per-sandbox host mount under + `~/.clawk/namespaces//state//` — `~/.claude`, `~/.codex`, + `~/.pi`, and opencode's two XDG directories — so `--resume` works across + stops and recreates for all of them. + + Existing sandboxes pick this up on their next boot; history from before + the fix was on the disposable disk and is not recoverable. Note that this + costs four more virtio devices on every sandbox against the 32-device + PCIe ceiling: a large multi-repo workspace that booted on v0.3.0 may now + be refused at start, with a message naming the count and what to cut. + +- **`env ( … )` vars now reach the agent process, not just its shells.** The + pty agent spawns the runner directly, so `/etc/profile.d/99-clawk-env.sh` + was never sourced for it: declared variables were visible to a login shell + the agent started, but absent from the agent's own environment. Anything + reading it saw nothing — including `${VAR}` expansion in MCP config, where + a forwarded token would silently become an empty header. The values now + ride the vsock handshake alongside `CLAUDE_CODE_OAUTH_TOKEN`, resolved from + your shell on every dispatch (so rotating a token needs no VM cycle), and + layered so that clawk's own vars fill in only the names your `clawk.mod` + didn't claim (see the entry above). As a side effect the agent's shell tool + sees `GITHUB_TOKEN` and friends without `bash -lc`. + ## v0.3.0 ### Added