diff --git a/.github/workflows/compat.yml b/.github/workflows/compat.yml new file mode 100644 index 0000000..e0bcc1d --- /dev/null +++ b/.github/workflows/compat.yml @@ -0,0 +1,115 @@ +name: Compat + +# Whether a checkpoint the newest release saved resumes on this change's machine, and whether the +# fingerprint still accounts for every ROM the machine maps (boot:compat, boot:roms): after every +# change on main to what a resumed guest is loaded into - the machine's definition, QEMU, the +# kernel, the firmware and their pins - and before every release. +# +# On the KVM runner, because it boots: the newest release is unpacked as the machine before, and +# the machine here is that tree with this change's QEMU, firmware and kernel built over it - the +# base image is the release's, since a checkpoint resumes with the disks it was saved with. The +# builds read and write a BuildKit cache that outlives the run (checkout keeps .cache/), so a +# change that leaves QEMU and the kernel alone rebuilds neither. +# +# The test holds the promise only where it is made: when the two fingerprints differ the contract +# moved on purpose, the test says so and passes, and the release notes say it for the fleet. +# +# Not on a pull request: the KVM runner's job hook refuses that event, since a workflow asking +# for its runners would run a pull request's code as root there. It runs on main after a merge +# that touches the machine, on a dispatch - a branch tried before its pull request merges - and +# as a job release.yml waits for, so no release is published that a checkpoint of the one +# before it does not resume on. + +on: + push: + branches: [main] + paths: + - machine/** + - cmd/spin-machine/** + - qemu/** + - kernel/** + - boot/compat_test.go + - boot/roms_test.go + - versions.yaml + - .github/workflows/compat.yml + workflow_dispatch: + workflow_call: + +permissions: + contents: read + +# Per ref and never cancelled: a run on main is a commit somebody needs the answer for, and a +# release's waits on it. Called from release.yml it is the release's job and takes its group. +concurrency: + group: compat-${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: false + +jobs: + compat: + runs-on: [self-hosted, linux, x64, kvm] + timeout-minutes: 120 + env: + GH_TOKEN: ${{ github.token }} + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # .cache/buildkit is the runner's QEMU and kernel cache; a clean checkout would empty it. + clean: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: false + + - name: Set up Task + uses: go-task/setup-task@a00fbb05ce67b35648be3c78cbc9fd85354c757e # v2.2.0 + + - name: Install cosign + uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + + # A builder that can export its cache to the directory above; the runner keeps it. + - name: A BuildKit that keeps its cache + run: docker buildx inspect compat > /dev/null 2>&1 && docker buildx use compat || docker buildx create --use --name compat --driver docker-container + + - name: Build this change's QEMU and kernel, and the tools + run: task qemu:build kernel:build tools + + # The newest release, checked against its sha256 and its signature, as the machine before; + # this change's machine is the same tree with what was just built in place of its own. + - name: The machine before, and this one + run: | + set -euo pipefail + tag=$(gh release view -R "$GITHUB_REPOSITORY" --json tagName -q .tagName) + name="spin-machine-${tag}-linux-x86_64" + dir=_output/compat/$tag + if [ ! -f "$dir/$name/usr/share/spin-stack/machine.env" ]; then + rm -rf "$dir" && mkdir -p "$dir" + gh release download "$tag" -R "$GITHUB_REPOSITORY" -D "$dir" \ + -p "$name.tar.gz" -p "$name.tar.gz.sha256" -p "$name.tar.gz.sigstore.json" + (cd "$dir" && sha256sum -c "$name.tar.gz.sha256") + cosign verify-blob --bundle "$dir/$name.tar.gz.sigstore.json" \ + --certificate-identity "https://github.com/${GITHUB_REPOSITORY}/.github/workflows/release.yml@refs/heads/main" \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com "$dir/$name.tar.gz" + tar -xzf "$dir/$name.tar.gz" -C "$dir" + fi + previous=$PWD/$dir/$name/usr/share/spin-stack + here=$PWD/_output/compat/here + rm -rf "$here" && cp -as "$previous" "$here" + rm -rf "$here/bin" "$here/qemu" "$here/kernel" + mkdir -p "$here/bin" "$here/kernel" + cp _output/bin/qemu-system-x86_64 _output/bin/qemu-img _output/bin/spin-machine "$here/bin/" + cp -r _output/qemu "$here/qemu" + cp _output/kernel/vmlinux "$here/kernel/vmlinux" + { + echo "SPIN_COMPAT_PREVIOUS=$previous" + echo "SPIN_MACHINE_OUTPUT=$here" + echo "Comparing against $tag" >&2 + } >> "$GITHUB_ENV" + + - name: Every ROM the machine maps is the fingerprint's + run: SPIN_ROM_TEST=1 go test ./boot/ -run '^TestEveryROMIsTheFingerprints$' -count=1 -v -timeout 3m + + - name: A checkpoint of the release before resumes here + run: go test ./boot/ -run '^TestACheckpointOfTheReleaseBeforeResumesHere$' -count=1 -v -timeout 10m diff --git a/.github/workflows/kernel.yml b/.github/workflows/kernel.yml index 79c6e5c..f2de8b8 100644 --- a/.github/workflows/kernel.yml +++ b/.github/workflows/kernel.yml @@ -3,11 +3,10 @@ name: Kernel # The guest kernel, for the same reason QEMU has its own workflow: compiling it is tens of # minutes and the merge gate must not wait for it. # -# It matters more than a build time, though. A changed kernel is a changed machine — the -# fingerprint hashes this binary by content — so every checkpoint taken against the previous -# one stops resuming. The job prints the checksum of what it produced for exactly that -# reason: a config change whose effect nobody looked at is a fleet-wide event nobody -# noticed. +# It matters more than a build time, though: a changed kernel is every guest's next boot. It +# does not strand checkpoints - a resumed guest runs the kernel in its saved memory, and the +# fingerprint is the machine's contract, not this binary - but the job prints the checksum of +# what it produced, so a config change is something somebody looked at. on: # kernel/**, and versions.yaml when it moves the kernel or its toolchain; see the note in @@ -101,28 +100,17 @@ jobs: run: | CACHE_EXPORT=${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} CACHE_BACKEND=gha task kernel:build KERNEL_NPROC=4 - # Compared against the last released machine, which every release publishes as a - # machine.env asset. The kernel's checksum is one of the numbers in it that decide - # whether a checkpoint still resumes (hack/fingerprint-diff names them). - # - # So a pull request that changes the config, or bumps the Debian toolchain the kernel - # is compiled with, says on its own summary that it strands every checkpoint in - # existence. That is not a reason to reject it — it is a release this repository makes - # deliberately, and the step exits 0 either way. It is a reason to know: the change - # that does this need not look like much, and a comment edited in this directory's - # Dockerfile is enough. - # - # It speaks only for the kernel: this workflow builds no QEMU, and the verdict says so - # rather than implying it checked the whole machine. - - name: What this kernel is, and what it breaks - env: - GH_TOKEN: ${{ github.token }} + # What it produced, on the run's summary: a pull request that changes the config, or + # bumps the Debian toolchain the kernel is compiled with, changes every guest's next + # boot, and the change that does it need not look like much. Checkpoints are not + # affected: the kernel is not in the fingerprint (machine.Spec.Fingerprint). + - name: What this kernel is run: | set -euo pipefail - kernel=$(sha256sum _output/kernel/vmlinux | cut -d' ' -f1) - echo "kernel sha256: ${kernel}" - echo "config sha256: $(sha256sum _output/kernel/kernel-config | cut -d' ' -f1)" - hack/fingerprint-diff "kernel_sha256=${kernel}" | tee -a "$GITHUB_STEP_SUMMARY" + { + echo "kernel sha256: $(sha256sum _output/kernel/vmlinux | cut -d' ' -f1)" + echo "config sha256: $(sha256sum _output/kernel/kernel-config | cut -d' ' -f1)" + } | tee -a "$GITHUB_STEP_SUMMARY" - name: Upload the kernel uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/.github/workflows/qemu.yml b/.github/workflows/qemu.yml index 0b22d04..20d51a9 100644 --- a/.github/workflows/qemu.yml +++ b/.github/workflows/qemu.yml @@ -166,24 +166,19 @@ jobs: - name: Check the built QEMU accepts the machine's arguments run: task verify:args - # The other thing a changed QEMU binary is: a different machine. It is hashed by - # content into the fingerprint exactly as the kernel is, so a configure flag added - # here, or a device removed from devices.mak, stops every checkpoint in the fleet from - # resuming. Compared against the last released machine.env and reported; never a - # failure, since changing the machine is a thing that gets done on purpose. - # - # The TCG binary is deliberately not compared. It is not part of a machine's identity - # and must never run a machine a KVM guest will later restore from. - - name: What this QEMU is, and what it breaks - env: - GH_TOKEN: ${{ github.token }} + # What it produced, and whether it still offers the machine type every checkpoint is + # saved on, on the run's summary. A changed QEMU binary is a new emulator of the same + # machine (machine.MachineType stays): it is not in the fingerprint, and whether a + # checkpoint the released QEMU saved resumes on it is boot:compat's to say, on the KVM + # runner (compat.yml). + - name: What this QEMU is run: | set -euo pipefail - qemu=$(sha256sum _output/bin/qemu-system-x86_64 | cut -d' ' -f1) - echo "qemu sha256: ${qemu}" - hack/fingerprint-diff "qemu_sha256=${qemu}" \ - "bios_sha256=$(sha256sum _output/qemu/qboot.bin | cut -d' ' -f1)" \ - "pvh_sha256=$(sha256sum _output/qemu/pvh.bin | cut -d' ' -f1)" | tee -a "$GITHUB_STEP_SUMMARY" + { + echo "qemu sha256: $(sha256sum _output/bin/qemu-system-x86_64 | cut -d' ' -f1)" + echo "qboot.bin: $(stat -c%s _output/qemu/qboot.bin) bytes" + go run ./hack/machinetype check -qemu _output/bin/qemu-system-x86_64 + } | tee -a "$GITHUB_STEP_SUMMARY" - name: Upload the extracted binaries uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b12e087..8e451f4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -40,7 +40,15 @@ concurrency: cancel-in-progress: false jobs: + # A checkpoint of the release before resumes on this one, when the two have one fingerprint, + # and the fingerprint accounts for every ROM the machine maps: on the KVM runner, before + # anything is built here to publish (compat.yml says why). + compat: + if: github.ref == 'refs/heads/main' + uses: ./.github/workflows/compat.yml + release: + needs: compat if: github.ref == 'refs/heads/main' runs-on: blacksmith-8vcpu-ubuntu-2404 # A warm run reads main's BuildKit cache and takes a few minutes (2m13s, 2026-09-30). The @@ -174,6 +182,10 @@ jobs: echo "Nothing to compare against: this is the first published release." > /tmp/verdict.md echo "The first release: nothing before it to say what changed against." > /tmp/changes.md fi + # Whether the QEMU shipped offers a machine type newer than the one every checkpoint is + # saved on: said, never acted on (machine.MachineType). + offer=$(go run ./hack/machinetype check -qemu "_output/release/${name}/usr/share/spin-stack/bin/qemu-system-x86_64") + if [ -n "$offer" ]; then printf '\n%s\n' "$offer" >> /tmp/verdict.md; fi cat /tmp/verdict.md cat /tmp/changes.md diff --git a/AGENTS.md b/AGENTS.md index d551176..8a8e26d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,15 +27,20 @@ These are release invariants, not defaults. Everything below them is a strong en default unless it says otherwise; an experiment may depart from a default when it is isolated and makes its scope and result clear. -1. **A release is one machine.** `machine.Spec.Fingerprint` hashes the QEMU binary, the - kernel, the initrd and the firmware the guest runs (the BIOS, qboot.bin, and pvh.bin) by - content, together with the five arguments that decide the machine's shape. A checkpoint - resumes only onto a machine with the fingerprint of the one it was saved on; across two - that differ a resume is undefined rather than an error. A content change to any of those - files is a fleet-wide event: every checkpoint taken against the previous machine stops - resuming. That is the design, not a bug to work around. A source or recipe change is a possible fleet-wide event; whether - it is one is decided by the resulting artifact hashes and shape, not by the diff's apparent - size. Verify them before promotion. +1. **A checkpoint resumes on any release with its fingerprint, and the fingerprint is the + machine's contract.** `machine.Spec.Fingerprint` hashes the versioned machine type + (`machine.MachineType`), the shape, the chipset's globals and every device as the command + line gives it (`Spec.Contract`), the BIOS's size, and the host CPU under `-cpu host`. Not + the QEMU binary, the kernel, the initrd or pvh.bin: QEMU promises a newer release loads what + an older one of the same versioned type saved, and a resumed guest runs the kernel in its + saved memory. `task boot:compat` holds every change to that promise - a checkpoint the newest + release saved must resume on this tree when the fingerprint is the same - and `task + boot:roms` holds the fingerprint to every ROM a booted machine maps. Across two fingerprints + a resume is undefined rather than an error, so a change to anything the fingerprint hashes - + the machine type, a device or one of its properties, the BIOS's size - is a fleet-wide event: + every checkpoint taken against the previous machine stops resuming. That is the design, made + on purpose and never as a side effect: a QEMU bump keeps the machine type, and moving it is + its own change. 2. **The base image is never written to.** Every VM maps it read-only through a qcow2 backing chain and many share one file. Anything that opens it for writing invalidates diff --git a/README.md b/README.md index 243541e..f065b78 100644 --- a/README.md +++ b/README.md @@ -9,16 +9,17 @@ A virtual machine: QEMU, the guest kernel, the base image, and the definition of machine they make. The four are one thing, and the reason is not tidiness. A VM resumed from a checkpoint -loads device and CPU state into a machine that has to be the same shape as the one the +loads device and CPU state into a machine that has to be the same hardware as the one the checkpoint was saved on, and nothing checks that at run time. So the machine's identity -is computed from the things that decide its shape — `machine.Spec.Fingerprint` hashes the -QEMU binary, the kernel, the initrd and the firmware the guest runs (the BIOS, qboot.bin and pvh.bin) -*by content*, together with the five arguments that decide what a guest sees. Two machines -with the same fingerprint can take each other's checkpoints. Two with different -fingerprints cannot, and a release in which any of those files moved has a different -fingerprint by construction. +is its contract — `machine.Spec.Fingerprint` hashes the versioned machine type +(`machine.MachineType`, `pc-q35-11.1`), the shape, every device as the command line gives it, +and the BIOS's size. Two machines with the same fingerprint can take each other's checkpoints, +and two with different fingerprints cannot. A new QEMU of the same machine type, a new kernel or +new firmware of the same size is the same machine: QEMU promises a newer release loads what an +older one of the same machine type saved, and `task boot:compat` holds every release to it. -One repository, one version, one machine a checkpoint resumes onto. +One repository, one version per release, and checkpoints that resume across releases until the +machine itself changes. **What is deliberately not in a release: software that owns a guest.** This repository builds a machine, and a release is not bootable on its own by design — whoever runs guests @@ -42,7 +43,7 @@ One tarball: | `qemu/patches/`, `qemu/qboot-*.patch`, `qemu/qboot-COPYING` | the changes this tree makes to QEMU and qboot, shipped with the binaries built from them | | `kernel/vmlinux` | plus `kernel-config` | | `image/rootfs.qcow2` | read-only, 0444 | -| `machine.env` | the version, the commit it was built from, and the checksums that decide whether a checkpoint resumes | +| `machine.env` | the version, the commit it was built from, each file's checksum, and `fingerprint_sha256`, the contract that decides whether a checkpoint resumes | | `SOURCES` | every upstream source by version, URL and SHA-256, and the written offer | | `packages.txt` | every package and exact version in the base image | @@ -53,8 +54,8 @@ either. What a release has beyond a build directory is what `hack/release` creat copies when it names one: `machine.env` and `SOURCES` it writes, and the QEMU and qboot patches under `qemu/` it takes from this repository, not from the build. A tree without a manifest is still a release for every purpose except being named — the fingerprint -decides whether checkpoints resume, and it is computed from the files, not from -`machine.env`. `_output/bin/` holds one thing more: `spin-machine`, +decides whether checkpoints resume, and it is computed from the machine's definition and its +BIOS, not from `machine.env`. `_output/bin/` holds one thing more: `spin-machine`, which `task tools` builds for working in this repository and `hack/release` does not ship — a release is the machine, not the tool that boots it. There is one layout: nothing rearranges the files on the way out of a build, into a tarball or into a consumer. Let the three differ diff --git a/Taskfile.yml b/Taskfile.yml index efdd380..431dc0c 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -1,10 +1,11 @@ # spin-machine — a virtual machine: QEMU, the guest kernel, the base image, and the # definition of the machine they make. # -# The four are one thing. `machine.Spec.Fingerprint` hashes by content the binaries and the -# firmware a guest runs, together with the machine's shape, so a release in which any of them -# moved has a different fingerprint by construction. That is why they ship as one tarball with -# one version: shipped separately, they could drift apart and leave the last one to notice. +# The four are one thing. `machine.Spec.Fingerprint` is the machine's contract - its versioned +# machine type, shape, devices and BIOS size - and the binaries, the kernel and the firmware are +# what make that contract run: built together, checked together (boot:compat, boot:roms), and +# shipped as one tarball with one version, because shipped separately they could drift apart +# and leave the last one to notice. # # This file holds the vars every part reads and the targets that cross all of them, in the # order a change travels: build, verify, release, then trying the machine and keeping the @@ -131,8 +132,8 @@ tasks: apk:lock: desc: Pin every package QEMU's and e2fsprogs' builders install (each part's apk.list) at what the pinned Alpine's mirror has today, into its apk.lock. # A fresh install of the list on the pinned Alpine, its database read back: the versions the - # mirror serves now, which is what a bump means. A new lock is a new QEMU or e2fsprogs build, - # and a changed QEMU is a changed machine: hack/fingerprint-diff says so in the release. + # mirror serves now, which is what a bump means. A new lock is a new QEMU or e2fsprogs build: + # the release notes say what moved, and boot:compat that checkpoints still resume on it. cmds: - for: [qemu, e2fsprogs] cmd: | @@ -444,6 +445,10 @@ tasks: # Named VERSION, a bump with no version asked for mkosi v20260927.01-2-g30b717b. cmds: - '{{.VERSIONS}} bump {{.NAME}} {{.TO}}' + # A QEMU bump keeps the machine type (machine.MachineType), and the bump's pull request + # shows what the newer types change against it, from the source just pinned. + - cmd: task qemu:compat + if: '[ "{{.NAME}}" = qemu ]' clean: desc: Remove build output. Leaves the caches; use clean:cache for those. diff --git a/boot/Taskfile.yml b/boot/Taskfile.yml index 6296167..2b629aa 100644 --- a/boot/Taskfile.yml +++ b/boot/Taskfile.yml @@ -15,6 +15,44 @@ env: SPIN_MACHINE_OUTPUT: '{{.OUTPUT_ABS}}' tasks: + roms: + desc: >- + Hold the fingerprint to every read-only block a started machine maps: the BIOS by its + file's size, the rest the machine type's own (machine.BIOSBlock, machine.MachineBlocks). + Needs a built release and /dev/kvm. + deps: [':tools'] + cmds: + - SPIN_ROM_TEST=1 go test ./boot/ -run '^TestEveryROMIsTheFingerprints$' -count=1 -v -timeout 3m + + compat: + desc: >- + A checkpoint the newest published release saves resumes on this tree's build, when the two + have one fingerprint (machine.Spec.Fingerprint). Fetches that release, checked against its + sha256 and its signature, into _output/compat//. PREVIOUS= names another. + Needs a built release and /dev/kvm. + deps: [':tools'] + vars: + PREVIOUS: '{{.PREVIOUS | default ""}}' + cmds: + - | + set -euo pipefail + tag='{{.PREVIOUS}}' + if [ -z "$tag" ]; then tag=$(gh release view -R spin-stack/spin-machine --json tagName -q .tagName); fi + dir={{.OUTPUT_ABS}}/compat/$tag + name="spin-machine-${tag}-linux-x86_64" + if [ ! -f "$dir/$name/usr/share/spin-stack/machine.env" ]; then + rm -rf "$dir" && mkdir -p "$dir" + gh release download "$tag" -R spin-stack/spin-machine -D "$dir" \ + -p "$name.tar.gz" -p "$name.tar.gz.sha256" -p "$name.tar.gz.sigstore.json" + (cd "$dir" && sha256sum -c "$name.tar.gz.sha256") + cosign verify-blob --bundle "$dir/$name.tar.gz.sigstore.json" \ + --certificate-identity https://github.com/spin-stack/spin-machine/.github/workflows/release.yml@refs/heads/main \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com "$dir/$name.tar.gz" + tar -xzf "$dir/$name.tar.gz" -C "$dir" + fi + SPIN_COMPAT_PREVIOUS="$dir/$name/usr/share/spin-stack" \ + go test ./boot/ -run '^TestACheckpointOfTheReleaseBeforeResumesHere$' -count=1 -v -timeout 10m + bench: desc: >- What a boot costs, measured from the host and repeated enough to believe. Needs diff --git a/boot/compat_test.go b/boot/compat_test.go new file mode 100644 index 0000000..65cd1b7 --- /dev/null +++ b/boot/compat_test.go @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: Apache-2.0 + +package boot_test + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "sync" + "testing" + "time" + + "github.com/spin-stack/spin-machine/machine" +) + +// A checkpoint the release before saved resumes on this tree's machine, when the two have one +// fingerprint: machine.Spec.Fingerprint promises it, by leaving out the QEMU binary, the kernel +// and the firmware a boot reads, and this is what holds the promise to QEMU's. The guest is +// booted on the previous release's QEMU, kernel and firmware, over an overlay of its base image, +// with a counter printing on its console; it is saved, and restored with this tree's QEMU over +// the same overlay, and the counter has to go on from where it was and the disk has to answer. +// +// When the two fingerprints differ, the contract moved and no checkpoint of the previous release +// is meant to resume: the test says so and holds nothing, and the release notes say it for the +// fleet (hack/fingerprint-diff). +// +// SPIN_COMPAT_PREVIOUS= the previous release's usr/share/spin-stack (task boot:compat) +func TestACheckpointOfTheReleaseBeforeResumesHere(t *testing.T) { + previous := os.Getenv("SPIN_COMPAT_PREVIOUS") + if previous == "" { + t.Skip("set SPIN_COMPAT_PREVIOUS to a release's usr/share/spin-stack: task boot:compat fetches the newest") + } + out := releaseDir(t) + cli := filepath.Join(out, "bin", "spin-machine") + before, here := fingerprintOf(t, cli, previous), fingerprintOf(t, cli, out) + if before != here { + t.Skipf("the machine's contract moved (%s, then %s): no checkpoint of %s is meant to resume here", before, here, previous) + } + + dir := t.TempDir() + overlay := filepath.Join(dir, "root.qcow2") + if b, err := exec.Command(filepath.Join(out, "bin", "qemu-img"), "create", "-q", "-f", "qcow2", //nolint:gosec // this tree's qemu-img + "-b", filepath.Join(previous, "image", "rootfs.qcow2"), "-F", "qcow2", overlay).CombinedOutput(); err != nil { + t.Fatalf("an overlay of the previous base image: %v: %s", err, b) + } + disk := []string{"--disk", overlay, "--disk-format", "qcow2", "--init", "/bin/sh"} + state := filepath.Join(dir, "state") + + // The previous release's machine, a counter on its console, saved. + a := startGuest(t, machineArgs(t, cli, previous, append([]string{"--qmp", filepath.Join(dir, "a.sock")}, disk...))) + a.type_(t, "mount -t proc proc /proc; ( i=0; while :; do i=$((i+1)); echo TICK-$i; sleep 0.2; done ) &") + a.await(t, regexp.MustCompile(`TICK-5\b`), 60*time.Second) + qa := dial(t, filepath.Join(dir, "a.sock"), &vm{}) + qa.do(t, "stop", nil) + last := a.lastTick() + qa.do(t, "migrate", map[string]any{"uri": "file:" + state}) + qa.until(t, "query-migrate", "completed", 60*time.Second) + qa.close() + a.kill() + + // This tree's machine, loading it. + b := startGuest(t, machineArgs(t, cli, out, append([]string{"--qmp", filepath.Join(dir, "b.sock"), "--incoming", "defer"}, disk...))) + qb := dial(t, filepath.Join(dir, "b.sock"), &vm{}) + defer qb.close() + qb.do(t, "migrate-incoming", map[string]any{"uri": "file:" + state}) + qb.until(t, "query-migrate", "completed", 60*time.Second) + qb.do(t, "cont", nil) + b.await(t, regexp.MustCompile(fmt.Sprintf(`TICK-%d\b`, last+3)), 30*time.Second) + // Read through the disk the guest had before it was saved: a file it never opened, so + // the read reaches the restored virtio-blk and the overlay behind it. + b.type_(t, "echo READ-$(wc -c < /etc/os-release)") + b.await(t, regexp.MustCompile(`READ-[1-9][0-9]*`), 30*time.Second) + rel, err := machine.OpenRelease(previous) + if err != nil { + t.Fatal(err) + } + t.Logf("a checkpoint of %s, saved at TICK-%d, went on counting on this tree's machine", rel.Version(), last) +} + +// fingerprintOf is `spin-machine fingerprint` of the release at dir, as this tree computes it. +func fingerprintOf(t *testing.T, cli, dir string) string { + t.Helper() + b, err := exec.Command(cli, "fingerprint", "--release", dir).Output() //nolint:gosec // this tree's spin-machine + if err != nil { + t.Fatalf("spin-machine fingerprint --release %s: %v", dir, err) + } + var f struct{ Fingerprint string } + if err := json.Unmarshal(b, &f); err != nil || f.Fingerprint == "" { + t.Fatalf("spin-machine fingerprint: %q: %v", b, err) + } + return f.Fingerprint +} + +// guest is one QEMU whose console is its stdin and stdout. +type guest struct { + cmd *exec.Cmd + stdin io.WriteCloser + mu sync.Mutex + out bytes.Buffer +} + +func startGuest(t *testing.T, args []string) *guest { + t.Helper() + g := &guest{cmd: exec.Command(args[0], args[1:]...)} //nolint:gosec // the machine's own command line + stdin, err := g.cmd.StdinPipe() + if err != nil { + t.Fatal(err) + } + g.stdin = stdin + r, w := io.Pipe() + g.cmd.Stdout, g.cmd.Stderr = w, w + if err := g.cmd.Start(); err != nil { + t.Fatalf("starting QEMU: %v", err) + } + go func() { + buf := make([]byte, 4096) + for { + n, err := r.Read(buf) + g.mu.Lock() + g.out.Write(buf[:n]) + g.mu.Unlock() + if err != nil { + return + } + } + }() + t.Cleanup(g.kill) + return g +} + +// type_ is a line on the console. The guest's shell is not reading it when QEMU starts, so the +// first bytes are lost: blank lines go first, each costing only a prompt. +func (g *guest) type_(t *testing.T, line string) { + t.Helper() + if _, err := io.WriteString(g.stdin, "\n\n\n\n\n\n\n\n\n\n"+line+"\n"); err != nil { + t.Fatalf("typing on the console: %v", err) + } +} + +func (g *guest) said() string { + g.mu.Lock() + defer g.mu.Unlock() + return g.out.String() +} + +// await is the console printing what matches re, or the test failing with what it did print. +func (g *guest) await(t *testing.T, re *regexp.Regexp, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for !re.MatchString(g.said()) { + if time.Now().After(deadline) { + t.Fatalf("the console never said %s in %s:\n%s", re, timeout, tail([]byte(g.said()), 2000)) + } + time.Sleep(50 * time.Millisecond) + } +} + +// lastTick is the highest TICK the console has printed. +func (g *guest) lastTick() int { + last := 0 + for _, m := range regexp.MustCompile(`TICK-(\d+)`).FindAllStringSubmatch(g.said(), -1) { + if n, _ := strconv.Atoi(m[1]); n > last { + last = n + } + } + return last +} + +func (g *guest) kill() { + if g.cmd.Process != nil { + _ = g.cmd.Process.Kill() + _ = g.cmd.Wait() + } +} diff --git a/boot/roms_test.go b/boot/roms_test.go new file mode 100644 index 0000000..a9693f7 --- /dev/null +++ b/boot/roms_test.go @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: Apache-2.0 + +package boot_test + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "slices" + "strconv" + "strings" + "testing" + + "github.com/spin-stack/spin-machine/machine" +) + +// The fingerprint sizes one ROM, the BIOS, and names the other read-only blocks the machine's +// own (machine.BIOSBlock, machine.MachineBlocks): a resume refuses a ROM block of another +// length, so a ROM a file sizes and the fingerprint does not is a checkpoint that matches a +// machine it cannot be loaded into. This starts the machine `spin-machine args` describes, at +// its smallest and at a workspace's whole shape, paused before the guest runs, and holds every +// read-only block QEMU mapped to those two lists, and the BIOS block to its file's size. A QEMU +// bump or a device that maps another ROM fails here, and is answered in machine.go. +func TestEveryROMIsTheFingerprints(t *testing.T) { + if os.Getenv("SPIN_ROM_TEST") == "" { + t.Skip("set SPIN_ROM_TEST=1: this starts the machine") + } + out := releaseDir(t) + bios, err := os.Stat(filepath.Join(out, "qemu", machine.DefaultBIOS)) + if err != nil { + t.Fatal(err) + } + for name, flags := range map[string][]string{ + "smallest": nil, + "workspace-sized": {"--max-memory", "8192", "--max-cpus", "8", "--hotplug-disks", "2", "--vsock-cid", "5"}, + } { + t.Run(name, func(t *testing.T) { + ro := readOnlyBlocks(t, ramBlocks(t, out, flags...)) + if got := ro[machine.BIOSBlock]; got != bios.Size() { + t.Errorf("%s is %d bytes and %s is %d: the fingerprint sizes the BIOS by its file", machine.BIOSBlock, got, machine.DefaultBIOS, bios.Size()) + } + for block := range ro { + if block != machine.BIOSBlock && !slices.Contains(machine.MachineBlocks, block) { + t.Errorf("QEMU maps a read-only block %s the fingerprint does not account for: size it there, or name it a machine block with why", block) + } + } + for _, block := range machine.MachineBlocks { + if _, ok := ro[block]; !ok { + t.Errorf("machine.MachineBlocks names %s, which this machine no longer maps", block) + } + } + }) + } +} + +// readOnlyBlocks is `info ramblock`'s read-only blocks by name, each at its used length. +func readOnlyBlocks(t *testing.T, table string) map[string]int64 { + t.Helper() + ro := map[string]int64{} + for _, line := range strings.Split(table, "\n") { + f := strings.Fields(line) + // Block Name, PSize as a number and a unit, Offset, Used, Total, HVA, RO. + if len(f) != 8 || f[7] != "ro" { + continue + } + used, err := strconv.ParseInt(strings.TrimPrefix(f[4], "0x"), 16, 64) + if err != nil { + t.Fatalf("info ramblock's used length %q: %v", f[4], err) + } + ro[f[0]] = used + } + if len(ro) == 0 { + t.Fatalf("no read-only block in info ramblock:\n%s", table) + } + return ro +} + +// ramBlocks is QEMU's `info ramblock` of the machine `spin-machine args` describes, started paused: +// every block of memory the machine maps, by name, before the guest runs a byte. +func ramBlocks(t *testing.T, out string, flags ...string) string { + t.Helper() + socket := filepath.Join(t.TempDir(), "qmp.sock") + args := machineArgs(t, filepath.Join(out, "bin", "spin-machine"), out, + append([]string{"--qmp", socket, "--init", "/bin/sh", "--console", ""}, flags...)) + cmd := exec.Command(args[0], append(args[1:], "-S")...) //nolint:gosec // the machine's own command line + log, err := os.CreateTemp(t.TempDir(), "qemu-*.log") + if err != nil { + t.Fatal(err) + } + cmd.Stdout, cmd.Stderr = log, log + if err := cmd.Start(); err != nil { + t.Fatalf("starting QEMU: %v", err) + } + t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait() }) + said := func() []byte { b, _ := os.ReadFile(log.Name()); return b } + q := dial(t, socket, &vm{out: said()}) + defer q.close() + var text string + if err := json.Unmarshal(q.do(t, "human-monitor-command", map[string]any{"command-line": "info ramblock"}), &text); err != nil { + t.Fatalf("reading info ramblock: %v", err) + } + return text +} diff --git a/cmd/spin-machine/main.go b/cmd/spin-machine/main.go index 7ccd1b5..4efbea4 100644 --- a/cmd/spin-machine/main.go +++ b/cmd/spin-machine/main.go @@ -20,6 +20,7 @@ package main import ( + "cmp" "encoding/json" "errors" "flag" @@ -379,11 +380,10 @@ func fingerprint(s machine.Spec) error { } out := struct { Fingerprint string `json:"fingerprint"` - QEMU string `json:"qemu"` - Kernel string `json:"kernel"` - Initrd string `json:"initrd,omitempty"` Shape machine.Shape `json:"shape"` - }{fp, s.QEMU, s.Kernel, s.Initrd, s.Shape()} + Contract []string `json:"contract"` + BIOS string `json:"bios"` + }{fp, s.Shape(), s.Contract(), filepath.Join(s.Firmware, cmp.Or(s.BIOS, machine.DefaultBIOS))} enc := json.NewEncoder(os.Stdout) enc.SetIndent("", " ") diff --git a/docs/machine.md b/docs/machine.md index d8c3f48..1ab2a34 100644 --- a/docs/machine.md +++ b/docs/machine.md @@ -39,24 +39,39 @@ something that otherwise boots and is subtly wrong. ## The fingerprint -`Spec.Fingerprint` is the machine's identity. A checkpoint resumes only onto a machine with +`Spec.Fingerprint` is the machine's contract. A checkpoint resumes only onto a machine with the fingerprint of the one it was saved on; across two that differ a resume is undefined rather than an error. It hashes: -- the QEMU binary, the kernel, the initrd and the firmware the guest runs (the BIOS, - `qboot.bin` by default, and `pvh.bin`), **by content** — not by path; -- the shape: the `-machine`, `-accel`, `-cpu`, `-smp` and `-m` arguments; -- the devices present when state is loaded: vmgenid, the RNG and the balloon, and, when the - spec has them, the vsock, the virtio-mem region, the serial port, the hotplug controller, - and the NICs by count and MTU; +- the shape: the `-machine` argument with its **versioned** machine type + (`machine.MachineType`, `pc-q35-11.1`), `-accel`, `-cpu`, `-smp` and `-m`; +- the chipset's globals and every device present when state is loaded, each as the command line + gives it, properties and slot included (`Spec.Contract`): vmgenid, the RNG and the balloon, + and, when the spec has them, the vsock, the virtio-mem region, the serial port, the hotplug + controller and the NICs with their MTU; +- the BIOS's size: it is the `pc.bios` RAM block, which a checkpoint carries whole and a resume + refuses at another length. Every other read-only block is the machine type's or the chipset's + (`machine.MachineBlocks`), and `task boot:roms` fails when a booted machine maps one that is + neither; - the host's CPU model, but only when the guest is shown the host's CPU (`host` under KVM, `max` under TCG). See [migration.md](migration.md). -It leaves out what a resume is allowed to differ in — paths, descriptors, the vsock context -id, MAC addresses — and the disks, which are the VM's and not the machine's: a host computes -its fingerprint before any VM exists, and whoever resumes a checkpoint gives it the disks it -was saved with. +It leaves out what a resume is allowed to differ in — paths, descriptors, ids, the backends +devices are bound to, the vsock context id, MAC addresses — and the disks, which are the VM's +and not the machine's: a host computes its fingerprint before any VM exists, and whoever resumes +a checkpoint gives it the disks it was saved with. -`spin-machine fingerprint` prints it with the shape that went into it, because the number -alone answers "did it change?" and not "what changed?". +And it leaves out the files a guest does not reload on a resume: the QEMU binary, the kernel, +the initrd and `pvh.bin`. QEMU's migration promise is per versioned machine type — a newer QEMU +loads what an older one of the same `pc-q35-X.Y` saved, the newer type's changes to defaults +switched back off for the older (`hw_compat_X_Y`, `pc_compat_X_Y`) — and a resumed guest runs the +kernel in its saved memory until it reboots. `task boot:compat` holds this tree to that promise: +a checkpoint the newest release saved resumes on this tree's build. + +Moving `machine.MachineType` is moving every guest to new hardware, decided, not a side effect of +a QEMU bump: `qemu:verify` says when a QEMU offers a newer type and fails once the pinned one is +deprecated, and `task qemu:compat` prints what the newer types change against it. + +`spin-machine fingerprint` prints it with the shape and the contract that went into it, because +the number alone answers "did it change?" and not "what changed?". diff --git a/docs/migration.md b/docs/migration.md index 0d7a02a..b6eb279 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -42,7 +42,7 @@ features` and exit 1, before the VM exists. ## The devices have to match too Down to whether there is a serial port — restoring without one says `Unknown section or -instance 'serial'`. That is why the device set is in the fingerprint -([machine.md](machine.md#the-fingerprint)). NICs are in it, by count and MTU: a NIC is on +instance 'serial'`. That is why every device, with its properties, is in the fingerprint +([machine.md](machine.md#the-fingerprint)). NICs are in it, by slot and MTU: a NIC is on the command line, so it is there when the state is loaded. Disks are not: they are the VM's and not the machine's, and whoever resumes a checkpoint gives it the disks it was saved with. diff --git a/docs/releasing.md b/docs/releasing.md index 76eea6d..a8e3829 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -28,8 +28,9 @@ combination of the machine's features, booted from the published tarball to a lo `release.yml` builds all three and packs one tarball. Versions are **CalVer**, `v20260909.01`: a release of this repository is the machine as it stood on a date. There is no API here to promise compatibility about, and the one thing a version could promise — -that checkpoints still resume — is decided by the fingerprint of the artefacts, not by a -number anybody chose. A release is the workflow dispatched on main, and nothing else - a pushed +that checkpoints still resume — is decided by the fingerprint, the machine's contract, not by a +number anybody chose: a release that keeps it resumes every checkpoint the one before it saved +(`task boot:compat`), whatever QEMU, kernel or firmware it ships. A release is the workflow dispatched on main, and nothing else - a pushed tag starts nothing: given a version it makes that one, with no input it generates the next sequence for today; either way it tags the commit once its files are signed, and writes the notes: what changed since the release before (`hack/releasenotes`: the commits that reached diff --git a/hack/fingerprint-diff b/hack/fingerprint-diff index 47345da..f4ea668 100755 --- a/hack/fingerprint-diff +++ b/hack/fingerprint-diff @@ -2,27 +2,26 @@ # # Whether this build is still the machine the fleet's checkpoints were taken against. # -# `machine.Spec.Fingerprint` hashes by content the files a guest runs - QEMU, the kernel, the -# initrd and the firmware - together with the machine's shape and devices, as machine/ defines -# them. The initrd is brought by whoever runs guests, so between two releases the question -# collapses to the checksums FP_KEYS names below, fingerprint_sha256 among them - Fingerprint -# itself, of one fixed machine, which also moves when machine/ changes what it hashes or how: -# **checkpoints carry over if and only if none of them changed.** +# `machine.Spec.Fingerprint` is the machine's contract: its versioned machine type +# (machine.MachineType), its shape, its devices as the command line gives them, and the BIOS's +# size. fingerprint_sha256 in machine.env is Fingerprint itself, of one fixed machine, so it +# moves when any of those does: **checkpoints carry over if and only if it did not change.** # -# image_sha256 is deliberately not one of them. The base image is mapped read-only through -# a backing chain and is not hashed into the fingerprint - a checkpoint resumes with its own -# disks, chain and all - so a release that moves only the image still resumes every -# checkpoint taken against its predecessor. Saying that plainly is -# half the value of this script: the other half is that the numbers which *do* decide it stop -# being something a human diffs by eye across two release pages. +# The files' own checksums - qemu_sha256, kernel_sha256, bios_sha256, pvh_sha256 and +# image_sha256 - are deliberately not what decides it. A newer QEMU of the same machine type is +# another emulator of one machine (QEMU's migration promise, which boot:compat holds a release +# to); a resumed guest runs the kernel in its saved memory; the BIOS travels in a checkpoint and +# only its size is the machine's; pvh.bin and the base image are read by a boot. So a release +# that moves only those still resumes every checkpoint taken against its predecessor, and this +# says so beside what moved. Saying that plainly is half the value of this script: the other half +# is that the number which *does* decide it stops being something a human diffs by eye across +# two release pages. # # It exits 0 whatever it finds. This is not a gate — changing the machine is a thing this # repository does on purpose, and a check that failed on it would be one people learn to -# force through. What changing the machine is not allowed to be is silent, and it has been -# exactly once here already: a comment edited in kernel/Dockerfile, a rebuilt vmlinux, and a -# fleet-wide invalidation that nothing announced. +# force through. What changing the machine is not allowed to be is silent. # -# hack/fingerprint-diff qemu_sha256= [kernel_sha256=] [bios_sha256=] [pvh_sha256=] [fingerprint_sha256=] [image_sha256=] +# hack/fingerprint-diff fingerprint_sha256= [qemu_sha256=] [kernel_sha256=] [bios_sha256=] [pvh_sha256=] [image_sha256=] # # Writes a Markdown verdict to stdout — meant to be pasted into release notes or a step # summary — and its workings to stderr. @@ -47,10 +46,9 @@ if [ "${#KEYS[@]}" -eq 0 ]; then exit 2 fi -# The classification this script exists to hold in one place. Adding a file to the -# fingerprint means adding it here; anything else is a number that is reported and does not -# bear on whether a checkpoint resumes. -FP_KEYS=(qemu_sha256 kernel_sha256 bios_sha256 pvh_sha256 fingerprint_sha256) +# The classification this script exists to hold in one place: the fingerprint is the contract, +# and every other number is reported and does not bear on whether a checkpoint resumes. +FP_KEYS=(fingerprint_sha256) bears_on_fingerprint() { local k for k in "${FP_KEYS[@]}"; do [ "$1" = "${k}" ] && return 0; done @@ -148,7 +146,8 @@ detail() { if [ "${#moved_fp[@]}" -gt 0 ]; then say "### ⚠️ Every checkpoint taken against \`${reference}\` stops resuming on this machine" say - say "The fingerprint hashes these by content, so a host resuming a checkpoint saved" + say "The machine's contract moved - its machine type, its shape, its devices or its BIOS's" + say "size (machine.Spec.Fingerprint) - so a host resuming a checkpoint saved" say "against \`${reference}\` on this machine is doing something undefined rather than" say "something that fails." say @@ -156,9 +155,10 @@ if [ "${#moved_fp[@]}" -gt 0 ]; then elif [ "${#unchecked_fp[@]}" -eq 0 ]; then say "### Checkpoints taken against \`${reference}\` still resume on this machine" say - say "$(printf '`%s` ' "${FP_KEYS[@]}")are unchanged, and they are the whole of what" - say "this repository contributes to the fingerprint — the initrd and each VM's own sizes" - say "come from whoever runs the guests and are untouched by a release." + say "\`fingerprint_sha256\`, the machine's contract, is unchanged: its machine type, shape," + say "devices and BIOS size are the same, whatever else below moved - a new QEMU of the same" + say "machine type, a new kernel or firmware a boot reads. Each VM's own sizes come from" + say "whoever runs the guests and are untouched by a release." else say "### Nothing compared here moves the fingerprint" say diff --git a/hack/machinetype/main.go b/hack/machinetype/main.go new file mode 100644 index 0000000..ffc95f4 --- /dev/null +++ b/hack/machinetype/main.go @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Command machinetype watches the versioned q35 the machine is pinned to (machine.MachineType) +// against the QEMU it runs on. +// +// check asks a built QEMU for its machine types: it fails when the pinned one is missing or +// deprecated - QEMU deprecates a versioned type after three years and removes it after six, so +// this fires years before a QEMU stops loading the checkpoints it saved - and says, without +// failing, when QEMU offers a newer one, for the build log and the release's notes. +// +// compat prints what the newer types change, from QEMU's own source: the properties +// hw_compat_X_Y (hw/core/machine.c) and pc_compat_X_Y (hw/i386/pc.c) switch back off for the +// pinned X.Y. That list is what moving the pin would change under every guest, and task bump +// NAME=qemu prints it from the tarball the pin names. +// +// go run ./hack/machinetype check -qemu _output/bin/qemu-system-x86_64 +// go run ./hack/machinetype compat /hw/core/machine.c /hw/i386/pc.c +package main + +import ( + "errors" + "fmt" + "io" + "os" + "os/exec" + "regexp" + "strconv" + "strings" + + "github.com/spin-stack/spin-machine/machine" +) + +func main() { + os.Exit(run(os.Args[1:], os.Stdout, os.Stderr, help)) // mutate-exempt: os.Args[0] is the program; run is what the tests drive +} + +// help is a QEMU binary's `-machine help`. +func help(qemu string) (string, error) { + out, err := exec.Command(qemu, "-machine", "help").CombinedOutput() //nolint:gosec // the QEMU this build made + if err != nil { + return "", fmt.Errorf("%s -machine help: %w: %s", qemu, err, out) + } + return string(out), nil +} + +// run answers the exit status: 0 the pin stands, 1 it does not, 2 the arguments are not a check. +func run(args []string, stdout, stderr io.Writer, help func(string) (string, error)) int { + if len(args) == 0 { + say(stderr, "machinetype: check -qemu | compat ") + return 2 + } + switch args[0] { + case "check": + if len(args) != 3 || args[1] != "-qemu" { + say(stderr, "machinetype: check -qemu ") + return 2 + } + text, err := help(args[2]) + if err != nil { + say(stderr, "machinetype:", err) + return 1 + } + notice, err := Check(text, machine.MachineType) + if err != nil { + say(stderr, "machinetype:", err) + return 1 + } + if notice != "" { + say(stdout, notice) + } + return 0 + case "compat": + if len(args) != 3 { + say(stderr, "machinetype: compat ") + return 2 + } + var src []string + for _, f := range args[1:] { + b, err := os.ReadFile(f) //nolint:gosec // a file of the QEMU source the bump unpacked + if err != nil { + say(stderr, "machinetype:", err) + return 1 + } + src = append(src, string(b)) + } + out, err := Compat(strings.Join(src, "\n"), machine.MachineType) + if errors.Is(err, errNoArray) { + say(stdout, "machinetype:", machine.MachineType, "is the newest type in this QEMU: nothing newer to differ from it") + return 0 + } + if err != nil { + say(stderr, "machinetype:", err) + return 1 + } + say(stdout, out) + return 0 + } + say(stderr, "machinetype: no command", args[0]) + return 2 +} + +// q35 is one line of `-machine help` that names a versioned q35: its X.Y and whether it is +// deprecated. +var q35 = regexp.MustCompile(`^pc-q35-(\d+)\.(\d+)\s.*?(\(deprecated\))?\s*$`) + +// version is a pc-q35-X.Y's X and Y. +type version struct{ major, minor int } + +func parse(name string) (version, bool) { + m := regexp.MustCompile(`^pc-q35-(\d+)\.(\d+)$`).FindStringSubmatch(name) + if m == nil { + return version{}, false + } + x, _ := strconv.Atoi(m[1]) + y, _ := strconv.Atoi(m[2]) + return version{x, y}, true +} + +func (v version) newer(than version) bool { + return v.major > than.major || v.major == than.major && v.minor > than.minor +} + +// Check is what `-machine help` says of pinned: an error when it is not offered or is +// deprecated, and a notice when a newer pc-q35 is. +func Check(help, pinned string) (notice string, err error) { + want, ok := parse(pinned) + if !ok { + return "", fmt.Errorf("%q is not a pc-q35-X.Y", pinned) + } + var newest version + found := false + for _, line := range strings.Split(help, "\n") { + m := q35.FindStringSubmatch(strings.TrimSpace(line)) + if m == nil { + continue + } + x, _ := strconv.Atoi(m[1]) + y, _ := strconv.Atoi(m[2]) + v := version{x, y} + if v == want { + if m[3] != "" { + return "", fmt.Errorf("this QEMU deprecates %s, the machine type every checkpoint is saved on: move machine.MachineType, a change of hardware under every guest, before QEMU removes it", pinned) + } + found = true + } + if v.newer(newest) { + newest = v + } + } + if !found { + return "", fmt.Errorf("this QEMU offers no %s, the machine type every checkpoint is saved on", pinned) + } + if newest.newer(want) { + return fmt.Sprintf("QEMU offers pc-q35-%d.%d; this machine stays on %s (machine.MachineType), and moving it is a new machine: task qemu:compat says what would change.", + newest.major, newest.minor, pinned), nil + } + return "", nil +} + +// Compat is QEMU's source's hw_compat_X_Y and pc_compat_X_Y for pinned's X.Y, each array as the +// source writes it: what every type newer than pinned turns on that pinned keeps off. +func Compat(src, pinned string) (string, error) { + v, ok := parse(pinned) + if !ok { + return "", fmt.Errorf("%q is not a pc-q35-X.Y", pinned) + } + var b strings.Builder + for _, name := range []string{fmt.Sprintf("hw_compat_%d_%d", v.major, v.minor), fmt.Sprintf("pc_compat_%d_%d", v.major, v.minor)} { + arr, err := array(src, name) + if err != nil { + return "", err + } + fmt.Fprintf(&b, "%s\n", arr) + } + return strings.TrimRight(b.String(), "\n"), nil +} + +// errNoArray is a source in which QEMU has not yet given pinned's type a compat array: no newer +// type exists there to differ from it. +var errNoArray = errors.New("no such array: this QEMU has no machine type newer than the pinned one") + +// array is the declaration `GlobalProperty [] = { ... };` in src, whole. +func array(src, name string) (string, error) { + start := regexp.MustCompile(`GlobalProperty\s+` + regexp.QuoteMeta(name) + `\[\]\s*=\s*\{`).FindStringIndex(src) + if start == nil { + return "", fmt.Errorf("%s: %w", name, errNoArray) + } + end := strings.Index(src[start[0]:], "};") + if end < 0 { // mutate-exempt: the match starts with "GlobalProperty", so "};" is never at its offset 0 + return "", fmt.Errorf("%s has no end in the source", name) + } + return src[start[0] : start[0]+end+2], nil +} + +func say(w io.Writer, a ...any) { _, _ = fmt.Fprintln(w, a...) } diff --git a/hack/machinetype/main_test.go b/hack/machinetype/main_test.go new file mode 100644 index 0000000..c0e798d --- /dev/null +++ b/hack/machinetype/main_test.go @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/spin-stack/spin-machine/machine" +) + +// QEMU 11.1.2's own `-machine help`, the QEMU this tree pins. +func help1112(t *testing.T) string { + t.Helper() + b, err := os.ReadFile(filepath.Join("testdata", "help-11.1.2.txt")) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +// The pinned type stands when QEMU offers it and has not deprecated it, with a notice when a +// newer one is offered; it does not when QEMU deprecates it or no longer offers it. +func TestThePinnedTypeAgainstAQEMU(t *testing.T) { + real := help1112(t) + newer := "pc-q35-11.2 Standard PC (Q35 + ICH9, 2009)\n" + real + for _, tc := range []struct { + name, help, pinned, notice, err string + }{ + {name: "this tree's QEMU and pin", help: real, pinned: machine.MachineType}, + {name: "a QEMU with a newer type", help: newer, pinned: "pc-q35-11.1", notice: "QEMU offers pc-q35-11.2; this machine stays on pc-q35-11.1"}, + {name: "a newer major", help: "pc-q35-12.0 Standard PC\n" + real, pinned: "pc-q35-11.1", notice: "QEMU offers pc-q35-12.0"}, + {name: "an older pin, still offered", help: real, pinned: "pc-q35-10.2", notice: "QEMU offers pc-q35-11.1; this machine stays on pc-q35-10.2"}, + {name: "a deprecated pin", help: real, pinned: "pc-q35-8.1", err: "deprecates pc-q35-8.1"}, + {name: "a pin QEMU removed", help: real, pinned: "pc-q35-2.4", err: "offers no pc-q35-2.4"}, + {name: "a pin that is no versioned q35", help: real, pinned: "q35", err: "is not a pc-q35-X.Y"}, + } { + t.Run(tc.name, func(t *testing.T) { + notice, err := Check(tc.help, tc.pinned) + switch { + case tc.err != "": + if err == nil || !strings.Contains(err.Error(), tc.err) { + t.Fatalf("Check: %v, want %q", err, tc.err) + } + case err != nil: + t.Fatalf("Check: %v", err) + case tc.notice == "" && notice != "": + t.Errorf("a notice %q with nothing newer", notice) + case !strings.Contains(notice, tc.notice): + t.Errorf("notice %q, want %q", notice, tc.notice) + } + }) + } +} + +// QEMU's source as an 11.2 tarball writes it: the arrays for the pinned 11.1, each whole, and +// nothing from the array beside it. +const source = ` +GlobalProperty hw_compat_11_1[] = { + { "virtio-net-pci", "x-new-feature", "off" }, +}; +const size_t hw_compat_11_1_len = G_N_ELEMENTS(hw_compat_11_1); + +GlobalProperty hw_compat_11_0[] = { + { "older", "x", "off" }, +}; +` + +const pcSource = ` +GlobalProperty pc_compat_11_1[] = { + { TYPE_X86_CPU, "x-something", "false" }, +}; +` + +func TestCompatIsThePinnedTypesArrays(t *testing.T) { + got, err := Compat(source+pcSource, "pc-q35-11.1") + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"GlobalProperty hw_compat_11_1[] = {", `"x-new-feature", "off"`, "GlobalProperty pc_compat_11_1[] = {", `"x-something", "false"`} { + if !strings.Contains(got, want) { + t.Errorf("compat\n%s\nwant %q", got, want) + } + } + if strings.Contains(got, "older") { + t.Errorf("compat\n%s\nholds the array of another type", got) + } + // QEMU writes an array with nothing to switch back off as one line (pc_compat_11_0 in 11.1.2). + empty, err := Compat(source+"\nGlobalProperty pc_compat_11_1[] = {};\n", "pc-q35-11.1") + if err != nil || !strings.HasSuffix(empty, "GlobalProperty pc_compat_11_1[] = {};") { + t.Errorf("an empty array: %q, %v", empty, err) + } + if _, err := Compat(source, "pc-q35-11.1"); !errors.Is(err, errNoArray) { + t.Errorf("no pc_compat_11_1: %v", err) + } + if _, err := Compat("GlobalProperty hw_compat_11_1[] = { { ", "pc-q35-11.1"); err == nil || errors.Is(err, errNoArray) { + t.Errorf("an array with no end: %v", err) + } + if _, err := Compat(source, "q35"); err == nil { + t.Error("an unversioned pin") + } +} + +// An array is its declaration to its closing brace and semicolon, and not a byte past them. +func TestAnArrayIsItsDeclarationWhole(t *testing.T) { + got, err := array("before GlobalProperty a_1[] = { { \"x\", \"y\", \"z\" },\n};\nafter", "a_1") + if err != nil || got != "GlobalProperty a_1[] = { { \"x\", \"y\", \"z\" },\n};" { + t.Fatalf("array: %q, %v", got, err) + } +} + +// The command: check prints the notice and passes, fails on a pin that does not stand or a QEMU +// that cannot be asked; compat prints the arrays; anything else is no command. +func TestTheCommand(t *testing.T) { + dir := t.TempDir() + mc, pc := filepath.Join(dir, "machine.c"), filepath.Join(dir, "pc.c") + if err := os.WriteFile(mc, []byte(source), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(pc, []byte(pcSource), 0o600); err != nil { + t.Fatal(err) + } + short := filepath.Join(dir, "short.c") + if err := os.WriteFile(short, []byte("GlobalProperty hw_compat_11_1[] = { {"), 0o600); err != nil { + t.Fatal(err) + } + newer := func(string) (string, error) { return "pc-q35-11.2 Standard PC\n" + help1112(t), nil } + broken := func(string) (string, error) { return "", errors.New("no such binary") } + removed := func(string) (string, error) { return "pc-q35-11.2 Standard PC\n", nil } + for _, tc := range []struct { + name string + args []string + help func(string) (string, error) + code int + out string + }{ + {name: "check, a newer type", args: []string{"check", "-qemu", "q"}, help: newer, code: 0, out: "QEMU offers pc-q35-11.2"}, + {name: "check, nothing newer", args: []string{"check", "-qemu", "q"}, help: func(string) (string, error) { return help1112(t), nil }, code: 0}, + {name: "check, the pin removed", args: []string{"check", "-qemu", "q"}, help: removed, code: 1}, + {name: "check, a QEMU that does not answer", args: []string{"check", "-qemu", "q"}, help: broken, code: 1}, + {name: "check with no binary", args: []string{"check"}, help: newer, code: 2}, + {name: "compat", args: []string{"compat", mc, pc}, code: 0, out: "x-new-feature"}, + {name: "compat of a file not there", args: []string{"compat", mc, filepath.Join(dir, "none")}, code: 1}, + {name: "compat with one file", args: []string{"compat", mc}, code: 2}, + {name: "compat of a QEMU with nothing newer", args: []string{"compat", pc, pc}, code: 0, out: "is the newest type in this QEMU"}, + {name: "compat of a source cut short", args: []string{"compat", short, short}, code: 1}, + {name: "no command", args: nil, code: 2}, + {name: "another command", args: []string{"other"}, code: 2}, + } { + t.Run(tc.name, func(t *testing.T) { + var out, errs bytes.Buffer + if code := run(tc.args, &out, &errs, tc.help); code != tc.code { + t.Fatalf("exit %d, want %d: %s", code, tc.code, errs.String()) + } + if !strings.Contains(out.String(), tc.out) { + t.Errorf("printed %q, want %q", out.String(), tc.out) + } + if tc.out == "" && out.Len() != 0 { + t.Errorf("printed %q, want nothing", out.String()) + } + }) + } +} + +// help asks a real binary: one that is not there is an error, not an empty answer. +func TestHelpAsksTheBinary(t *testing.T) { + if _, err := help(filepath.Join(t.TempDir(), "no-qemu")); err == nil { + t.Error("no binary answered") + } +} diff --git a/hack/machinetype/testdata/help-11.1.2.txt b/hack/machinetype/testdata/help-11.1.2.txt new file mode 100644 index 0000000..de2221b --- /dev/null +++ b/hack/machinetype/testdata/help-11.1.2.txt @@ -0,0 +1,21 @@ +Supported machines are: +pc-q35-9.2 Standard PC (Q35 + ICH9, 2009) +pc-q35-9.1 Standard PC (Q35 + ICH9, 2009) +pc-q35-9.0 Standard PC (Q35 + ICH9, 2009) +pc-q35-8.2 Standard PC (Q35 + ICH9, 2009) +pc-q35-8.1 Standard PC (Q35 + ICH9, 2009) (deprecated) +pc-q35-8.0 Standard PC (Q35 + ICH9, 2009) (deprecated) +pc-q35-7.2 Standard PC (Q35 + ICH9, 2009) (deprecated) +pc-q35-7.1 Standard PC (Q35 + ICH9, 2009) (deprecated) +pc-q35-7.0 Standard PC (Q35 + ICH9, 2009) (deprecated) +pc-q35-6.2 Standard PC (Q35 + ICH9, 2009) (deprecated) +pc-q35-6.1 Standard PC (Q35 + ICH9, 2009) (deprecated) +pc-q35-6.0 Standard PC (Q35 + ICH9, 2009) (deprecated) +pc-q35-5.2 Standard PC (Q35 + ICH9, 2009) (deprecated) +q35 Standard PC (Q35 + ICH9, 2009) (alias of pc-q35-11.1) +pc-q35-11.1 Standard PC (Q35 + ICH9, 2009) +pc-q35-11.0 Standard PC (Q35 + ICH9, 2009) +pc-q35-10.2 Standard PC (Q35 + ICH9, 2009) +pc-q35-10.1 Standard PC (Q35 + ICH9, 2009) +pc-q35-10.0 Standard PC (Q35 + ICH9, 2009) +none empty machine diff --git a/hack/release b/hack/release index 9677349..1e2d172 100755 --- a/hack/release +++ b/hack/release @@ -114,14 +114,13 @@ require "${OUTPUT_DIR}/image/rootfs.qcow2" "${SHARE}/image/rootfs.qcow2" "the ba require "${OUTPUT_DIR}/packages.txt" "${SHARE}/packages.txt" "the image's package list" require "${OUTPUT_DIR}/licenses.txt" "${SHARE}/licenses.txt" "the image's licence texts" -# What this release is, in a form something other than a human can read. The qemu, kernel, -# bios and pvh checksums are the files `machine.Spec.Fingerprint` hashes (with the initrd's, -# which is not in this tarball) to decide whether a checkpoint may be resumed; the image's is -# reported beside them and is not. fingerprint_sha256 is Fingerprint itself, of the machine -# `spin-machine fingerprint` describes with a named CPU, so no runner's own CPU enters it: it -# moves with those files and also with machine/ - its devices, its shape, how it hashes - which -# a consumer builds from this release's commit. A host wondering why its checkpoints stopped -# resuming diffs this against the previous release's. +# What this release is, in a form something other than a human can read. fingerprint_sha256 is +# `machine.Spec.Fingerprint`, the machine's contract, of the machine `spin-machine fingerprint` +# describes with a named CPU, so no runner's own CPU enters it: it moves with the machine type, +# the shape, the devices and the BIOS's size, which a consumer builds from this release's commit, +# and it alone decides whether a checkpoint resumes. The files' checksums beside it say what this +# release is made of and decide nothing about checkpoints. A host wondering why its checkpoints +# stopped resuming diffs this against the previous release's (hack/fingerprint-diff). fingerprint=$(go run ./cmd/spin-machine fingerprint --release "${SHARE}" --cpu Skylake-Server-v4 | sed -n 's/^ "fingerprint": "\([0-9a-f]\{64\}\)",$/\1/p') if [ -z "${fingerprint}" ]; then diff --git a/kernel/README.md b/kernel/README.md index c3a7eac..4a0ff94 100644 --- a/kernel/README.md +++ b/kernel/README.md @@ -1,9 +1,10 @@ # Guest kernel A version and a config file (`config--`). It is here because a kernel and the -machine that boots it are one release: changing either strands every checkpoint taken -against the previous pair. `task kernel:build` builds it; `task kernel:verify` reads the ELF -notes out of the kernel it just stripped. +machine that boots it are released together. A new kernel does not strand checkpoints: a +resumed guest runs the kernel in its saved memory until it reboots, and boots this one then +(`machine.Spec.Fingerprint` hashes the machine's contract, not the kernel). `task kernel:build` +builds it; `task kernel:verify` reads the ELF notes out of the kernel it just stripped. ## Memory that moves @@ -67,16 +68,17 @@ load 6-10, where the p95 was 342 ms. ## Changing it -Every change to the config strands every checkpoint in the fleet, which is the design — -it should not be quiet. The kernel workflow runs `hack/fingerprint-diff` and puts the -answer in its summary. +A change to the config is a new kernel for every guest's next boot, and the release notes say +what changed (`hack/releasenotes` lists the options turned on, off or changed). It does not +move the fingerprint, and checkpoints keep resuming: each resumed guest goes on running the +kernel it was saved with until it reboots. `Dockerfile` builds with a pinned toolchain — Debian trixie by image digest, its packages from -snapshot.debian.org at a fixed date — because a different compiler is a different `vmlinux`, -which is a different fingerprint. Both are `versions.yaml` entries (`debian`, -`debian-snapshot`), and they move on purpose, not when Debian publishes a point release. +snapshot.debian.org at a fixed date — because a different compiler is a different `vmlinux`. +Both are `versions.yaml` entries (`debian`, `debian-snapshot`), and they move on purpose, not +when Debian publishes a point release. And the build is reproducible: two builds from the same inputs produce the same `vmlinux`, byte for byte. That needs trixie's pahole 1.30, which is deterministic with `-j`. Bookworm's 1.24 encodes `.BTF` in whatever order make's `-j` threads finish, which gives every release a -new fingerprint whether or not the kernel changed. +different kernel whether or not its source changed. diff --git a/kernel/Taskfile.yml b/kernel/Taskfile.yml index 159d66f..f73e1b5 100644 --- a/kernel/Taskfile.yml +++ b/kernel/Taskfile.yml @@ -3,10 +3,10 @@ version: '3' # The guest kernel. See kernel/Dockerfile; the kernel and its toolchain are pinned in # versions.yaml, and KERNEL_ARCH is the root Taskfile.yml's. # -# A changed kernel is a changed machine: the fingerprint hashes it by content, so every -# checkpoint on every host stops resuming. That is the point — it is how a fleet is stopped -# from resuming a checkpoint into a machine of another shape — but it means a config change -# here is a fleet-wide event and should be a deliberate commit with a measurement in it. +# A changed kernel is every guest's next boot: a resumed guest runs the kernel in its saved +# memory, so the fingerprint, the machine's contract, does not hash it and checkpoints keep +# resuming. A config change is still every guest's kernel once they reboot, and is a deliberate +# commit with a measurement in it. # The pinned version, for the name of the configuration file menuconfig writes. The builds # are handed it by `versions args`; see the note in qemu/Taskfile.yml. diff --git a/machine/machine.go b/machine/machine.go index 699ae99..ef67c94 100644 --- a/machine/machine.go +++ b/machine/machine.go @@ -11,11 +11,11 @@ // detail of whoever launches a VM; it is part of what a release *is*, and it has // to move with the binary and the kernel it was written for. // -// The rule that follows: Fingerprint hashes the QEMU binary, the kernel, the initrd -// and the firmware the guest runs by content, together with the five arguments that -// decide the machine's shape. A checkpoint resumes only onto a machine with the -// fingerprint of the one it was saved on, and a release in which any of those files -// moved has a different fingerprint by construction. +// The rule that follows: Fingerprint is the machine's contract - its versioned machine +// type, its shape, every device as the command line gives it, and the BIOS's size. A +// checkpoint resumes only onto a machine with the fingerprint of the one it was saved on, +// and onto any release with that fingerprint: a new QEMU of the same machine type, or a +// new kernel, is the same machine to a checkpoint (boot:compat holds a release to it). // // What is here is what a machine is. What is not here is everything about // running one: allocating a vsock context id, opening a TAP file descriptor, @@ -30,11 +30,11 @@ import ( "encoding/json" "errors" "fmt" - "io" "net" "os" "path/filepath" "slices" + "strconv" "strings" ) @@ -508,10 +508,8 @@ type Spec struct { // TCG can present — QEMU refuses it with "CPU model 'host' requires KVM or HVF" — so // the default below is "max" here and "host" under KVM. // - // Checkpoints do not cross the two, and nothing extra is needed to keep them apart: the - // fingerprint hashes the QEMU binary by content, and the TCG build is a different file. - // This lands in the shape as well, which makes the separation visible in an argument - // rather than only in a hash. + // Checkpoints do not cross the two: the accelerator is in the shape, which the fingerprint + // hashes, and a guest's state under TCG is not a guest's state under KVM. Accel string Disks []Disk @@ -748,7 +746,21 @@ func (s Spec) accel() string { // kernel-irqchip=on stays under either accelerator. It names KVM's in-kernel interrupt // controller and TCG has none, but QEMU accepts the option and ignores it rather than // refusing — measured against 11.1.1 — so the shape is one string and not two. -const machineOpts = "q35,kernel-irqchip=on,hpet=off,acpi=on,sata=off,smbus=off" +const machineOpts = MachineType + ",kernel-irqchip=on,hpet=off,acpi=on,sata=off,smbus=off" + +// MachineType is the versioned q35 this machine is, and never the bare "q35", which is an +// alias of whichever pc-q35-X.Y the QEMU running it is newest at. +// +// The version is what QEMU's migration promise is made of: a newer QEMU loads the state an +// older one saved when both run the same pc-q35-X.Y with the same devices, because the newer +// type's changes to defaults (hw_compat_X_Y in hw/core/machine.c, pc_compat_X_Y in +// hw/i386/pc.c) are switched back off for the older one. Under the alias a QEMU bump to a new +// minor is a new machine; under the version it is a new emulator of the same one, and a +// checkpoint resumes across it (Fingerprint). Moving this is moving every guest to new +// hardware: a decision, made here, that every checkpoint taken before it stops resuming. +// qemu:verify says when the QEMU built offers a newer one, and fails once this one is +// deprecated (QEMU deprecates a versioned type after three years and removes it after six). +const MachineType = "pc-q35-11.1" // accelArg is -accel: the accelerator, and under KVM the descriptor set /dev/kvm is in // when the caller opened it (validate refuses one for any other accelerator). @@ -1209,20 +1221,22 @@ func (s Spec) bios() string { return s.BIOS } -// Fingerprint identifies the machine this spec describes, by content. +// Fingerprint is the machine's contract: what a checkpoint saved on it needs of the machine it +// resumes into. Two machines with one fingerprint present the same hardware to a guest, and a +// checkpoint saved on one resumes on the other; across two that differ a resume is undefined +// rather than an error, because QEMU does not check it. // -// It hashes the QEMU binary, the kernel, the initrd and the firmware the guest -// runs (the BIOS and pvh.bin) — the files, not their paths — together with the -// five arguments that decide the machine's shape. Two machines with the same -// fingerprint present the same thing to a guest, and a checkpoint saved on one -// resumes on the other; across two with different fingerprints a resume is -// undefined rather than an error, because QEMU does not check it. +// It hashes the versioned machine type and the shape (Shape), the chipset's globals and every +// device as the command line gives it with what is one VM's taken out (Contract), the size of +// the BIOS, and the host's CPU when the guest is shown it. Not the QEMU binary, the kernel, the +// initrd or pvh.bin: QEMU's promise is per versioned machine type, so a newer QEMU of the same +// type is the same machine; a resumed guest runs the kernel in its saved memory, and the +// kernel, the initrd and the option ROM that entered it are read again only by a boot. +// boot:compat holds the promise: a checkpoint the newest release saved resumes on this tree's. // -// The consequence is deliberate: a release in which any of those files moved -// has a different fingerprint, so every checkpoint taken against the previous one -// stops resuming. That is the design. A checkpoint resumed into a machine of -// another shape is memory and device state loaded into hardware that is not the -// hardware it came from. +// A change to any of what it does hash is a change of hardware under every guest, and every +// checkpoint taken against the previous machine stops resuming. That is the design, and the +// release notes say so (hack/fingerprint-diff). func (s Spec) Fingerprint() (string, error) { return s.fingerprint(hostCPUModel) } @@ -1244,37 +1258,27 @@ func (s Spec) fingerprint(hostCPU func() (string, error)) (string, error) { _, _ = fmt.Fprintf(h, "%s=%d:%s\n", key, len(value), value) } - for _, f := range []struct{ name, path string }{ - {"qemu", s.QEMU}, - {"kernel", s.Kernel}, - {"initrd", s.Initrd}, - // The firmware the guest runs: the BIOS, and pvh.bin, the option ROM that enters - // the kernel. Both are ROM a saved machine carries, sized and filled by them. - {"bios", filepath.Join(s.Firmware, s.bios())}, - {"pvh", filepath.Join(s.Firmware, "pvh.bin")}, - } { - if f.path == "" { - // An absent initrd is part of the identity too: a machine that boots - // one and a machine that does not are different machines. - write(f.name, "none") - continue - } - sum, err := fileSum(f.path) - if err != nil { - return "", fmt.Errorf("fingerprinting %s: %w", f.name, err) - } - write(f.name, sum) + // The BIOS by its size. It is the one ROM a file sizes: QEMU maps it as the RAM block + // pc.bios, which a saved machine carries whole and a resume refuses at another length. Its + // contents travel with the checkpoint, so a rebuilt qboot of the same size is the same + // machine. Every other read-only block is the machine type's or the chipset's own, and + // boot:roms fails when one appears that is neither (BIOSBlock, MachineBlocks). + size, err := fileSize(filepath.Join(s.Firmware, s.bios())) + if err != nil { + return "", fmt.Errorf("fingerprinting the BIOS: %w", err) } + write("bios-size", strconv.FormatInt(size, 10)) // mutate-exempt: the base the size is written in; any base names each size once - // The machine's shape, its device topology, and the host's own CPU when the guest is - // being shown it. + // The machine's shape, its devices, and the host's own CPU when the guest is being shown it. shape := s.Shape() write("machine", shape.Machine) write("accel", shape.Accel) write("cpu", shape.CPU) write("smp", shape.SMP) write("memory", shape.Memory) - write("devices", s.topology()) + for _, c := range s.Contract() { + write("contract", c) + } // The host's own CPU, but only when the guest is being shown it. // @@ -1301,79 +1305,76 @@ func (s Spec) fingerprint(hostCPU func() (string, error)) (string, error) { return hex.EncodeToString(h.Sum(nil)), nil } -// topology is the machine's device list — which models, at which slots — with -// everything about *this* VM removed: no paths, no file descriptors, no context -// id, no MAC. +// Contract is the chipset's globals and every device of the machine, each as the command line +// gives it (appendChipset, appendDevices, appendNICs) with what is one VM's taken out, and +// whether it has a serial port. // -// It is in the fingerprint because a resume loads device state into a machine -// that has to have the same devices. Adding a device to this package changes what -// a checkpoint may resume into, and without this it would not move the hash — a -// balloon was added at slot 0x04, and every existing saved state would have gone -// on matching a machine it can no longer be loaded into. +// From the command line itself and not a description kept beside it, because a resume loads +// device state into a machine that has to have the same devices with the same properties: a +// balloon added at slot 0x04, free-page-reporting turned on, a NIC's MTU announced +// (VIRTIO_NET_F_MTU, negotiated at probe and carried in the stream) are each a machine a +// checkpoint of the other cannot be loaded into, and each moves this the moment Args changes. // -// What is deliberately not in it: the backing files and the identifiers, and the -// disks. +// What is one VM's, and so taken out, is what a resume is known to be allowed to differ in: +// an id, the backend a device is bound to (netdev, memdev), a NIC's MAC, the vsock's context +// id, which the migration stream does not carry and the guest re-reads when QEMU resets the +// transport, and a descriptor (vhostfd). The backends themselves (-netdev, -drive, -object, +// -chardev) are files and descriptors on the host, never the guest's hardware; the memory +// backend's size is the shape's. // -// The files and identifiers, because each is a thing a resume is known to be -// allowed to differ in, and each for its own reason: the vsock context id is not -// carried in the migration stream at all, and the guest re-reads it when QEMU -// resets the transport after the resume; a disk's contents are whatever file the -// resuming machine opens; and what sits behind a NIC is a host-side file -// descriptor the guest never sees. What is present here is the device *model at -// its slot*, which is what the state being loaded describes. -// -// The disks, because they are the VM's and not the machine's. A fingerprint names -// what a host runs, and a host computes it before any VM exists to say how many -// disks it has; a checkpoint is resumed with the disks it was saved with, and -// supplying them is the resuming caller's part of the contract. -func (s Spec) topology() string { - var b strings.Builder - fmt.Fprintf(&b, "vmgenid;virtio-rng-pci@%#x;virtio-balloon-pci@%#x", slotRNG, slotBalloon) - // Whose PCI hotplug is: the chipset's state, and the ACPI tables the guest read, differ with - // it (see Args), and a state loaded into the other kind is a guest whose slots nothing - // answers for. - b.WriteString(";ich9-lpc:pcie-native-hotplug") - if s.VsockCID != 0 { - fmt.Fprintf(&b, ";vhost-vsock-pci@%#x", slotVsock) - } - if s.Memory.MaxMB > s.Memory.SizeMB { - fmt.Fprintf(&b, ";virtio-mem-pci@%#x", slotMem) +// The disks are not here at all: they are the VM's and not the machine's. A host computes its +// fingerprint before any VM exists to say how many disks it has, and a checkpoint is resumed +// with the disks it was saved with, which is the resuming caller's part of the contract. +func (s Spec) Contract() []string { + args := appendChipset(nil) + args = s.appendDevices(args) + args = s.appendNICs(args) + var out []string + for i := 0; i+1 < len(args); i++ { + switch args[i] { + case "-global": + out = append(out, "global "+args[i+1]) + i++ + case "-device": + out = append(out, "device "+withoutTheVMs(args[i+1])) + i++ + } } if s.Serial != "" || s.SerialFDSet != 0 { - b.WriteString(";isa-serial") + out = append(out, "serial isa-serial") } - // The hotplug controller, which is on the command line whatever it is given. How many - // disks it may be given is not here: they are targets on the one controller, and not - // devices on the bus. - if s.HotplugDisks > 0 { - fmt.Fprintf(&b, ";virtio-scsi-pci@%#x", slotHotplug) - } - // The NICs, by how many and where, and deliberately not by MAC or by descriptor. - // - // A NIC is on the command line when the machine starts, so it is present when state is - // loaded, and a checkpoint saved on a machine without one cannot be loaded into a - // machine that has one. - // - // What is left out matters as much. A descriptor number is not the machine's shape — - // it is which file the backend reads, the way a disk's path is — and the MAC is a - // property of the device rather than of the bus. Both are left out so that machines - // which differ only in those two have one fingerprint. - // - // The MTU is here, unlike those two, because it is not a setting on the backend: it - // is VIRTIO_NET_F_MTU, negotiated at probe and carried in the migration stream, so a - // checkpoint saved with it announced cannot be loaded into a machine that does not - // announce it, or announces another value. Including it turns "the operator changed - // the MTU" into a checkpoint that is refused before the load rather than a resume - // that fails. - if len(s.NICs) > 0 { - fmt.Fprintf(&b, ";virtio-net-pci@%#x*%d", slotNICBase, len(s.NICs)) - for _, n := range s.NICs { - fmt.Fprintf(&b, ",mtu=%d", n.MTU) + return out +} + +// vmProperties are the device properties that are one VM's and not its machine's (Contract). +var vmProperties = []string{"id", "netdev", "memdev", "mac", "guest-cid", "vhostfd"} + +// withoutTheVMs is a -device value with vmProperties taken out, the rest in the order Args +// wrote them. The values Contract reads are Args' own, with no escaped comma among them: the +// one that may carry one, a disk's serial, belongs to a disk, which is not in the contract. +func withoutTheVMs(dev string) string { + parts := strings.Split(dev, ",") + kept := parts[:1] + for _, p := range parts[1:] { + if k, _, _ := strings.Cut(p, "="); !slices.Contains(vmProperties, k) { + kept = append(kept, p) } } - return b.String() + return strings.Join(kept, ",") } +// BIOSBlock is the RAM block the BIOS is mapped as, the read-only block whose size is a file's +// (Fingerprint). +const BIOSBlock = "pc.bios" + +// MachineBlocks are the read-only RAM blocks besides the BIOS that this machine maps, each sized +// by the machine type or by the chipset and not by a file: the option ROM area (a fixed 128 KiB), +// and the ACPI tables, their loader and the RSDP, which QEMU builds and resizes on a resume. +// pvh.bin and the kernel reach the guest through fw_cfg and are no block of their own, and a +// NIC loads no option ROM (romfile=). boot:roms holds a booted machine to this list, so a +// QEMU or a device that maps another ROM fails there rather than in a resume. +var MachineBlocks = []string{"pc.rom", "/rom@etc/acpi/tables", "/rom@etc/table-loader", "/rom@etc/acpi/rsdp"} + // hostCPUModel reports the host CPU's model name, which model "host" makes part // of what a checkpoint describes. // @@ -1408,16 +1409,10 @@ func hostCPUModel() (string, error) { return "", errors.New("no model name in /proc/cpuinfo") } -func fileSum(path string) (string, error) { - f, err := os.Open(path) // #nosec G304 -- a path this caller chose +func fileSize(path string) (int64, error) { + fi, err := os.Stat(path) if err != nil { - return "", err + return 0, err } - defer func() { _ = f.Close() }() - - h := sha256.New() - if _, err := io.Copy(h, f); err != nil { - return "", err - } - return hex.EncodeToString(h.Sum(nil)), nil + return fi.Size(), nil } diff --git a/machine/machine_test.go b/machine/machine_test.go index b091d7e..7038d15 100644 --- a/machine/machine_test.go +++ b/machine/machine_test.go @@ -176,7 +176,7 @@ func TestArgs(t *testing.T) { s.KVMFDSet, s.VsockFD, s.NICs[0].VhostFD = 1, 5, 6 }, want: []string{ - "-machine q35,kernel-irqchip=on,", "-accel kvm,device=/dev/fdset/1 ", + "-machine pc-q35-11.1,kernel-irqchip=on,", "-accel kvm,device=/dev/fdset/1 ", "addr=0x2,vhostfd=5 ", "tap,id=net0,fd=3,vhost=on,vhostfd=6 ", }, absent: []string{"accel=kvm"}, @@ -396,9 +396,10 @@ func TestAChainsBacking(t *testing.T) { // The fingerprint, one pair of machines per row: a is spec() changed by a, b is a // changed by b, and the row says whether a checkpoint saved on one resumes on the other. // -// It may only if the binary, the kernel, the initrd, the shape and the devices present -// when state is loaded all agree — and must be able to when all that differs is what is -// behind the devices, or a checkpoint resumes nowhere but on the VM that saved it. +// It may only if the machine type, the shape, the devices present when state is loaded and the +// BIOS's size all agree — and must be able to when all that differs is the emulator, what a +// boot reads, or what is behind the devices, or a checkpoint resumes nowhere but on the release +// that saved it. func TestFingerprint(t *testing.T) { const intel, amd = "13th Gen Intel(R) Core(TM) i9-13900HK", "AMD EPYC 9634 84-Core Processor" rewrite := func(t *testing.T, path, content string) { @@ -468,16 +469,25 @@ func TestFingerprint(t *testing.T) { {name: "more memory", b: func(_ *testing.T, s *Spec) { s.Memory.SizeMB = 1024 }}, {name: "more vCPUs", b: func(_ *testing.T, s *Spec) { s.BootCPUs = 4 }}, - {name: "a new kernel", b: func(t *testing.T, s *Spec) { rewrite(t, s.Kernel, "a different kernel") }}, - {name: "a new QEMU", b: func(t *testing.T, s *Spec) { rewrite(t, s.QEMU, "a different qemu") }}, - // The firmware runs in the guest, and its ROM is carried by a saved machine. - {name: "a new BIOS", b: func(t *testing.T, s *Spec) { + // A resumed guest runs the kernel in its saved memory: the file is read by a boot. + {name: "a new kernel", same: true, b: func(t *testing.T, s *Spec) { rewrite(t, s.Kernel, "a different kernel") }}, + {name: "an initrd", same: true, b: func(t *testing.T, s *Spec) { s.Initrd = s.Kernel }}, + // Another QEMU of the same versioned machine type is another emulator of one machine: + // QEMU's migration promise, which boot:compat holds a release to. + {name: "a new QEMU", same: true, b: func(t *testing.T, s *Spec) { rewrite(t, s.QEMU, "a different qemu") }}, + // pvh.bin enters the kernel through fw_cfg at a boot and is no block a resume loads. + {name: "a new pvh.bin", same: true, b: func(t *testing.T, s *Spec) { + rewrite(t, filepath.Join(s.Firmware, "pvh.bin"), "a different pvh") + }}, + // The BIOS is the pc.bios block a saved machine carries whole, contents included: one + // of another length is refused, and a rebuild of the same length is the same machine. + {name: "a BIOS rebuilt at its size", same: true, b: func(t *testing.T, s *Spec) { + rewrite(t, filepath.Join(s.Firmware, DefaultBIOS), "QBOOT") + }}, + {name: "a BIOS of another size", b: func(t *testing.T, s *Spec) { rewrite(t, filepath.Join(s.Firmware, DefaultBIOS), "a different qboot") }}, {name: "SeaBIOS instead of qboot", b: func(_ *testing.T, s *Spec) { s.BIOS = "bios-256k.bin" }}, - {name: "a new pvh.bin", b: func(t *testing.T, s *Spec) { - rewrite(t, filepath.Join(s.Firmware, "pvh.bin"), "a different pvh") - }}, {name: "the same firmware in another directory", same: true, b: func(t *testing.T, s *Spec) { s.Firmware = spec(t).Firmware }}, @@ -771,36 +781,56 @@ func TestValidateRefuses(t *testing.T) { } } -// What the fingerprint says is present when a checkpoint's state is loaded. Two machines -// that differ in one of these differ on the bus, so each has to be in the string when the -// device is there and out of it when it is not; a check that is only "the two -// fingerprints differ" holds none of them, because an inverted test still differs. -func TestTopologyNamesTheDevicesPresentAtRestore(t *testing.T) { +// What the contract says is present when a checkpoint's state is loaded, each device with its +// properties and its slot and without what is one VM's. Two machines that differ in one of these +// differ on the bus, so each has to be in the contract when the device is there and out of it +// when it is not; a check that is only "the two fingerprints differ" holds none of them, because +// an inverted test still differs. +func TestTheContractIsTheDevicesPresentAtRestore(t *testing.T) { for _, tc := range []struct { name string set func(*Spec) - want string - absent string + want []string + absent []string }{ - {name: "no vsock", set: func(*Spec) {}, absent: "vhost-vsock-pci"}, - {name: "a vsock", set: func(s *Spec) { s.VsockCID = 7 }, want: ";vhost-vsock-pci@0x2"}, - {name: "no memory ceiling", set: func(s *Spec) { s.Memory.MaxMB = s.Memory.SizeMB }, absent: "virtio-mem-pci"}, - {name: "a memory ceiling", set: func(s *Spec) { s.Memory.MaxMB = s.Memory.SizeMB + 1 }, want: ";virtio-mem-pci@0x1e"}, - {name: "no console", set: func(*Spec) {}, absent: "isa-serial"}, - {name: "a console by path", set: func(s *Spec) { s.Serial = "file:/log" }, want: ";isa-serial"}, - {name: "a console by descriptor", set: func(s *Spec) { s.SerialFDSet = 1 }, want: ";isa-serial"}, - {name: "no hotplug controller", set: func(*Spec) {}, absent: "virtio-scsi-pci"}, - {name: "a hotplug controller", set: func(s *Spec) { s.HotplugDisks = 1 }, want: ";virtio-scsi-pci@0x1a"}, + {name: "the machine's own", set: func(*Spec) {}, + want: []string{ + "device vmgenid,guid=auto", + "device virtio-rng-pci,disable-legacy=on,addr=0x3", + "device virtio-balloon-pci,free-page-reporting=on,deflate-on-oom=on,disable-legacy=on,addr=0x4", + "global ICH9-LPC.disable_s3=1", + "global ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off", + }, + absent: []string{"vhost-vsock-pci", "virtio-mem-pci", "isa-serial", "virtio-scsi-pci", "virtio-net-pci", "virtio-blk-pci"}}, + {name: "a vsock, without its context id or descriptor", set: func(s *Spec) { s.VsockCID, s.VsockFD = 7, 14 }, + want: []string{"device vhost-vsock-pci,disable-legacy=on,addr=0x2"}, absent: []string{"guest-cid", "vhostfd"}}, + {name: "a memory ceiling, without its backend", set: func(s *Spec) { s.Memory.MaxMB = s.Memory.SizeMB + 1 }, + want: []string{"device virtio-mem-pci,requested-size=0,disable-legacy=on,addr=0x1e"}, absent: []string{"memdev", ",id="}}, + {name: "a console by path", set: func(s *Spec) { s.Serial = "file:/log" }, want: []string{"serial isa-serial"}}, + {name: "a console by descriptor", set: func(s *Spec) { s.SerialFDSet = 1 }, want: []string{"serial isa-serial"}}, + {name: "a hotplug controller", set: func(s *Spec) { s.HotplugDisks = 1 }, + want: []string{"device virtio-scsi-pci,disable-legacy=on,addr=0x1a"}}, + {name: "a NIC, by its slot and MTU and not its MAC or backend", set: func(s *Spec) { + n := nic() + n.MTU, n.VhostFD = 1500, 15 + s.NICs = []NIC{n} + }, want: []string{"device virtio-net-pci,romfile=,disable-legacy=on,addr=0x10,host_mtu=1500"}, absent: []string{"mac=", "netdev", "vhostfd"}}, + {name: "a disk is the VM's", set: func(s *Spec) { s.Disks = []Disk{{Path: "/a", Format: "raw", Serial: "x"}} }, + absent: []string{"virtio-blk-pci", "serial=x"}}, } { t.Run(tc.name, func(t *testing.T) { s := spec(t) tc.set(&s) - got := s.topology() - if tc.want != "" && !strings.Contains(got, tc.want) { - t.Errorf("topology %q, want it to hold %q", got, tc.want) + got := strings.Join(s.Contract(), "\n") + for _, w := range tc.want { + if !slices.Contains(s.Contract(), w) { + t.Errorf("contract\n%s\nwant it to hold %q", got, w) + } } - if tc.absent != "" && strings.Contains(got, tc.absent) { - t.Errorf("topology %q, want no %q", got, tc.absent) + for _, a := range tc.absent { + if strings.Contains(got, a) { + t.Errorf("contract\n%s\nwant no %q", got, a) + } } }) } diff --git a/machine/release.go b/machine/release.go index 8e08499..9280697 100644 --- a/machine/release.go +++ b/machine/release.go @@ -121,7 +121,7 @@ func (r *Release) optional(name, what string) (string, error) { // tree has no manifest. // // It names a build for a human reading a log. It is not what decides whether a -// checkpoint may be resumed — Fingerprint is, by content — so two hosts agreeing +// checkpoint may be resumed — Fingerprint, the machine's contract, is — so two hosts agreeing // on this string is evidence and not proof. func (r *Release) Version() string { return r.env["version"] } diff --git a/qemu/Taskfile.yml b/qemu/Taskfile.yml index 60cbd92..f760d62 100644 --- a/qemu/Taskfile.yml +++ b/qemu/Taskfile.yml @@ -84,6 +84,26 @@ tasks: exit 1; } echo "OK: both qemu-system binaries run here, qemu-img runs, six firmware files," echo " and only the CI binary can emulate" + # The versioned q35 every checkpoint is saved on (machine.MachineType), offered and not + # deprecated by both binaries; a newer one is said here and in a release's notes, and is + # not a failure: moving to it is a decision, not a side effect of a bump. + - for: [qemu-system-x86_64, qemu-system-x86_64-tcg] + cmd: go run ./hack/machinetype check -qemu {{.OUTPUT_ABS}}/bin/{{.ITEM}} + + compat: + desc: >- + Print what QEMU's newer machine types change against the pinned one (machine.MachineType): + hw_compat_X_Y and pc_compat_X_Y from the source of the pinned QEMU, checked against its + sha256. task bump NAME=qemu runs it. + cmds: + - | + set -euo pipefail + eval "$({{.VERSIONS}} env qemu)" + tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT + curl -fsSL "$({{.VERSIONS}} url qemu)" -o "$tmp/qemu.tar.xz" + echo "${QEMU_SHA256} $tmp/qemu.tar.xz" | sha256sum -c - > /dev/null + tar -xJf "$tmp/qemu.tar.xz" -C "$tmp" "qemu-${QEMU_VERSION}/hw/core/machine.c" "qemu-${QEMU_VERSION}/hw/i386/pc.c" + go run ./hack/machinetype compat "$tmp/qemu-${QEMU_VERSION}/hw/core/machine.c" "$tmp/qemu-${QEMU_VERSION}/hw/i386/pc.c" fetch: desc: >- diff --git a/qemu/qboot/README.md b/qemu/qboot/README.md index 76aca5b..e6dbf18 100644 --- a/qemu/qboot/README.md +++ b/qemu/qboot/README.md @@ -70,6 +70,8 @@ The same commit, patch and flags saved 6.3 ms over SeaBIOS built by a host's GCC recipe is pinned rather than convenient, and why a number here compares only with one built the same way. Against this machine's real boot, the firmware is about 10 ms of a few hundred. -Firmware contents are part of Spec.Fingerprint — the BIOS and pvh.bin, by content — so -two firmware builds are two machines, and a checkpoint saved under one does not resume -under the other. +The BIOS's size is part of Spec.Fingerprint and its contents are not: it is the `pc.bios` RAM +block, which a checkpoint carries whole, so a resumed guest keeps running the qboot it booted +with and only a block of another length is refused. qboot.bin is linked for its 64 KiB window +(`qemu:verify` holds it there), so a rebuilt qboot is the same machine to a checkpoint and new +firmware to the next boot. pvh.bin reaches the guest through fw_cfg at a boot and is not in it. diff --git a/versions.yaml b/versions.yaml index a6cd106..ba3f505 100644 --- a/versions.yaml +++ b/versions.yaml @@ -42,8 +42,9 @@ pin: 731b5681e4bb18be313231579b8efd0296c5b015fa36dc533874b639ba838016 track: tags https://gitlab.com/qemu-project/qemu.git note: >- - qemu/Dockerfile. A new machine. After the bump: `task qemu:build verify:args`, and the lane's - fingerprint-diff says what it breaks. The build directories' cache mounts are keyed by the + qemu/Dockerfile. A new emulator of the same machine: the machine type (machine.MachineType) + stays, and the bump prints what newer types change against it (task qemu:compat). After the + bump: `task qemu:build verify:args boot:roms boot:compat`. The build directories' cache mounts are keyed by the version, so QEMU_CONFIG_REV does not move with it. A device renamed or dropped upstream fails qemu:verify or verify:args, and is fixed in qemu/devices.mak or the machine package, not by removing the check. @@ -69,7 +70,8 @@ pin: 382d6aa48640351df7da6321e156c39155b4a99b0985d4370f7e608729987453 track: kernel-stable note: >- - kernel/Dockerfile. A new machine. The configuration + kernel/Dockerfile. A new kernel for every guest's next boot; checkpoints keep resuming, each + on the kernel it was saved with (machine.Spec.Fingerprint). The configuration is kernel/config--x86_64: rename it to the new version in the same commit, then `task kernel:build`, `task boot:bench`. A series that is over is a decision, not a bump. Pinned at 7.3's release candidates, from git.kernel.org, until 7.3.0 is out: the bump to it