diff --git a/cmd/core/vmconfig.go b/cmd/core/vmconfig.go index e9eca739..cf246f3d 100644 --- a/cmd/core/vmconfig.go +++ b/cmd/core/vmconfig.go @@ -35,8 +35,13 @@ func VMConfigFromFlags(cmd *cobra.Command, image string) (*types.VMConfig, error windows, _ := cmd.Flags().GetBool("windows") sharedMemory, _ := cmd.Flags().GetBool("shared-memory") hugePages, _ := cmd.Flags().GetBool("hugepages") + mergeable, _ := cmd.Flags().GetBool("mergeable") dataDiskRaw, _ := cmd.Flags().GetStringArray("data-disk") + if mergeable && (hugePages || sharedMemory) { + return nil, fmt.Errorf("--mergeable needs plain private memory; drop --hugepages/--shared-memory") + } + vmName = cmp.Or(vmName, sanitizeVMName(image)) memBytes, err := units.RAMInBytes(memStr) @@ -67,6 +72,7 @@ func VMConfigFromFlags(cmd *cobra.Command, image string) (*types.VMConfig, error Windows: windows, SharedMemory: sharedMemory, HugePages: hugePages, + Mergeable: mergeable, CPUWeight: cpuWeight, CPUQuotaUs: cpuQuotaUs, CPUPeriodUs: cpuPeriodUs, @@ -131,6 +137,7 @@ func CloneVMConfigFromFlags(cmd *cobra.Command, snapCfg types.SnapshotConfig) (* Windows: snapCfg.Windows, SharedMemory: snapCfg.SharedMemory, HugePages: snapCfg.HugePages, + Mergeable: snapCfg.Mergeable, CPUWeight: flagCPUWeight, CPUQuotaUs: flagCPUQuotaUs, CPUPeriodUs: flagCPUPeriodUs, diff --git a/cmd/vm/commands.go b/cmd/vm/commands.go index 8ea2d35b..0e400f1c 100644 --- a/cmd/vm/commands.go +++ b/cmd/vm/commands.go @@ -339,6 +339,7 @@ func addVMFlags(cmd *cobra.Command) { cmd.Flags().Bool("windows", false, "Windows guest (UEFI boot, kvm_hyperv=on, no cidata)") cmd.Flags().Bool("shared-memory", false, "enable CH memory shared=on; required to attach vhost-user-fs later (CH only, fixed for VM lifetime)") cmd.Flags().Bool("hugepages", false, "back guest memory with hugetlbfs (CH only, fixed for VM lifetime); snapshots of such a VM restore via eager copy, never mmap") + cmd.Flags().Bool("mergeable", false, "mark guest memory MADV_MERGEABLE so host KSM can dedup it (CH only, fixed for VM lifetime); needs KSM enabled on the host, excludes --hugepages/--shared-memory") cmd.Flags().StringArray("data-disk", nil, "extra data disk: size=20G[,name=...][,fstype=ext4|none][,mount=/mnt/x][,directio=on|off|auto]; repeatable") } diff --git a/cmd/vm/debug.go b/cmd/vm/debug.go index 3b02d565..143e3f6a 100644 --- a/cmd/vm/debug.go +++ b/cmd/vm/debug.go @@ -221,15 +221,8 @@ func printCommonCHArgs(s chDebugSpec) { if s.VMCfg.Windows { cpuExtra = ",kvm_hyperv=on" } - memExtra := "" - if s.VMCfg.HugePages { - memExtra += ",hugepages=on" - } - if s.VMCfg.SharedMemory { - memExtra += ",shared=on" - } fmt.Printf(" --cpus boot=%d,max=%d%s \\\n", s.VMCfg.CPU, s.MaxCPU, cpuExtra) - fmt.Printf(" --memory size=%dM%s \\\n", s.VMCfg.Memory>>20, memExtra) //nolint:mnd + fmt.Printf(" --memory %s \\\n", cloudhypervisor.DebugMemoryCLIArg(&s.VMCfg.Config)) fmt.Print(" --rng src=/dev/urandom \\\n") if s.Balloon > 0 { fmt.Printf(" --balloon size=%dM,deflate_on_oom=on,free_page_reporting=on \\\n", s.Balloon) diff --git a/cmd/vm/run.go b/cmd/vm/run.go index b81e3fe2..ef424e79 100644 --- a/cmd/vm/run.go +++ b/cmd/vm/run.go @@ -489,6 +489,8 @@ func validateBackendFlags(conf *config.Config, vmCfg *types.VMConfig) error { return fmt.Errorf("--fc and --shared-memory are mutually exclusive: Firecracker does not support vhost-user-fs hot-plug") case vmCfg.HugePages: return fmt.Errorf("--fc and --hugepages are mutually exclusive: Firecracker cannot restore hugetlbfs-backed snapshots") + case vmCfg.Mergeable: + return fmt.Errorf("--fc and --mergeable are mutually exclusive: Firecracker has no KSM madvise knob") } return nil } diff --git a/docs/cli.md b/docs/cli.md index e95e4a38..7252c320 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -105,6 +105,7 @@ Applies to `cocoon vm create`, `cocoon vm run`, and `cocoon vm debug`: | `--windows` | `false` | Windows guest (UEFI boot, kvm_hyperv=on, no cidata) | | `--shared-memory` | `false` | Enable CH `memory shared=on`; required for later `vm fs attach` (CH only, fixed for VM lifetime) | | `--hugepages` | `false` | Back guest memory with hugetlbfs (CH only, fixed for VM lifetime); snapshots of such a VM restore via eager copy, never mmap | +| `--mergeable` | `false` | Mark guest memory `MADV_MERGEABLE` for host KSM dedup (CH only, fixed for VM lifetime; persists through clone/restore); needs KSM enabled on the host, excludes `--hugepages`/`--shared-memory` | | `--cpu-weight` | `0` (= vCPU count) | cgroup `cpu.weight` 1..10000 — work-conserving share under host contention | | `--cpu-quota-us` | `0` (= vCPU count × period) | cgroup `cpu.max` quota in µs per period — the hard CPU ceiling | | `--cpu-period-us` | `0` (= 100000) | cgroup `cpu.max` period in µs | diff --git a/docs/vm.md b/docs/vm.md index 9d53e8f3..889ab8d8 100644 --- a/docs/vm.md +++ b/docs/vm.md @@ -86,6 +86,7 @@ With `cgroup_cpus=0-14`, the reserved core 15 has no VM competition and acts as ## Performance Tuning - **Hugepages** (Cloud Hypervisor only): opt-in via `vm create --hugepages`; VM memory is backed by 2 MiB hugepages for reduced TLB pressure, and in exchange snapshots of that VM restore via eager copy only (the mmap fast path needs plain private-anon memory). Firecracker rejects `--hugepages`: FC cannot restore a hugetlbfs-backed snapshot, which would break hibernate/clone +- **Mergeable memory / KSM** (Cloud Hypervisor only): opt-in via `--mergeable` at golden creation; guest memory is madvised `MADV_MERGEABLE` so host KSM can dedup identical pages across VMs — the flag persists through snapshot/clone/restore (it lives in the snapshot's CH config, not the CLI), so build the golden with it or rebuild. cocoon only sets the madvise: enabling and tuning the scanner (`/sys/kernel/mm/ksm/run`, `pages_to_scan`) is the operator's. Excludes `--hugepages`/`--shared-memory` (KSM merges only plain private pages); mmap-cloned siblings already share untouched pages via the page cache, so KSM's gain is dirtied-but-equal and cross-golden pages — measure density on your fleet, and weigh ksmd CPU plus the cross-VM dedup timing side channel in multi-tenant setups - **Disk I/O**: multi-queue virtio-blk; readonly base disks keep host page cache (`direct=off`), while writable raw/qcow2 COW disks use O_DIRECT (`direct=on`) to avoid host cache buildup and guest flush storms - **Balloon**: 25% of memory auto-returned via virtio-balloon with deflate-on-OOM and free-page reporting (VMs with < 256 MiB memory skip balloon) - **Watchdog**: hardware watchdog enabled by default for automatic guest reset on hang diff --git a/hypervisor/cloudhypervisor/api.go b/hypervisor/cloudhypervisor/api.go index 96349b44..edd93c20 100644 --- a/hypervisor/cloudhypervisor/api.go +++ b/hypervisor/cloudhypervisor/api.go @@ -46,6 +46,7 @@ type chMemory struct { Size int64 `json:"size"` HugePages bool `json:"hugepages,omitempty"` Shared bool `json:"shared,omitempty"` + Mergeable bool `json:"mergeable,omitempty"` } type chDisk struct { diff --git a/hypervisor/cloudhypervisor/args.go b/hypervisor/cloudhypervisor/args.go index 246e2683..d5152d4e 100644 --- a/hypervisor/cloudhypervisor/args.go +++ b/hypervisor/cloudhypervisor/args.go @@ -38,13 +38,18 @@ func DebugDiskCLIArgs(storageConfigs []*types.StorageConfig, cpuCount, diskQueue return args } +// DebugMemoryCLIArg uses the same memory mapping as launch. +func DebugMemoryCLIArg(cfg *types.Config) string { + return memoryCLIArg(chMemory{Size: cfg.Memory, HugePages: cfg.HugePages, Shared: cfg.SharedMemory, Mergeable: cfg.Mergeable}) +} + func buildVMConfig(rec *hypervisor.VMRecord, consoleSockPath string, allowed []int) *chVMConfig { cpu := rec.Config.CPU mem := rec.Config.Memory cfg := &chVMConfig{ CPUs: chCPUs{BootVCPUs: cpu, MaxVCPUs: hypervisor.HostCPUCount(), KVMHyperV: rec.Config.Windows}, - Memory: chMemory{Size: mem, HugePages: rec.Config.HugePages, Shared: rec.Config.SharedMemory}, + Memory: chMemory{Size: mem, HugePages: rec.Config.HugePages, Shared: rec.Config.SharedMemory, Mergeable: rec.Config.Mergeable}, RNG: chRNG{Src: "/dev/urandom"}, Watchdog: true, Vsock: &chVsock{CID: hypervisor.VsockGuestCID, Socket: hypervisor.VsockSockPath(rec.RunDir)}, @@ -93,14 +98,7 @@ func buildCLIArgs(cfg *chVMConfig, socketPath string) []string { cpuKV.addIf(cfg.CPUs.KVMHyperV, "kvm_hyperv=on") args = append(args, "--cpus", cpuKV.String()) - mem := fmt.Sprintf("size=%d", cfg.Memory.Size) - if cfg.Memory.HugePages { - mem += ",hugepages=on" - } - if cfg.Memory.Shared { - mem += ",shared=on" - } - args = append(args, "--memory", mem) + args = append(args, "--memory", memoryCLIArg(cfg.Memory)) if len(cfg.Disks) > 0 { args = append(args, "--disk") @@ -230,6 +228,15 @@ func queueAffinity(cpuCount int, allowed []int) []chQueueAffinity { return qa } +func memoryCLIArg(m chMemory) string { + var kv kvBuilder + kv.add(fmt.Sprintf("size=%d", m.Size)) + kv.addIf(m.HugePages, "hugepages=on") + kv.addIf(m.Shared, "shared=on") + kv.addIf(m.Mergeable, "mergeable=on") + return kv.String() +} + func diskToCLIArg(d chDisk) string { var b kvBuilder b.add("path=" + d.Path) diff --git a/hypervisor/cloudhypervisor/args_test.go b/hypervisor/cloudhypervisor/args_test.go new file mode 100644 index 00000000..8283adbd --- /dev/null +++ b/hypervisor/cloudhypervisor/args_test.go @@ -0,0 +1,32 @@ +package cloudhypervisor + +import ( + "slices" + "strings" + "testing" + + "github.com/cocoonstack/cocoon/hypervisor" + "github.com/cocoonstack/cocoon/types" +) + +func TestMemoryCLIArg(t *testing.T) { + tests := []struct { + name string + cfg types.Config + want string + }{ + {name: "plain", cfg: types.Config{Memory: 1 << 30}, want: "size=1073741824"}, + {name: "hugepages+shared", cfg: types.Config{Memory: 1 << 30, HugePages: true, SharedMemory: true}, want: "size=1073741824,hugepages=on,shared=on"}, + {name: "mergeable", cfg: types.Config{Memory: 1 << 30, Mergeable: true}, want: "size=1073741824,mergeable=on"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rec := &hypervisor.VMRecord{VM: types.VM{Config: types.VMConfig{Config: tt.cfg}}} + args := buildCLIArgs(buildVMConfig(rec, "", nil), "api.sock") + i := slices.Index(args, "--memory") + if i < 0 || i+1 >= len(args) || args[i+1] != tt.want { + t.Fatalf("memory arg not %q (args: %s)", tt.want, strings.Join(args, " ")) + } + }) + } +} diff --git a/hypervisor/firecracker/create.go b/hypervisor/firecracker/create.go index fc78a18e..83299fc2 100644 --- a/hypervisor/firecracker/create.go +++ b/hypervisor/firecracker/create.go @@ -30,6 +30,9 @@ func (fc *Firecracker) Create(ctx context.Context, id string, vmCfg *types.VMCon if vmCfg.HugePages { return nil, fmt.Errorf("firecracker does not support hugepages (restore cannot map hugetlbfs-backed snapshots)") } + if vmCfg.Mergeable { + return nil, fmt.Errorf("firecracker does not support mergeable memory (no KSM madvise knob)") + } if !hypervisor.IsDirectBoot(bootCfg) { return nil, fmt.Errorf("firecracker requires direct kernel boot (OCI image)") } diff --git a/types/config.go b/types/config.go index 9cd79714..92d224a5 100644 --- a/types/config.go +++ b/types/config.go @@ -23,6 +23,8 @@ type Config struct { SharedMemory bool `json:"shared_memory,omitempty"` // HugePages backs CH guest memory with hugetlbfs (costs snapshots the mmap fast path); fixed at create, persists through clone/restore. HugePages bool `json:"hugepages,omitempty"` + // Mergeable marks CH guest memory MADV_MERGEABLE for host KSM dedup (needs plain private memory); fixed at create, persists through clone/restore. + Mergeable bool `json:"mergeable,omitempty"` // Raw cgroup v2 CPU knobs; zero derives the Guaranteed-at-N defaults from CPU (CPUSetCPUs empty = no placement). CPUWeight int `json:"cpu_weight,omitempty"`