vm: mergeable guest memory for host KSM dedup

cmd/core/vmconfig.go

@@ -35,8 +35,13 @@ func VMConfigFromFlags(cmd *cobra.Command, image string) (*types.VMConfig, error
 	windows, _ := cmd.Flags().GetBool("windows")
 	sharedMemory, _ := cmd.Flags().GetBool("shared-memory")
 	hugePages, _ := cmd.Flags().GetBool("hugepages")
+	mergeable, _ := cmd.Flags().GetBool("mergeable")
 	dataDiskRaw, _ := cmd.Flags().GetStringArray("data-disk")
 
+	if mergeable && (hugePages || sharedMemory) {
+		return nil, fmt.Errorf("--mergeable needs plain private memory; drop --hugepages/--shared-memory")
+	}
+
 	vmName = cmp.Or(vmName, sanitizeVMName(image))
 
 	memBytes, err := units.RAMInBytes(memStr)
@@ -67,6 +72,7 @@ func VMConfigFromFlags(cmd *cobra.Command, image string) (*types.VMConfig, error
 			Windows:       windows,
 			SharedMemory:  sharedMemory,
 			HugePages:     hugePages,
+			Mergeable:     mergeable,
 			CPUWeight:     cpuWeight,
 			CPUQuotaUs:    cpuQuotaUs,
 			CPUPeriodUs:   cpuPeriodUs,
@@ -131,6 +137,7 @@ func CloneVMConfigFromFlags(cmd *cobra.Command, snapCfg types.SnapshotConfig) (*
 			Windows:       snapCfg.Windows,
 			SharedMemory:  snapCfg.SharedMemory,
 			HugePages:     snapCfg.HugePages,
+			Mergeable:     snapCfg.Mergeable,
 			CPUWeight:     flagCPUWeight,
 			CPUQuotaUs:    flagCPUQuotaUs,
 			CPUPeriodUs:   flagCPUPeriodUs,

cmd/vm/commands.go

@@ -339,6 +339,7 @@ func addVMFlags(cmd *cobra.Command) {
 	cmd.Flags().Bool("windows", false, "Windows guest (UEFI boot, kvm_hyperv=on, no cidata)")
 	cmd.Flags().Bool("shared-memory", false, "enable CH memory shared=on; required to attach vhost-user-fs later (CH only, fixed for VM lifetime)")
 	cmd.Flags().Bool("hugepages", false, "back guest memory with hugetlbfs (CH only, fixed for VM lifetime); snapshots of such a VM restore via eager copy, never mmap")
+	cmd.Flags().Bool("mergeable", false, "mark guest memory MADV_MERGEABLE so host KSM can dedup it (CH only, fixed for VM lifetime); needs KSM enabled on the host, excludes --hugepages/--shared-memory")
 	cmd.Flags().StringArray("data-disk", nil, "extra data disk: size=20G[,name=...][,fstype=ext4|none][,mount=/mnt/x][,directio=on|off|auto]; repeatable")
 }
 

cmd/vm/debug.go

@@ -221,15 +221,8 @@ func printCommonCHArgs(s chDebugSpec) {
 	if s.VMCfg.Windows {
 		cpuExtra = ",kvm_hyperv=on"
 	}
-	memExtra := ""
-	if s.VMCfg.HugePages {
-		memExtra += ",hugepages=on"
-	}
-	if s.VMCfg.SharedMemory {
-		memExtra += ",shared=on"
-	}
 	fmt.Printf("  --cpus boot=%d,max=%d%s \\\n", s.VMCfg.CPU, s.MaxCPU, cpuExtra)
-	fmt.Printf("  --memory size=%dM%s \\\n", s.VMCfg.Memory>>20, memExtra) //nolint:mnd
+	fmt.Printf("  --memory %s \\\n", cloudhypervisor.DebugMemoryCLIArg(&s.VMCfg.Config))
 	fmt.Print("  --rng src=/dev/urandom \\\n")
 	if s.Balloon > 0 {
 		fmt.Printf("  --balloon size=%dM,deflate_on_oom=on,free_page_reporting=on \\\n", s.Balloon)

cmd/vm/run.go

@@ -489,6 +489,8 @@ func validateBackendFlags(conf *config.Config, vmCfg *types.VMConfig) error {
 		return fmt.Errorf("--fc and --shared-memory are mutually exclusive: Firecracker does not support vhost-user-fs hot-plug")
 	case vmCfg.HugePages:
 		return fmt.Errorf("--fc and --hugepages are mutually exclusive: Firecracker cannot restore hugetlbfs-backed snapshots")
+	case vmCfg.Mergeable:
+		return fmt.Errorf("--fc and --mergeable are mutually exclusive: Firecracker has no KSM madvise knob")
 	}
 	return nil
 }

docs/cli.md

@@ -105,6 +105,7 @@ Applies to `cocoon vm create`, `cocoon vm run`, and `cocoon vm debug`:
 | `--windows` | `false`          | Windows guest (UEFI boot, kvm_hyperv=on, no cidata) |
 | `--shared-memory` | `false`     | Enable CH `memory shared=on`; required for later `vm fs attach` (CH only, fixed for VM lifetime) |
 | `--hugepages` | `false`         | Back guest memory with hugetlbfs (CH only, fixed for VM lifetime); snapshots of such a VM restore via eager copy, never mmap |
+| `--mergeable` | `false`         | Mark guest memory `MADV_MERGEABLE` for host KSM dedup (CH only, fixed for VM lifetime; persists through clone/restore); needs KSM enabled on the host, excludes `--hugepages`/`--shared-memory` |
 | `--cpu-weight` | `0` (= vCPU count) | cgroup `cpu.weight` 1..10000 — work-conserving share under host contention |
 | `--cpu-quota-us` | `0` (= vCPU count × period) | cgroup `cpu.max` quota in µs per period — the hard CPU ceiling |
 | `--cpu-period-us` | `0` (= 100000) | cgroup `cpu.max` period in µs |

docs/vm.md

@@ -86,6 +86,7 @@ With `cgroup_cpus=0-14`, the reserved core 15 has no VM competition and acts as
 ## Performance Tuning
 
 - **Hugepages** (Cloud Hypervisor only): opt-in via `vm create --hugepages`; VM memory is backed by 2 MiB hugepages for reduced TLB pressure, and in exchange snapshots of that VM restore via eager copy only (the mmap fast path needs plain private-anon memory). Firecracker rejects `--hugepages`: FC cannot restore a hugetlbfs-backed snapshot, which would break hibernate/clone
+- **Mergeable memory / KSM** (Cloud Hypervisor only): opt-in via `--mergeable` at golden creation; guest memory is madvised `MADV_MERGEABLE` so host KSM can dedup identical pages across VMs — the flag persists through snapshot/clone/restore (it lives in the snapshot's CH config, not the CLI), so build the golden with it or rebuild. cocoon only sets the madvise: enabling and tuning the scanner (`/sys/kernel/mm/ksm/run`, `pages_to_scan`) is the operator's. Excludes `--hugepages`/`--shared-memory` (KSM merges only plain private pages); mmap-cloned siblings already share untouched pages via the page cache, so KSM's gain is dirtied-but-equal and cross-golden pages — measure density on your fleet, and weigh ksmd CPU plus the cross-VM dedup timing side channel in multi-tenant setups
 - **Disk I/O**: multi-queue virtio-blk; readonly base disks keep host page cache (`direct=off`), while writable raw/qcow2 COW disks use O_DIRECT (`direct=on`) to avoid host cache buildup and guest flush storms
 - **Balloon**: 25% of memory auto-returned via virtio-balloon with deflate-on-OOM and free-page reporting (VMs with < 256 MiB memory skip balloon)
 - **Watchdog**: hardware watchdog enabled by default for automatic guest reset on hang

hypervisor/cloudhypervisor/api.go

@@ -46,6 +46,7 @@ type chMemory struct {
 	Size      int64 `json:"size"`
 	HugePages bool  `json:"hugepages,omitempty"`
 	Shared    bool  `json:"shared,omitempty"`
+	Mergeable bool  `json:"mergeable,omitempty"`
 }
 
 type chDisk struct {

hypervisor/cloudhypervisor/args.go

@@ -38,13 +38,18 @@ func DebugDiskCLIArgs(storageConfigs []*types.StorageConfig, cpuCount, diskQueue
 	return args
 }
 
+// DebugMemoryCLIArg uses the same memory mapping as launch.
+func DebugMemoryCLIArg(cfg *types.Config) string {
+	return memoryCLIArg(chMemory{Size: cfg.Memory, HugePages: cfg.HugePages, Shared: cfg.SharedMemory, Mergeable: cfg.Mergeable})
+}
+
 func buildVMConfig(rec *hypervisor.VMRecord, consoleSockPath string, allowed []int) *chVMConfig {
 	cpu := rec.Config.CPU
 	mem := rec.Config.Memory
 
 	cfg := &chVMConfig{
 		CPUs:     chCPUs{BootVCPUs: cpu, MaxVCPUs: hypervisor.HostCPUCount(), KVMHyperV: rec.Config.Windows},
-		Memory:   chMemory{Size: mem, HugePages: rec.Config.HugePages, Shared: rec.Config.SharedMemory},
+		Memory:   chMemory{Size: mem, HugePages: rec.Config.HugePages, Shared: rec.Config.SharedMemory, Mergeable: rec.Config.Mergeable},
 		RNG:      chRNG{Src: "/dev/urandom"},
 		Watchdog: true,
 		Vsock:    &chVsock{CID: hypervisor.VsockGuestCID, Socket: hypervisor.VsockSockPath(rec.RunDir)},
@@ -93,14 +98,7 @@ func buildCLIArgs(cfg *chVMConfig, socketPath string) []string {
 	cpuKV.addIf(cfg.CPUs.KVMHyperV, "kvm_hyperv=on")
 	args = append(args, "--cpus", cpuKV.String())
 
-	mem := fmt.Sprintf("size=%d", cfg.Memory.Size)
-	if cfg.Memory.HugePages {
-		mem += ",hugepages=on"
-	}
-	if cfg.Memory.Shared {
-		mem += ",shared=on"
-	}
-	args = append(args, "--memory", mem)
+	args = append(args, "--memory", memoryCLIArg(cfg.Memory))
 
 	if len(cfg.Disks) > 0 {
 		args = append(args, "--disk")
@@ -230,6 +228,15 @@ func queueAffinity(cpuCount int, allowed []int) []chQueueAffinity {
 	return qa
 }
 
+func memoryCLIArg(m chMemory) string {
+	var kv kvBuilder
+	kv.add(fmt.Sprintf("size=%d", m.Size))
+	kv.addIf(m.HugePages, "hugepages=on")
+	kv.addIf(m.Shared, "shared=on")
+	kv.addIf(m.Mergeable, "mergeable=on")
+	return kv.String()
+}
+
 func diskToCLIArg(d chDisk) string {
 	var b kvBuilder
 	b.add("path=" + d.Path)

hypervisor/cloudhypervisor/args_test.go

@@ -0,0 +1,32 @@
+package cloudhypervisor
+
+import (
+	"slices"
+	"strings"
+	"testing"
+
+	"github.com/cocoonstack/cocoon/hypervisor"
+	"github.com/cocoonstack/cocoon/types"
+)
+
+func TestMemoryCLIArg(t *testing.T) {
+	tests := []struct {
+		name string
+		cfg  types.Config
+		want string
+	}{
+		{name: "plain", cfg: types.Config{Memory: 1 << 30}, want: "size=1073741824"},
+		{name: "hugepages+shared", cfg: types.Config{Memory: 1 << 30, HugePages: true, SharedMemory: true}, want: "size=1073741824,hugepages=on,shared=on"},
+		{name: "mergeable", cfg: types.Config{Memory: 1 << 30, Mergeable: true}, want: "size=1073741824,mergeable=on"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rec := &hypervisor.VMRecord{VM: types.VM{Config: types.VMConfig{Config: tt.cfg}}}
+			args := buildCLIArgs(buildVMConfig(rec, "", nil), "api.sock")
+			i := slices.Index(args, "--memory")
+			if i < 0 || i+1 >= len(args) || args[i+1] != tt.want {
+				t.Fatalf("memory arg not %q (args: %s)", tt.want, strings.Join(args, " "))
+			}
+		})
+	}
+}

hypervisor/firecracker/create.go

@@ -30,6 +30,9 @@ func (fc *Firecracker) Create(ctx context.Context, id string, vmCfg *types.VMCon
 	if vmCfg.HugePages {
 		return nil, fmt.Errorf("firecracker does not support hugepages (restore cannot map hugetlbfs-backed snapshots)")
 	}
+	if vmCfg.Mergeable {
+		return nil, fmt.Errorf("firecracker does not support mergeable memory (no KSM madvise knob)")
+	}
 	if !hypervisor.IsDirectBoot(bootCfg) {
 		return nil, fmt.Errorf("firecracker requires direct kernel boot (OCI image)")
 	}

types/config.go

@@ -23,6 +23,8 @@ type Config struct {
 	SharedMemory bool `json:"shared_memory,omitempty"`
 	// HugePages backs CH guest memory with hugetlbfs (costs snapshots the mmap fast path); fixed at create, persists through clone/restore.
 	HugePages bool `json:"hugepages,omitempty"`
+	// Mergeable marks CH guest memory MADV_MERGEABLE for host KSM dedup (needs plain private memory); fixed at create, persists through clone/restore.
+	Mergeable bool `json:"mergeable,omitempty"`
 
 	// Raw cgroup v2 CPU knobs; zero derives the Guaranteed-at-N defaults from CPU (CPUSetCPUs empty = no placement).
 	CPUWeight   int    `json:"cpu_weight,omitempty"`