From 640d22f115ffdf9469035da60d1cc77d0443fc34 Mon Sep 17 00:00:00 2001 From: Haseeb Tariq Date: Mon, 17 Aug 2026 14:03:04 -0700 Subject: [PATCH] DO NOT MERGE: OCPBUGS-103516: Toggle madvise around compaction for perfscale Throwaway variant to benchmark an upstream suggestion on etcd-io/bbolt#939 of toggling MADV_RANDOM off only around compaction instead of removing it. Add SetMmapAdvice(random bool) to the Backend interface, forwarding to bbolt's DB.SetMmapAdvice. scheduleCompaction sets MADV_NORMAL for the whole compaction pass and restores MADV_RANDOM on exit (covering both the normal and stop-signal returns), so the pass is bracketed once rather than per batch. The bbolt replace directive points at the madvise-toggle fork (v1.4.3 + SetMmapAdvice): MADV_RANDOM steady state plus the runtime toggle. Note: the setter takes bbolt's mmaplock for reading, not writing; a write lock deadlocks against etcd's long-lived read transaction. Not intended to merge; deleted once the perfscale comparison is captured. Assisted-by: Claude Code (Opus 4.8) --- etcdutl/go.mod | 2 +- etcdutl/go.sum | 2 ++ go.mod | 2 +- go.sum | 2 ++ server/go.mod | 2 +- server/go.sum | 2 ++ server/storage/backend/backend.go | 12 ++++++++++++ server/storage/mvcc/kvstore_compaction.go | 8 ++++++++ server/storage/mvcc/kvstore_test.go | 1 + tests/go.mod | 2 +- tests/go.sum | 2 ++ 11 files changed, 33 insertions(+), 4 deletions(-) diff --git a/etcdutl/go.mod b/etcdutl/go.mod index 923018c59776..853d28ac8594 100644 --- a/etcdutl/go.mod +++ b/etcdutl/go.mod @@ -95,4 +95,4 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -replace go.etcd.io/bbolt => github.com/openshift/bbolt v0.0.0-20260806041816-148dcef86a15 +replace go.etcd.io/bbolt => github.com/hasbro17/bbolt v0.0.0-20260817210116-ce271e770d58 diff --git a/etcdutl/go.sum b/etcdutl/go.sum index 7ca5909af13a..356f8d7f538d 100644 --- a/etcdutl/go.sum +++ b/etcdutl/go.sum @@ -43,6 +43,8 @@ github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwn github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= +github.com/hasbro17/bbolt v0.0.0-20260817210116-ce271e770d58 h1:VWYfk5jGeYCNrp6atOe4lXXWc4pRKRWIA1f/8I9sPxc= +github.com/hasbro17/bbolt v0.0.0-20260817210116-ce271e770d58/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= diff --git a/go.mod b/go.mod index 41632a94d240..b95e8cf3fe2a 100644 --- a/go.mod +++ b/go.mod @@ -102,4 +102,4 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -replace go.etcd.io/bbolt => github.com/openshift/bbolt v0.0.0-20260806041816-148dcef86a15 +replace go.etcd.io/bbolt => github.com/hasbro17/bbolt v0.0.0-20260817210116-ce271e770d58 diff --git a/go.sum b/go.sum index 6a9970e3e057..4f03663cd732 100644 --- a/go.sum +++ b/go.sum @@ -73,6 +73,8 @@ github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwn github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= +github.com/hasbro17/bbolt v0.0.0-20260817210116-ce271e770d58 h1:VWYfk5jGeYCNrp6atOe4lXXWc4pRKRWIA1f/8I9sPxc= +github.com/hasbro17/bbolt v0.0.0-20260817210116-ce271e770d58/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= diff --git a/server/go.mod b/server/go.mod index 78f413add02a..cd50b26fa0c4 100644 --- a/server/go.mod +++ b/server/go.mod @@ -91,4 +91,4 @@ replace go.etcd.io/etcd => ./FORBIDDEN_DEPENDENCY replace go.etcd.io/tests/v3 => ./FORBIDDEN_DEPENDENCY -replace go.etcd.io/bbolt => github.com/openshift/bbolt v0.0.0-20260806041816-148dcef86a15 +replace go.etcd.io/bbolt => github.com/hasbro17/bbolt v0.0.0-20260817210116-ce271e770d58 diff --git a/server/go.sum b/server/go.sum index c67f71781412..aa43dffab91e 100644 --- a/server/go.sum +++ b/server/go.sum @@ -65,6 +65,8 @@ github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwn github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= +github.com/hasbro17/bbolt v0.0.0-20260817210116-ce271e770d58 h1:VWYfk5jGeYCNrp6atOe4lXXWc4pRKRWIA1f/8I9sPxc= +github.com/hasbro17/bbolt v0.0.0-20260817210116-ce271e770d58/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= diff --git a/server/storage/backend/backend.go b/server/storage/backend/backend.go index c1a8e2592b52..d67770a34db4 100644 --- a/server/storage/backend/backend.go +++ b/server/storage/backend/backend.go @@ -70,6 +70,10 @@ type Backend interface { OpenReadTxN() int64 Defrag() error ForceCommit() + // SetMmapAdvice toggles the bbolt mmap madvise hint (Linux only). random=true + // restores MADV_RANDOM (steady state); random=false sets MADV_NORMAL, intended + // to bracket a compaction pass. See go.etcd.io/bbolt DB.SetMmapAdvice. + SetMmapAdvice(random bool) error Close() error // SetTxPostLockInsideApplyHook sets a txPostLockInsideApplyHook. @@ -369,6 +373,14 @@ func (b *backend) ForceCommit() { b.batchTx.Commit() } +func (b *backend) SetMmapAdvice(random bool) error { + // RLock guards against defrag swapping b.db; bbolt serializes the actual + // mmap mutation internally via its mmaplock. + b.mu.RLock() + defer b.mu.RUnlock() + return b.db.SetMmapAdvice(random) +} + func (b *backend) Snapshot() Snapshot { b.batchTx.Commit() diff --git a/server/storage/mvcc/kvstore_compaction.go b/server/storage/mvcc/kvstore_compaction.go index a052c93943e0..34fbfeed2403 100644 --- a/server/storage/mvcc/kvstore_compaction.go +++ b/server/storage/mvcc/kvstore_compaction.go @@ -36,6 +36,14 @@ func (s *store) scheduleCompaction(compactMainRev, prevCompactRev int64) (KeyVal defer func() { dbCompactionKeysCounter.Add(float64(keyCompactions)) }() defer func() { dbCompactionLast.Set(float64(time.Now().Unix())) }() + // Benchmark (OCPBUGS-103516 / bbolt#939): bracket the whole compaction pass + // with MADV_NORMAL and restore MADV_RANDOM afterwards. Set once per pass, not + // per batch, so the hint stays NORMAL across the batches and inter-batch + // sleeps. The defer covers both the normal and stop-signal returns. Errors are + // non-fatal (madvise is best-effort; Linux-only). + _ = s.b.SetMmapAdvice(false) + defer func() { _ = s.b.SetMmapAdvice(true) }() + end := make([]byte, 8) binary.BigEndian.PutUint64(end, uint64(compactMainRev+1)) diff --git a/server/storage/mvcc/kvstore_test.go b/server/storage/mvcc/kvstore_test.go index 65ad4f240ec7..d4f94771d39e 100644 --- a/server/storage/mvcc/kvstore_test.go +++ b/server/storage/mvcc/kvstore_test.go @@ -1001,6 +1001,7 @@ func (b *fakeBackend) OpenReadTxN() int64 func (b *fakeBackend) Snapshot() backend.Snapshot { return nil } func (b *fakeBackend) ForceCommit() {} func (b *fakeBackend) Defrag() error { return nil } +func (b *fakeBackend) SetMmapAdvice(bool) error { return nil } func (b *fakeBackend) Close() error { return nil } func (b *fakeBackend) SetTxPostLockInsideApplyHook(func()) {} diff --git a/tests/go.mod b/tests/go.mod index e247ad494a4a..92bf505670ce 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -107,4 +107,4 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -replace go.etcd.io/bbolt => github.com/openshift/bbolt v0.0.0-20260806041816-148dcef86a15 +replace go.etcd.io/bbolt => github.com/hasbro17/bbolt v0.0.0-20260817210116-ce271e770d58 diff --git a/tests/go.sum b/tests/go.sum index ba8c922aa880..014b856470af 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -78,6 +78,8 @@ github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwn github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= +github.com/hasbro17/bbolt v0.0.0-20260817210116-ce271e770d58 h1:VWYfk5jGeYCNrp6atOe4lXXWc4pRKRWIA1f/8I9sPxc= +github.com/hasbro17/bbolt v0.0.0-20260817210116-ce271e770d58/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I=