feat(nativemem): measure allocator overhead instead of assuming a blanket factor

ddprof-lib/src/main/cpp/codeCache.cpp

@@ -73,6 +73,7 @@ CodeCache::CodeCache(const char *name, short lib_index,
 
   _memory_usage = (long long)_capacity * sizeof(CodeBlob) +
                   (long long)NativeFunc::allocSize(_name);
+  _name_overhead = (long long)NativeFunc::nameOverhead(_name);
 
   _published.store(false, std::memory_order_relaxed);
 }
@@ -132,6 +133,10 @@ void CodeCache::copyFrom(const CodeCache& other) {
   // no need to recompute (and `other` may itself already be published, whose
   // _blobs this copy must not depend on after construction).
   _memory_usage = other._memory_usage;
+  // Likewise for the overhead: the name strings above are fresh allocations,
+  // but of exactly the same sizes, and overhead is a function of the requested
+  // size, so `other`'s total describes this copy just as well.
+  _name_overhead = other._name_overhead;
 
   // A copy is a fresh, not-yet-registered cache.
   _published.store(false, std::memory_order_relaxed);
@@ -209,6 +214,9 @@ void CodeCache::add(const void *start, int length, const char *name,
          "add() on a published CodeCache races memoryUsage()");
   char *name_copy = NativeFunc::create(name, _lib_index);
   _memory_usage += (long long)NativeFunc::allocSize(name);
+  // Measured off the block just handed back, while the pointer is in hand: the
+  // gauge is published as a total at add() time, with no pointer left to probe.
+  _name_overhead += (long long)NativeFunc::nameOverhead(name_copy);
   // Replace non-printable characters
   for (char *s = name_copy; *s != 0; s++) {
     if (*s < ' ')

ddprof-lib/src/main/cpp/codeCache.h

@@ -10,6 +10,7 @@
 #include "common.h"
 #include "counters.h"
 #include "dwarf.h"
+#include "mallocFootprint.h"
 #include "nativeMem.h"
 #include "utils.h"
 
@@ -86,6 +87,18 @@ class NativeFunc {
     return align_up(sizeof(NativeFunc) + 1 + strlen(name), sizeof(NativeFunc *));
   }
 
+  // Allocator overhead on that allocation -- rounding to the size quantum plus
+  // the per-chunk header, measured rather than assumed. Lives here because only
+  // NativeFunc knows the real allocation base: `name` points *into* the block at
+  // offset sizeof(NativeFunc), so querying the allocator with `name` itself
+  // would be undefined. 0 if null.
+  static size_t nameOverhead(const char *name) {
+    if (name == nullptr) {
+      return 0;
+    }
+    return MallocFootprint::overheadOf(from(name), allocSize(name));
+  }
+
   static short libIndex(const char *name) {
     if (name == nullptr) {
       return -1;
@@ -173,6 +186,20 @@ class CodeCache {
   // total could be stale relative to a fresh recompute.
   long long _memory_usage;
 
+  // Measured allocator overhead -- rounding to the size quantum plus the
+  // per-chunk header -- on the name allocations counted in _memory_usage.
+  // Tracked incrementally alongside it, by the same mutators, because overhead
+  // is a function of *per-allocation* size and so cannot be recovered later
+  // from a byte total: a 512 KB chunk pays ~0.003 %, a 96-byte node 16.7 %.
+  //
+  // The _blobs array is deliberately excluded. It is one allocation per
+  // library, large enough that its overhead is a rounding error, and it comes
+  // from new CodeBlob[] -- whose returned pointer is not guaranteed to be the
+  // allocator's block base, so querying the allocator with it would be unsound.
+  // Excluding it understates by a negligible amount rather than risking a wrong
+  // reading.
+  long long _name_overhead;
+
   // Set once the cache is registered into a CodeCacheArray (see markPublished()).
   // After that, memoryUsage() may be read lock-free from another thread (dump),
   // so the mutators that touch the fields it reads (add()/expand()/
@@ -290,13 +317,17 @@ class CodeCache {
 
   // Live size of what this CodeCache owns on the heap: the blob array, the
   // per-symbol name strings (variable length), this cache's own name, and the
-  // DWARF unwind table. Recomputed on demand (called only at dump time), so it
-  // reflects the current contents. The build-id string is deliberately excluded
-  // — it is mutated by the background refresher and negligible in size (see the
-  // definition). Const and lock-free: reads only fields that are stable once the
-  // library is published.
+  // DWARF unwind table. An O(1) read of a running total (see _memory_usage).
+  // The build-id string is deliberately excluded — it is mutated by the
+  // background refresher and negligible in size (see the definition). Const and
+  // lock-free: reads only fields that are stable once the library is published.
   long long memoryUsage() const;
 
+  // Measured allocator overhead on the name allocations that memoryUsage()
+  // counts (see _name_overhead). Same O(1) running-total read, same lock-free
+  // guarantees.
+  long long nameOverhead() const { return _name_overhead; }
+
   int count() { return _count; }
   CodeBlob* blob(int idx) {
     return &_blobs[idx];
@@ -348,6 +379,10 @@ class CodeCacheArray {
     // aggregate) can never clobber each other's contribution -- unlike the
     // read-modify-write a periodic setLive(sum-of-everything) would need.
     NativeMem::record(NM_NATIVE_SYMBOLS, lib->memoryUsage());
+    // The allocator overhead on those same allocations, measured rather than
+    // assumed, recorded at the same instant and by the same relaxed-atomic add
+    // so it stays consistent with the logical bytes above.
+    NativeMem::recordOverhead(NM_NATIVE_SYMBOLS, lib->nameOverhead());
     // Mark published before the RELEASE store makes the pointer visible, so any
     // later add()/expand()/setDwarfTable() on this cache trips the assert (its
     // _blobs would then be read lock-free by memoryUsage() at dump time).

ddprof-lib/src/main/cpp/counters.h

@@ -56,6 +56,20 @@
   X(NATIVE_MEM_LIVE_BYTES, "native_mem_live_bytes")                            \
   X(NATIVE_MEM_MAX_BYTES, "native_mem_max_bytes")                              \
   X(NATIVE_MEM_AVG_BYTES, "native_mem_avg_bytes")                              \
+  /* Process-wide malloc arena state, from glibc's own accounting. These are   \
+   * NOT profiler memory and deliberately are not NM_* categories: those must  \
+   * partition the profiler's own allocations (see nativeMem.h), and arena     \
+   * slack is mostly other subsystems' chunks stranded by interleaving, so it  \
+   * is not attributable to any profiler allocation. Reported so that the      \
+   * allocator's own overhead is visible rather than folded into a "profiler   \
+   * cost" figure -- it differs substantially between glibc, tcmalloc and      \
+   * jemalloc. Sampled on the JFR flush path only: mallinfo2() walks every     \
+   * arena taking locks, so it is neither cheap nor async-signal-safe. */      \
+  X(MALLOC_ARENA_BYTES, "malloc_arena_bytes")                                  \
+  X(MALLOC_IN_USE_BYTES, "malloc_in_use_bytes")                                \
+  X(MALLOC_FREE_HELD_BYTES, "malloc_free_held_bytes")                          \
+  X(MALLOC_TRIMMABLE_BYTES, "malloc_trimmable_bytes")                          \
+  X(MALLOC_MMAP_BYTES, "malloc_mmap_bytes")                                    \
   X(THREAD_IDS_COUNT, "thread_ids_count")                                      \
   X(THREAD_NAMES_COUNT, "thread_names_count")                                  \
   X(THREAD_FILTER_PAGES, "thread_filter_pages")                                \

ddprof-lib/src/main/cpp/countingAllocator.h

@@ -25,12 +25,18 @@ class CountingAllocator {
 
   T *allocate(std::size_t n) {
     T *p = static_cast<T *>(::operator new(n * sizeof(T)));
-    NativeMem::record(Cat, (long long)(n * sizeof(T)));
+    // recordAlloc rather than record: STL nodes are small, so the allocator's
+    // rounding and per-chunk header are a large fraction of their real cost
+    // (a 96-byte MethodMap node occupies 112 bytes -- 16.7 %). Logical bytes
+    // still land in the live gauge; the extra goes to the overhead gauge.
+    NativeMem::recordAlloc(Cat, p, n * sizeof(T));
     return p;
   }
 
   void deallocate(T *p, std::size_t n) noexcept {
-    NativeMem::record(Cat, -(long long)(n * sizeof(T)));
+    // Must run before operator delete: the overhead is read back off the live
+    // chunk.
+    NativeMem::recordFreeBefore(Cat, p, n * sizeof(T));
     ::operator delete(p);
   }
 

ddprof-lib/src/main/cpp/flightRecorder.cpp

@@ -7,6 +7,15 @@
 #include <assert.h>
 #include <inttypes.h>
 
+// mallinfo2() replaced the int-based mallinfo() in glibc 2.33; the older struct
+// silently truncates past 2 GiB, so it is not a usable fallback for byte
+// accounting. Absent on musl and macOS, where the arena counters stay zero.
+#if defined(__linux__) && defined(__GLIBC__) &&                                \
+    (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 33))
+  #include <malloc.h>
+  #define DD_HAVE_MALLINFO2 1
+#endif
+
 #include "buffers.h"
 #include "callTraceHashTable.h"
 #include "context.h"
@@ -2024,6 +2033,36 @@ void Recording::writeLogLevels(Buffer *buf) {
   }
 }
 
+// Snapshot the process-wide malloc arena state from glibc's own accounting.
+//
+// This is NOT profiler memory. Free-but-held arena pages are mostly other
+// subsystems' chunks stranded by interleaving, so they cannot be attributed to
+// any profiler allocation -- reporting them separately keeps them out of the
+// per-category figures while still making the allocator's overhead visible.
+// The numbers also differ substantially between glibc, tcmalloc and jemalloc,
+// which is exactly what a reader comparing allocators needs to see.
+//
+// Only safe on the flush path: mallinfo2() walks every arena taking each arena
+// lock, so it is neither cheap nor async-signal-safe. Must never be reached
+// from the sampling signal handler. Once per JFR chunk is negligible.
+void Recording::updateMallocArenaStats() {
+#ifdef DD_HAVE_MALLINFO2
+  struct mallinfo2 mi = mallinfo2();
+  // arena:    bytes obtained from the OS via brk, excluding mmap'd chunks
+  // uordblks: bytes currently handed out to callers
+  // fordblks: bytes free but retained in the arenas -- the waste term
+  // keepcost: the trimmable top block, i.e. what malloc_trim could return;
+  //           separating it distinguishes trim-threshold policy from genuine
+  //           fragmentation, which trimming cannot reclaim
+  // hblkhd:   bytes in mmap'd chunks, which are returned to the OS on free
+  Counters::set(MALLOC_ARENA_BYTES, (long long)mi.arena);
+  Counters::set(MALLOC_IN_USE_BYTES, (long long)mi.uordblks);
+  Counters::set(MALLOC_FREE_HELD_BYTES, (long long)mi.fordblks);
+  Counters::set(MALLOC_TRIMMABLE_BYTES, (long long)mi.keepcost);
+  Counters::set(MALLOC_MMAP_BYTES, (long long)mi.hblkhd);
+#endif
+}
+
 void Recording::capturePostFlushNativeMem() {
   for (int c = 0; c < NM_NUM_CATEGORIES; c++) {
     NativeMemCategory cat = (NativeMemCategory)c;
@@ -2050,6 +2089,10 @@ void Recording::updateNativeMemStats() {
   // here; the total peak is bracketed instead (see writeNativeMem).
   NativeMem::sample();
 
+  // Process-wide allocator state, sampled at the same instant as the
+  // per-category figures so the two can be compared coherently.
+  updateMallocArenaStats();
+
   // Mirror the totals into the flat counter table so they flow out through the
   // existing counter path (JFR T_DATADOG_COUNTER events and the JNI debug
   // counters). NATIVE_MEM_MAX_BYTES carries the upper bound on the total peak
@@ -2093,6 +2136,14 @@ void Recording::writeNativeMem(Buffer *buf) {
         {"native_mem_live_bytes.", NativeMem::live(cat)},
         {"native_mem_avg_bytes.", NativeMem::avg(cat)},
         {"native_mem_max_bytes.", NativeMem::max(cat)},
+        // Measured allocator overhead on the live allocations -- rounding to
+        // the size quantum plus the per-chunk header. Reported separately so
+        // native_mem_live_bytes stays comparable to sizeof() arithmetic, and so
+        // that reconciliation against RSS can add a measured figure instead of
+        // multiplying by a factor derived from some other workload's
+        // allocation-size mix. Zero for categories whose call sites still use
+        // record() rather than recordAlloc().
+        {"native_mem_chunk_overhead_bytes.", NativeMem::overhead(cat)},
     };
     for (const auto &m : metrics) {
       char label[64];

ddprof-lib/src/main/cpp/flightRecorder.h

@@ -365,6 +365,9 @@ class Recording {
   void writeCounters(Buffer *buf);
 
   void updateNativeMemStats();
+  // Process-wide malloc arena state (glibc mallinfo2). Flush path only --
+  // takes every arena lock, so not async-signal-safe. No-op off glibc 2.33+.
+  void updateMallocArenaStats();
   void writeNativeMem(Buffer *buf);
 
   void writeUnwindFailures(Buffer *buf);

ddprof-lib/src/main/cpp/mallocFootprint.h

@@ -0,0 +1,113 @@
+/*
+ * Copyright 2026, Datadog, Inc.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#ifndef _MALLOCFOOTPRINT_H
+#define _MALLOCFOOTPRINT_H
+
+#include <cstddef>
+
+#if defined(__linux__)
+  #include <malloc.h>
+  #define DD_HAVE_MALLOC_USABLE_SIZE 1
+#elif defined(__APPLE__)
+  #include <malloc/malloc.h>
+  #define DD_HAVE_MALLOC_SIZE 1
+#endif
+
+// Real resident cost of a heap allocation, as opposed to the size that was
+// requested.
+//
+// The profiler's NM_* gauges record requested (logical) bytes, which is what
+// makes them comparable against sizeof() arithmetic. RSS, however, is paid in
+// allocator chunks: the request is rounded up to an alignment quantum and
+// carries a per-chunk header. Reconciliation previously multiplied logical bytes
+// by a single blanket factor measured on one workload, which is wrong whenever
+// the allocation-size mix differs -- overhead is a function of *per-allocation*
+// size, not of total bytes. A 512 KB chunk pays ~0.003 %; a 96-byte tree node
+// pays 16.7 %.
+//
+// This measures it instead: usable size comes from the allocator, and the
+// per-chunk header is probed once at first use rather than assumed.
+class MallocFootprint {
+private:
+  // Determined empirically: allocate several same-size blocks and take the
+  // smallest positive address stride between any two. That stride is
+  // usable + header, so header = stride - usable.
+  //
+  // Probed rather than hardcoded because the allocator in force at runtime is
+  // not knowable at compile time -- an LD_PRELOAD'd tcmalloc or jemalloc leaves
+  // __GLIBC__ defined while adding no per-object header at all (their metadata
+  // is out of band). Assuming glibc's 8 bytes would then invent overhead that
+  // does not exist, at one allocation's worth per allocation.
+  static size_t probeHeaderBytes() {
+#ifdef DD_HAVE_MALLOC_USABLE_SIZE
+    const int N = 16;
+    const size_t SZ = 48;
+    void *p[N];
+    for (int i = 0; i < N; i++) {
+      p[i] = malloc(SZ);
+      if (p[i] == NULL) {  // give up cleanly rather than guess
+        for (int j = 0; j < i; j++) free(p[j]);
+        return 0;
+      }
+    }
+    size_t usable = malloc_usable_size(p[0]);
+    long best = 0;
+    for (int i = 0; i < N; i++) {
+      for (int j = 0; j < N; j++) {
+        long d = (char *)p[j] - (char *)p[i];
+        if (d > 0 && (best == 0 || d < best)) {
+          best = d;
+        }
+      }
+    }
+    for (int i = 0; i < N; i++) free(p[i]);
+    long header = best - (long)usable;
+    // Sanity-bound the result. A negative or implausibly large value means the
+    // blocks were not laid out contiguously (a size-class allocator, or an
+    // arena boundary landed mid-probe), in which case 0 is the honest answer:
+    // report only the rounding we can see, and understate rather than invent.
+    if (header < 0 || header > 64) {
+      return 0;
+    }
+    return (size_t)header;
+#else
+    return 0;
+#endif
+  }
+
+public:
+  // Per-chunk header size for the allocator actually in force. Probed once;
+  // the C++11 function-local static makes initialisation thread-safe. NOT
+  // async-signal-safe (it allocates), so first use must not be from a signal
+  // handler -- every current call site is on a normal thread.
+  static size_t headerBytes() {
+    static const size_t header = probeHeaderBytes();
+    return header;
+  }
+
+  // Bytes this allocation actually costs: allocator-reported usable size plus
+  // the per-chunk header. Page rounding for large mmap'd chunks is already
+  // inside the usable size, so it must not be added again.
+  static size_t of(void *p, size_t requested) {
+    if (p == NULL) {
+      return 0;
+    }
+#ifdef DD_HAVE_MALLOC_USABLE_SIZE
+    return malloc_usable_size(p) + headerBytes();
+#elif defined(DD_HAVE_MALLOC_SIZE)
+    return malloc_size(p) + headerBytes();
+#else
+    return requested;  // no introspection available: report no overhead
+#endif
+  }
+
+  // Overhead alone -- the part RSS pays for that the logical counters miss.
+  static size_t overheadOf(void *p, size_t requested) {
+    size_t total = of(p, requested);
+    return total > requested ? total - requested : 0;
+  }
+};
+
+#endif // _MALLOCFOOTPRINT_H

ddprof-lib/src/main/cpp/nativeMem.cpp

@@ -5,6 +5,7 @@
 #include "nativeMem.h"
 
 volatile long long NativeMem::_live[NM_NUM_CATEGORIES] = {};
+volatile long long NativeMem::_overhead[NM_NUM_CATEGORIES] = {};
 volatile long long NativeMem::_max[NM_NUM_CATEGORIES] = {};
 long long NativeMem::_window[NM_NUM_CATEGORIES][NativeMem::WINDOW] = {};
 long long NativeMem::_total_window[NativeMem::WINDOW] = {};
@@ -14,6 +15,17 @@ long long NativeMem::_avg[NM_NUM_CATEGORIES] = {};
 long long NativeMem::_total_avg = 0;
 long long NativeMem::_total_max_observed = 0;
 
+long long NativeMem::overheadTotal() {
+  long long total = 0;
+  for (int c = 0; c < NM_NUM_CATEGORIES; c++) {
+    long long v = load(_overhead[c]);
+    if (v > 0) {
+      total += v;
+    }
+  }
+  return total;
+}
+
 long long NativeMem::liveTotal() {
   long long total = 0;
   for (int c = 0; c < NM_NUM_CATEGORIES; c++) {
@@ -84,6 +96,7 @@ void NativeMem::reset() {
   for (int c = 0; c < NM_NUM_CATEGORIES; c++) {
     store(_live[c], (long long)0);
     store(_max[c], (long long)0);
+    store(_overhead[c], (long long)0);
     _avg[c] = 0;
     for (int i = 0; i < WINDOW; i++) {
       _window[c][i] = 0;