Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
0cb3e73
feat: Add sliding window to lock free metrics batcher
alexander-alderman-webb Nov 18, 2026
c800bc7
merge
alexander-alderman-webb Nov 18, 2026
84db651
merge and simplify tests
alexander-alderman-webb Nov 18, 2026
21fd75e
per thread state
alexander-alderman-webb Nov 18, 2026
343bef6
simplify more
alexander-alderman-webb Nov 18, 2026
ff10f51
Merge branch 'webb/sliding-window-metrics-batcher' into webb/sliding-…
alexander-alderman-webb Nov 18, 2026
7e49836
only use class if option set
alexander-alderman-webb Nov 18, 2026
14893a6
simplify
alexander-alderman-webb Nov 18, 2026
183d1a1
simplify again
alexander-alderman-webb Nov 18, 2026
5181903
remove some tests
alexander-alderman-webb Nov 18, 2026
c626961
remove indirection and tests
alexander-alderman-webb Nov 18, 2026
33aa0d7
.
alexander-alderman-webb Nov 18, 2026
4afc3ba
simplify
alexander-alderman-webb Nov 18, 2026
3f07315
merge
alexander-alderman-webb Nov 18, 2026
4fdf682
fix typo
alexander-alderman-webb Nov 18, 2026
7ed73d5
iteration clean up
alexander-alderman-webb Nov 18, 2026
c7cc2dd
remove indirection
alexander-alderman-webb Nov 18, 2026
3119a60
fix
alexander-alderman-webb Nov 18, 2026
44d2d61
iterate on tests
alexander-alderman-webb Nov 19, 2026
eb22208
make test intersting
alexander-alderman-webb Nov 19, 2026
eaf3ab6
fix client options
alexander-alderman-webb Nov 19, 2026
3f45ee5
.
alexander-alderman-webb Nov 19, 2026
cb36693
simplify test
alexander-alderman-webb Nov 19, 2026
4ce601c
merge
alexander-alderman-webb Nov 19, 2026
ffc6b96
Merge branch 'webb/sliding-window-metrics-batcher' into webb/sliding-…
alexander-alderman-webb Nov 19, 2026
c89e37d
inline _merge()
alexander-alderman-webb Nov 19, 2026
7967d7f
use list.copy() for consistency
alexander-alderman-webb Nov 19, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
159 changes: 153 additions & 6 deletions sentry_sdk/_lock_free_metrics_batcher.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
import bisect
import itertools
import os
import random
import threading
from collections import deque
import time
from collections import defaultdict, deque
from datetime import datetime, timezone
from typing import TYPE_CHECKING

Expand All @@ -18,6 +22,41 @@
from sentry_sdk._types import Metric


class _Reservoir:
__slots__ = ("kept", "seen")

def __init__(self) -> None:
self.kept: "list[Metric]" = []
self.seen = 0


class _ThreadState:
"""The metrics collected by one thread in a given window and their count."""

__slots__ = ("reservoirs", "size")

def __init__(self) -> None:
self.reservoirs: "defaultdict[str, _Reservoir]" = defaultdict(_Reservoir)
self.size = 0


class _Window:
"""
One time window in which multiple threads concurrently capture metrics.
Only the thread with the given identifier should change its local state.
"""

__slots__ = ("start", "threads", "retire_claims")

def __init__(self) -> None:
self.start = time.monotonic()
self.threads: "defaultdict[int, _ThreadState]" = defaultdict(_ThreadState)
self.retire_claims: "list[object]" = []

def size(self) -> int:
return sum(state.size for state in self.threads.copy().values())


class LockFreeMetricsBatcher(Batcher["Metric"]):
MAX_BEFORE_FLUSH = 1000
MAX_BEFORE_DROP = 10_000
Expand All @@ -31,15 +70,28 @@ def __init__(
self,
capture_func: "Callable[[Envelope], None]",
record_lost_func: "Callable[..., None]",
window_sampling: "dict[str, float]",
) -> None:
super().__init__(capture_func, record_lost_func)

# Metric name -> metrics kept per window. Never below 1, so there's
# always at least one metric per name to extrapolate from
self._limits = {
name: min(max(1, int(rate * self.FLUSH_WAIT_TIME)), self.MAX_BEFORE_DROP)
for name, rate in window_sampling.items()
}

self._queue: "deque[Metric]" = deque()
self._window = _Window()
self._retired: "deque[_Window]" = deque()
# Holds the claim of the one thread that gets to start the flusher.
self._flusher_claim: "dict[str, object]" = {}

def _reset_thread_state(self) -> None:
super()._reset_thread_state()
self._queue = deque()
self._window = _Window()
self._retired = deque()
self._flusher_claim = {}

def _ensure_thread(self) -> bool:
Expand Down Expand Up @@ -76,16 +128,53 @@ def add(self, item: "Metric") -> None:
if not self._ensure_thread():
return None

queue = self._queue
if len(queue) >= self.MAX_BEFORE_DROP:
window = self._window
if time.monotonic() - window.start >= self.FLUSH_WAIT_TIME:
self._retire_window(window)
window = self._window

if len(self._queue) + window.size() >= self.MAX_BEFORE_DROP:
Comment thread
alexander-alderman-webb marked this conversation as resolved.
Resolved
self._record_lost(item)
return None

queue.append(item)
if len(queue) < self.MAX_BEFORE_FLUSH or self._flush_event.is_set():
limit = self._limits.get(item["name"])
if limit is None:
self._queue.append(item)
if len(self._queue) < self.MAX_BEFORE_FLUSH or self._flush_event.is_set():
return

self._flush_event.set()
return

self._add_to_window(item, window, limit)
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved

def _add_to_window(self, item: "Metric", window: "_Window", limit: int) -> None:
state = window.threads[threading.get_ident()]
reservoir = state.reservoirs[item["name"]]
reservoir.seen += 1
if len(reservoir.kept) < limit:
state.size += 1
reservoir.kept.append(item)
return

self._flush_event.set()
index = random.randrange(reservoir.seen)
if index >= limit:
return

reservoir.kept[index] = item

def _retire_window(self, window: "_Window") -> None:
claim = object()
window.retire_claims.append(claim)
if window.retire_claims[0] is not claim:
return

self._window = _Window()
self._retired.append(window)

def flush(self) -> None:
self._retire_window(self._window)
super().flush()

def _flush(self) -> "Optional[Envelope]":
queue = self._queue
Expand All @@ -97,6 +186,18 @@ def _flush(self) -> "Optional[Envelope]":
except IndexError:
break

sampled_out = 0

window = self._window
if time.monotonic() - window.start >= self.FLUSH_WAIT_TIME:
self._retire_window(window)
for _ in range(len(self._retired)):
try:
window = self._retired.popleft()
except IndexError:
break
sampled_out += self._harvest(window, items)

if not items:
return None

Expand All @@ -106,8 +207,54 @@ def _flush(self) -> "Optional[Envelope]":
with capture_internal_exceptions():
envelope.add_item(self._to_envelope_item(items))
self._capture_func(envelope)

if sampled_out:
self._record_lost_func(
reason="sample_rate",
data_category=self.CLIENT_REPORT_TYPE,
quantity=sampled_out,
)

return envelope

def _harvest(self, window: "_Window", items: "list[Metric]") -> int:
"""Uniformly gather metrics from the window and append to `items`."""
by_name: "dict[str, list[_Reservoir]]" = {}
for state in window.threads.copy().values():
for name, reservoir in state.reservoirs.copy().items():
by_name.setdefault(name, []).append(reservoir)

sampled_out = 0
for name, reservoirs in by_name.items():
limit = self._limits.get(name)
if limit is None:
continue

bounds = list(
itertools.accumulate(reservoir.seen for reservoir in reservoirs)
)
slots = [0] * len(reservoirs)
for position in random.sample(range(bounds[-1]), min(limit, bounds[-1])):
slots[bisect.bisect_right(bounds, position)] += 1

kept: "list[Metric]" = []
for reservoir, count in zip(reservoirs, slots):
thread_kept = reservoir.kept.copy()
kept.extend(random.sample(thread_kept, min(count, len(thread_kept))))
if not kept:
continue

items.extend(kept)
if bounds[-1] <= len(kept):
continue

rate = len(kept) / bounds[-1]
for item in kept:
item["attributes"]["sentry.client_sample_rate"] = rate
sampled_out += bounds[-1] - len(kept)

return sampled_out

@staticmethod
def _to_transport_format(item: "Metric") -> "Any":
res = {
Expand Down
43 changes: 39 additions & 4 deletions sentry_sdk/client.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import math
import os
import random
import socket
Expand All @@ -9,6 +10,7 @@
from typing import TYPE_CHECKING, Dict, cast, overload

from sentry_sdk._compat import check_uwsgi_thread_support
from sentry_sdk._lock_free_metrics_batcher import LockFreeMetricsBatcher
from sentry_sdk._log_batcher import LogBatcher
from sentry_sdk._metrics_batcher import MetricsBatcher
from sentry_sdk._span_batcher import SpanBatcher
Expand Down Expand Up @@ -128,6 +130,29 @@ def _get_options(*args: "Optional[str]", **kwargs: "Any") -> "Dict[str, Any]":

rv["data_collection"] = _resolve_data_collection(rv)

window_sampling = rv["_experiments"].get("metrics_window_sampling")
if window_sampling is not None:
valid_rates = {}
if isinstance(window_sampling, dict):
for name, rate in window_sampling.items():
if isinstance(rate, (int, float)) and math.isfinite(rate) and rate > 0:
valid_rates[name] = rate
else:
logger.warning(
"[Metrics] Given window sampling rate for %s is invalid. It must be a positive number. Got %s, not sampling it.",
name,
rate,
)
else:
logger.warning(
"[Metrics] metrics_window_sampling must map metric names to rates. Got %s, not sampling.",
window_sampling,
)
rv["_experiments"] = {
**rv["_experiments"],
"metrics_window_sampling": valid_rates,
}

if rv["socket_options"] and not isinstance(rv["socket_options"], list):
logger.warning(
"Ignoring socket_options because of unexpected format. See urllib3.HTTPConnection.socket_options for the expected format."
Expand Down Expand Up @@ -160,7 +185,7 @@ def __init__(self, options: "Optional[Dict[str, Any]]" = None) -> None:
self.transport: "Optional[Transport]" = None
self.monitor: "Optional[Monitor]" = None
self.log_batcher: "Optional[LogBatcher]" = None
self.metrics_batcher: "Optional[MetricsBatcher]" = None
self.metrics_batcher: "Optional[Union[MetricsBatcher, LockFreeMetricsBatcher]]" = None
self.span_batcher: "Optional[SpanBatcher]" = None
self.integrations: "dict[str, Integration]" = {}

Expand Down Expand Up @@ -379,10 +404,20 @@ def _record_lost_event(
record_lost_func=_record_lost_event,
)

self.metrics_batcher = MetricsBatcher(
capture_func=_capture_envelope,
record_lost_func=_record_lost_event,
window_sampling = self.options["_experiments"].get(
"metrics_window_sampling"
)
if window_sampling:
self.metrics_batcher = LockFreeMetricsBatcher(
capture_func=_capture_envelope,
record_lost_func=_record_lost_event,
window_sampling=window_sampling,
)
else:
self.metrics_batcher = MetricsBatcher(
capture_func=_capture_envelope,
record_lost_func=_record_lost_event,
)

self.span_batcher = SpanBatcher(
capture_func=_capture_envelope,
Expand Down
3 changes: 3 additions & 0 deletions sentry_sdk/consts.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,9 @@ class CompressionAlgo(Enum):
"transport_num_pools": Optional[int],
"transport_http2": Optional[bool],
"transport_async": Optional[bool],
# Metric name -> metrics per second to keep. Names not in here
# aren't sampled
"metrics_window_sampling": Optional[Dict[str, float]],
},
total=False,
)
Expand Down
Loading
Loading