From 0cb3e73a95972c15a48b79de8a273b2693cb82cb Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 18 Nov 2026 13:23:32 +0100 Subject: [PATCH 01/20] feat: Add sliding window to lock free metrics batcher --- sentry_sdk/_lock_free_metrics_batcher.py | 275 ++++++++++++++++++-- sentry_sdk/_types.py | 6 + sentry_sdk/client.py | 19 +- sentry_sdk/consts.py | 4 + tests/test_metrics.py | 312 ++++++++++++++++++++--- 5 files changed, 555 insertions(+), 61 deletions(-) diff --git a/sentry_sdk/_lock_free_metrics_batcher.py b/sentry_sdk/_lock_free_metrics_batcher.py index d343ebfdc8..785828fcf4 100644 --- a/sentry_sdk/_lock_free_metrics_batcher.py +++ b/sentry_sdk/_lock_free_metrics_batcher.py @@ -1,21 +1,40 @@ """ -Experimental lock-free metrics batcher. +Experimental lock-free metrics batcher with window sampling, enabled with the +``metrics_window_sampling`` experiment. Producers (``add``) and consumers (the flusher thread and explicit ``flush``) never take a lock. Shared state is only changed through operations that are atomic on builtin containers, both with the GIL and on free-threaded CPython: - ``deque.append`` / ``deque.popleft`` -- ``dict.setdefault`` +- ``list.append`` and storing to an existing list index +- ``dict.get`` / ``dict.setdefault`` / ``dict.copy`` +- ``next()`` on an ``itertools.count`` - rebinding an attribute -This is independent of ``Batcher`` on purpose, so it can evolve (e.g. gain -sampling) without affecting the other batchers. +Anything that needs several of these to happen together (starting the flusher +thread, retiring a sampling window) is decided by a "first one wins" claim +instead of a lock. + +Window sampling keeps at most ``limit`` metrics per metric name per window with +reservoir sampling (Algorithm R), where every step is one of the atomic +operations above. A window is retired by rebinding ``self._window`` to a new +one. A thread may still hold a reference to the old window and be about to +write to it, so retired windows are only harvested after a grace period, giving +those writes time to land (the same idea as RCU). An explicit ``flush`` +harvests immediately, since it's used at shutdown and by callers who want +everything sent now. + +This is independent of ``Batcher`` on purpose, so it can evolve without +affecting the other batchers. """ +import itertools +import math import os import random import threading +import time import weakref from collections import deque from datetime import datetime, timezone @@ -25,13 +44,45 @@ from sentry_sdk.utils import ( capture_internal_exceptions, format_timestamp, + logger, serialize_attribute, ) if TYPE_CHECKING: - from typing import Any, Callable, Deque, Dict, List, Optional + from typing import Any, Callable, Deque, Dict, List, Optional, Tuple, Union - from sentry_sdk._types import Metric + from sentry_sdk._types import Metric, MetricWindowSamplingOptions + +DEFAULT_SAMPLE_PER_SECOND = 2 +# Matches FLUSH_WAIT_TIME, so each window goes out in one flush +WINDOW_SECONDS = 5.0 +# How long a retired window is left alone before it's harvested, so that adds +# which picked it up right before it was retired can finish writing to it +RETIRE_GRACE_SECONDS = 1.0 + + +class _Reservoir: + __slots__ = ("kept", "seen", "overflowed") + + def __init__(self) -> None: + self.kept: "List[Metric]" = [] + self.seen = itertools.count() + # Metrics that were seen but couldn't get a slot because the window ran + # out of memory budget. They're reported as queue_overflow, so they must + # not also count towards the sample rate + self.overflowed = itertools.count() + + +class _Window: + __slots__ = ("start", "reservoirs", "held", "retired_at", "retire_claims") + + def __init__(self) -> None: + self.start = time.monotonic() + self.reservoirs: "Dict[str, _Reservoir]" = {} + # Slots handed out across all reservoirs, to bound memory + self.held = itertools.count() + self.retired_at = math.inf + self.retire_claims: "List[object]" = [] class LockFreeMetricsBatcher: @@ -47,9 +98,26 @@ def __init__( self, capture_func: "Callable[[Envelope], None]", record_lost_func: "Callable[..., None]", + window_sampling: "Union[bool, MetricWindowSamplingOptions, None]" = None, ) -> None: self._capture_func = capture_func self._record_lost_func = record_lost_func + + self._default_limit: "Optional[int]" = None + self._name_limits: "Dict[str, int]" = {} + if window_sampling: + options = window_sampling if isinstance(window_sampling, dict) else {} + per_name = options.get("per_name") + if isinstance(per_name, dict): + self._name_limits = { + name: self._parse_limit(rate) for name, rate in per_name.items() + } + if "sample_per_second" in options or not self._name_limits: + self._default_limit = self._parse_limit( + options.get("sample_per_second", DEFAULT_SAMPLE_PER_SECOND) + ) + self._sampling = bool(self._name_limits) or self._default_limit is not None + self._reset_thread_state() # See https://github.com/getsentry/sentry-python/blob/051cc01640a29bfd64b1f1e2e3414c02f027dd1b/sentry_sdk/monitor.py#L41-L50 @@ -65,6 +133,8 @@ def _reset_in_child() -> None: def _reset_thread_state(self) -> None: self._queue: "Deque[Metric]" = deque() + self._window = _Window() + self._retired: "Deque[_Window]" = deque() self._running = True self._active: "threading.local" = threading.local() self._flush_event: "threading.Event" = threading.Event() @@ -74,6 +144,24 @@ def _reset_thread_state(self) -> None: # process self._flusher_claims: "Dict[int, object]" = {} + def _parse_limit(self, value: float) -> int: + try: + rate = float(value) + except (TypeError, ValueError): + rate = math.nan + + if not (math.isfinite(rate) and rate > 0): + logger.warning( + "[Metrics] Given window sampling rate is invalid. It must be a positive number. Got %r, using %s instead.", + value, + DEFAULT_SAMPLE_PER_SECOND, + ) + rate = DEFAULT_SAMPLE_PER_SECOND + + # Never go below 1, so we always keep at least one metric per name to + # extrapolate from + return min(max(1, int(rate * WINDOW_SECONDS)), self.MAX_BEFORE_DROP) + def _ensure_thread(self) -> bool: """For forking processes we might need to restart this thread. This ensures that our process actually has that thread running. @@ -134,19 +222,67 @@ def add(self, item: "Metric") -> None: if not self._ensure_thread(): return None - queue = self._queue - # Checking and appending isn't atomic, so the queue can go over the - # limit by at most one metric per concurrently adding thread - if len(queue) >= self.MAX_BEFORE_DROP: + limit = ( + self._name_limits.get(item["name"], self._default_limit) + if self._sampling + else None + ) + if limit is None: + kept = self._add_to_queue(item) + else: + kept = self._add_to_window(item, limit) + + # Record the loss outside of the hot path, since this serializes + # the metric to report lost bytes + if not kept: self._record_lost(item) - return None - - queue.append(item) - if len(queue) >= self.MAX_BEFORE_FLUSH and not self._flush_event.is_set(): - self._flush_event.set() finally: self._active.flag = False + def _add_to_queue(self, item: "Metric") -> bool: + queue = self._queue + # Checking and appending isn't atomic, so the queue can go over the + # limit by at most one metric per concurrently adding thread + if len(queue) >= self.MAX_BEFORE_DROP: + return False + + queue.append(item) + if len(queue) >= self.MAX_BEFORE_FLUSH and not self._flush_event.is_set(): + self._flush_event.set() + return True + + def _add_to_window(self, item: "Metric", limit: int) -> bool: + now = time.monotonic() + if now - self._window.start >= WINDOW_SECONDS: + self._retire_window(now, force=False) + + window = self._window + name = item["name"] + reservoir = window.reservoirs.get(name) + if reservoir is None: + reservoir = window.reservoirs.setdefault(name, _Reservoir()) + + seen = next(reservoir.seen) + if seen < limit: + # Replacing a slot in a full reservoir doesn't need more memory, only + # a new slot does + if next(window.held) >= self.MAX_BEFORE_DROP: + next(reservoir.overflowed) + return False + reservoir.kept.append(item) + return True + + index = random.randrange(seen + 1) + if index < limit: + try: + reservoir.kept[index] = item + except IndexError: + # The slot hasn't been filled yet by a concurrent add, or never + # will be because that add overflowed. Either way this metric is + # sampled out, which keeps the reservoir within its memory budget + pass + return True + def kill(self) -> None: if self._flusher is None: return @@ -159,11 +295,75 @@ def flush(self) -> None: was_active = getattr(self._active, "flag", False) self._active.flag = True try: - self._flush() + self._flush(force=True) finally: self._active.flag = was_active - def _flush(self) -> "Optional[Envelope]": + def _retire_window(self, now: float, force: bool) -> None: + window = self._window + if not force and now - window.start < WINDOW_SECONDS: + # Early flushes leave the window open, so a name can't go over its + # rate + return + + claim = object() + window.retire_claims.append(claim) + if window.retire_claims[0] is not claim: + # Someone else is retiring this window + return + + self._window = _Window() + window.retired_at = now + self._retired.append(window) + + def _take_retired_windows(self, now: float, force: bool) -> "List[_Window]": + ready = [] + waiting = [] + while True: + try: + window = self._retired.popleft() + except IndexError: + break + if force or now - window.retired_at >= RETIRE_GRACE_SECONDS: + ready.append(window) + else: + waiting.append(window) + + # Put back the windows that are still in their grace period, oldest first + self._retired.extendleft(reversed(waiting)) + return ready + + def _harvest( + self, window: "_Window", items: "List[Metric]" + ) -> "List[Tuple[Metric, int]]": + sampled_out = [] + for reservoir in window.reservoirs.copy().values(): + kept = reservoir.kept + if not kept: + continue + + items.extend(kept) + + # The counters are dead once a window is harvested, so reading them + # by advancing them is fine + seen = next(reservoir.seen) - next(reservoir.overflowed) + if seen <= len(kept): + # Nothing was sampled out + continue + + rate = len(kept) / seen + for item in kept: + attributes = item["attributes"] + client_rate = attributes.get("sentry.client_sample_rate") + if isinstance(client_rate, (int, float)) and 0 < client_rate <= 1: + attributes["sentry.client_sample_rate"] = rate * client_rate + else: + attributes["sentry.client_sample_rate"] = rate + sampled_out.append((kept[0], seen - len(kept))) + + return sampled_out + + def _flush(self, force: bool = False) -> "Optional[Envelope]": items: "List[Metric]" = [] queue = self._queue for _ in range(len(queue)): @@ -173,14 +373,41 @@ def _flush(self) -> "Optional[Envelope]": # Drained concurrently by another flush break - if not items: - return None + sampled_out: "List[Tuple[Metric, int]]" = [] + if self._sampling: + now = time.monotonic() + # Take the already retired windows before retiring the current one, + # so a window always gets its grace period unless forced + ready = self._take_retired_windows(now, force) + self._retire_window(now, force) + if force: + ready.extend(self._take_retired_windows(now, force)) + for window in ready: + sampled_out.extend(self._harvest(window, items)) + + envelope = None + if items: + envelope = Envelope( + headers={"sent_at": format_timestamp(datetime.now(timezone.utc))} + ) + envelope.add_item(self._to_envelope_item(items)) + self._capture_func(envelope) + + for item, count in sampled_out: + self._record_lost_func( + reason="sample_rate", + data_category=self.CLIENT_REPORT_TYPE, + quantity=count, + ) + # Serializing every sampled out metric would cost as much as we saved + # by sampling, so we estimate from one + with capture_internal_exceptions(): + self._record_lost_func( + reason="sample_rate", + data_category="trace_metric_byte", + quantity=count * len(self._to_envelope_item([item]).get_bytes()), + ) - envelope = Envelope( - headers={"sent_at": format_timestamp(datetime.now(timezone.utc))} - ) - envelope.add_item(self._to_envelope_item(items)) - self._capture_func(envelope) return envelope def _record_lost(self, item: "Metric") -> None: diff --git a/sentry_sdk/_types.py b/sentry_sdk/_types.py index 40efa2d228..d77bee2b24 100644 --- a/sentry_sdk/_types.py +++ b/sentry_sdk/_types.py @@ -359,6 +359,12 @@ class DataCollection(TypedDict): }, ) + MetricWindowSamplingOptions = TypedDict( + "MetricWindowSamplingOptions", + {"sample_per_second": float, "per_name": Dict[str, float]}, + total=False, + ) + SpanJSON = TypedDict( "SpanJSON", { diff --git a/sentry_sdk/client.py b/sentry_sdk/client.py index 7f84cea140..507d97c3e3 100644 --- a/sentry_sdk/client.py +++ b/sentry_sdk/client.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Dict, cast, overload from sentry_sdk._compat import check_uwsgi_thread_support +from sentry_sdk._lock_free_metrics_batcher import LockFreeMetricsBatcher from sentry_sdk._log_batcher import LogBatcher from sentry_sdk._metrics_batcher import MetricsBatcher from sentry_sdk._span_batcher import SpanBatcher @@ -160,7 +161,7 @@ def __init__(self, options: "Optional[Dict[str, Any]]" = None) -> None: self.transport: "Optional[Transport]" = None self.monitor: "Optional[Monitor]" = None self.log_batcher: "Optional[LogBatcher]" = None - self.metrics_batcher: "Optional[MetricsBatcher]" = None + self.metrics_batcher: "Optional[Union[MetricsBatcher, LockFreeMetricsBatcher]]" = None self.span_batcher: "Optional[SpanBatcher]" = None self.integrations: "dict[str, Integration]" = {} @@ -379,10 +380,20 @@ def _record_lost_event( record_lost_func=_record_lost_event, ) - self.metrics_batcher = MetricsBatcher( - capture_func=_capture_envelope, - record_lost_func=_record_lost_event, + window_sampling = self.options["_experiments"].get( + "metrics_window_sampling" ) + if window_sampling: + self.metrics_batcher = LockFreeMetricsBatcher( + capture_func=_capture_envelope, + record_lost_func=_record_lost_event, + window_sampling=window_sampling, + ) + else: + self.metrics_batcher = MetricsBatcher( + capture_func=_capture_envelope, + record_lost_func=_record_lost_event, + ) self.span_batcher = SpanBatcher( capture_func=_capture_envelope, diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index 953b693744..83ff4d161d 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -54,6 +54,7 @@ class CompressionAlgo(Enum): IgnoreSpansConfig, Log, Metric, + MetricWindowSamplingOptions, SpanJSON, TracesSampler, ) @@ -73,6 +74,9 @@ class CompressionAlgo(Enum): "transport_num_pools": Optional[int], "transport_http2": Optional[bool], "transport_async": Optional[bool], + "metrics_window_sampling": Optional[ + Union[bool, MetricWindowSamplingOptions] + ], }, total=False, ) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 32428bb339..fc7dfc1994 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -585,15 +585,19 @@ def test_metrics_batcher_lock_reset_in_child_after_fork(sentry_init): or not hasattr(os, "register_at_fork"), reason="requires POSIX fork and os.register_at_fork (Python 3.7+)", ) -def test_lock_free_metrics_batcher_state_reset_in_child_after_fork(): - """The child must not inherit the parent's queued metrics or flusher - thread, and must be able to start its own flusher.""" - batcher = LockFreeMetricsBatcher( - capture_func=lambda envelope: None, - record_lost_func=lambda *args, **kwargs: None, +def test_metrics_window_sampling_state_reset_in_child_after_fork(sentry_init): + """The child must not inherit the parent's queued metrics, sampling window + or flusher thread, and must be able to start its own flusher.""" + sentry_init( + _experiments={"metrics_window_sampling": {"per_name": {"test.sampled": 2}}} ) + batcher = sentry_sdk.get_client().metrics_batcher + assert isinstance(batcher, LockFreeMetricsBatcher) - batcher.add(_metric()) + sentry_sdk.metrics.count("test.sampled", 1) + sentry_sdk.metrics.count("test.unsampled", 1) + original_window = batcher._window + batcher._retire_window(time.monotonic(), force=True) batcher._active.flag = True batcher._flush_event.set() batcher._running = False @@ -604,6 +608,9 @@ def test_lock_free_metrics_batcher_state_reset_in_child_after_fork(): batcher._flusher is None and batcher._flusher_pid is None and len(batcher._queue) == 0 + and len(batcher._retired) == 0 + and batcher._window is not original_window + and not batcher._window.reservoirs and not getattr(batcher._active, "flag", False) and not batcher._flush_event.is_set() and batcher._running is True @@ -611,30 +618,31 @@ def test_lock_free_metrics_batcher_state_reset_in_child_after_fork(): restarted = batcher._ensure_thread() and batcher._flusher_pid == os.getpid() os._exit(0 if reset and restarted else 1) - batcher.kill() _, status = os.waitpid(pid, 0) assert os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0 -def test_lock_free_metrics_batcher_concurrent_adds(monkeypatch): +def test_metrics_window_sampling_concurrent_adds( + sentry_init, capture_items, capture_record_lost_event_calls, monkeypatch +): + # Keep everything in one window, however slow the interleaved adds are + monkeypatch.setattr(sentry_sdk._lock_free_metrics_batcher, "WINDOW_SECONDS", 60.0) # Small enough that the queue overflows, so both paths race monkeypatch.setattr(LockFreeMetricsBatcher, "MAX_BEFORE_DROP", 2000) monkeypatch.setattr(LockFreeMetricsBatcher, "MAX_BEFORE_FLUSH", 500) - sent = [] - lost = [] - batcher = LockFreeMetricsBatcher( - capture_func=lambda envelope: sent.extend( - envelope.items[0].payload.json["items"] - ), - record_lost_func=lambda *args, **kwargs: lost.append(kwargs["quantity"]), + sentry_init( + _experiments={"metrics_window_sampling": {"per_name": {"test.hot": 1 / 6}}} ) + items = capture_items("trace_metric") + record_lost_event_calls = capture_record_lost_event_calls() num_threads, per_thread = 8, 1000 def add_metrics(): for _ in range(per_thread): - batcher.add(_metric()) + sentry_sdk.metrics.count("test.hot", 1) + sentry_sdk.metrics.count("test.cold", 1) # Switch threads as often as possible to interleave adds original_interval = sys.getswitchinterval() @@ -648,28 +656,266 @@ def add_metrics(): finally: sys.setswitchinterval(original_interval) - batcher.flush() - batcher.kill() + get_client().flush() total = num_threads * per_thread + + def lost(reason): + return sum( + quantity + for lost_reason, data_category, _, quantity in record_lost_event_calls + if lost_reason == reason and data_category == "trace_metric" + ) + + def sent(name): + return [item.payload for item in items if item.payload["name"] == name] + # flush() doesn't wait for a batch the background flusher already took, so # give it a moment deadline = time.monotonic() + 5 - while len(sent) + sum(lost) < total and time.monotonic() < deadline: + while ( + len(sent("test.cold")) + lost("queue_overflow") < total + and time.monotonic() < deadline + ): time.sleep(0.01) - # Every metric is either sent or reported as lost, exactly once - assert len(sent) + sum(lost) == total + # Every unsampled metric is either sent or reported as overflow, exactly once + assert len(sent("test.cold")) + lost("queue_overflow") == total + # Every sampled metric is either kept or reported as sampled out + hot = sent("test.hot") + assert 1 <= len(hot) <= 10 + assert len(hot) + lost("sample_rate") == total + for metric in hot: + assert metric["attributes"]["sentry.client_sample_rate"] == len(hot) / total -def _metric(): - return { - "timestamp": 0.0, - "trace_id": None, - "span_id": None, - "name": "test.counter", - "type": "counter", - "value": 1.0, - "unit": None, - "attributes": {}, - } + +def test_metrics_not_sampled_by_default(sentry_init, capture_items): + sentry_init() + items = capture_items("trace_metric") + + for _ in range(100): + sentry_sdk.metrics.count("test.counter", 1) + + get_client().flush() + + metrics = [item.payload for item in items] + assert len(metrics) == 100 + + for metric in metrics: + assert "sentry.client_sample_rate" not in metric["attributes"] + + +def test_metrics_window_sampling_basics( + sentry_init, capture_items, capture_record_lost_event_calls +): + sentry_init(_experiments={"metrics_window_sampling": True}) + items = capture_items("trace_metric") + record_lost_event_calls = capture_record_lost_event_calls() + + for _ in range(200): + sentry_sdk.metrics.count("test.hot", 1) + for _ in range(3): + sentry_sdk.metrics.count("test.cold", 1) + + get_client().flush() + + metrics = [item.payload for item in items] + assert len(metrics) == 13 + + hot = [metric for metric in metrics if metric["name"] == "test.hot"] + assert len(hot) == 10 + for metric in hot: + assert metric["attributes"]["sentry.client_sample_rate"] == 0.05 + + cold = [metric for metric in metrics if metric["name"] == "test.cold"] + assert len(cold) == 3 + for metric in cold: + assert "sentry.client_sample_rate" not in metric["attributes"] + + assert record_lost_event_calls == [ + ("sample_rate", "trace_metric", None, 190), + ("sample_rate", "trace_metric_byte", None, mock.ANY), + ] + + +@pytest.mark.parametrize( + "sample_per_second,expected_count", + [ + (4, 20), + (0.2, 1), + (0.05, 1), + (float("inf"), 10), + (float("nan"), 10), + (0, 10), + (-1, 10), + ("fast", 10), + ], +) +def test_metrics_window_sampling_sample_per_second( + sentry_init, capture_items, sample_per_second, expected_count +): + sentry_init( + _experiments={ + "metrics_window_sampling": {"sample_per_second": sample_per_second} + } + ) + items = capture_items("trace_metric") + + for _ in range(200): + sentry_sdk.metrics.count("test.counter", 1) + + get_client().flush() + + assert len(items) == expected_count + + +def test_metrics_window_sampling_per_name(sentry_init, capture_items): + sentry_init( + _experiments={"metrics_window_sampling": {"per_name": {"test.hot": 0.2}}} + ) + items = capture_items("trace_metric") + + for _ in range(200): + sentry_sdk.metrics.count("test.hot", 1) + sentry_sdk.metrics.count("test.other", 1) + + get_client().flush() + + metrics = [item.payload for item in items] + assert len(metrics) == 201 + + (hot,) = [metric for metric in metrics if metric["name"] == "test.hot"] + assert hot["attributes"]["sentry.client_sample_rate"] == 1 / 200 + + other = [metric for metric in metrics if metric["name"] == "test.other"] + for metric in other: + assert "sentry.client_sample_rate" not in metric["attributes"] + + +@pytest.mark.parametrize( + "client_sample_rate,expected_sample_rate", + [ + (0.5, 0.05), + (5, 0.1), + (0, 0.1), + (-1, 0.1), + (float("nan"), 0.1), + ("0.5", 0.1), + ], +) +def test_metrics_window_sampling_multiplies_client_sample_rate( + sentry_init, capture_items, client_sample_rate, expected_sample_rate +): + sentry_init(_experiments={"metrics_window_sampling": True}) + items = capture_items("trace_metric") + + for _ in range(100): + sentry_sdk.metrics.count( + "test.counter", + 1, + attributes={"sentry.client_sample_rate": client_sample_rate}, + ) + + get_client().flush() + + metrics = [item.payload for item in items] + assert len(metrics) == 10 + + for metric in metrics: + assert metric["attributes"]["sentry.client_sample_rate"] == expected_sample_rate + + +def test_metrics_window_sampling_new_window(sentry_init, capture_items, monkeypatch): + # Shorten the window so that the test doesn't have to wait 5 seconds for it + # to close + monkeypatch.setattr(sentry_sdk._lock_free_metrics_batcher, "WINDOW_SECONDS", 1.0) + + sentry_init(_experiments={"metrics_window_sampling": {"sample_per_second": 10}}) + items = capture_items("trace_metric") + + for _ in range(30): + sentry_sdk.metrics.count("test.counter", 1) + time.sleep(1.1) + for _ in range(30): + sentry_sdk.metrics.count("test.counter", 1) + + get_client().flush() + + metrics = [item.payload for item in items] + assert len(metrics) == 20 + + for metric in metrics: + assert metric["attributes"]["sentry.client_sample_rate"] == 1 / 3 + + +def test_metrics_window_sampling_does_not_cap_names(sentry_init, capture_items): + sentry_init(_experiments={"metrics_window_sampling": True}) + items = capture_items("trace_metric") + + for i in range(600): + sentry_sdk.metrics.count(f"test.counter{i}", 1) + sentry_sdk.metrics.count(f"test.counter{i}", 1) + + get_client().flush() + + assert len(items) == 1200 + + +def test_metrics_window_sampling_drops_after_max_reached( + sentry_init, capture_items, capture_record_lost_event_calls, monkeypatch +): + monkeypatch.setattr(LockFreeMetricsBatcher, "MAX_BEFORE_DROP", 25) + + sentry_init(_experiments={"metrics_window_sampling": True}) + items = capture_items("trace_metric") + record_lost_event_calls = capture_record_lost_event_calls() + + for name in ("test.a", "test.b", "test.c"): + for _ in range(10): + sentry_sdk.metrics.count(name, 1) + + get_client().flush() + + metrics = [item.payload for item in items] + assert len(metrics) == 25 + + # No name went over its limit, so nothing was sampled and the only loss is + # the overflow + for metric in metrics: + assert "sentry.client_sample_rate" not in metric["attributes"] + + assert len(record_lost_event_calls) == 5 + for lost_event_call in record_lost_event_calls: + reason, data_category, item, quantity = lost_event_call + + assert reason == "queue_overflow" + assert data_category == "trace_metric" + assert quantity == 1 + assert item.payload.json["items"][0]["name"] == "test.c" + + +def test_metrics_window_sampling_after_before_send_metric(sentry_init, capture_items): + before_send_metric_calls = 0 + + def _before_metric(record, hint): + nonlocal before_send_metric_calls + before_send_metric_calls += 1 + return record + + sentry_init( + before_send_metric=_before_metric, + _experiments={"metrics_window_sampling": True}, + ) + items = capture_items("trace_metric") + + for _ in range(200): + sentry_sdk.metrics.count("test.counter", 1) + + get_client().flush() + + # Sampling happens after before_send_metric, so it still sees every metric + assert before_send_metric_calls == 200 + + metrics = [item.payload for item in items] + assert len(metrics) == 10 From 21fd75e3bb752414d399a42bca4c02b5861be21e Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 18 Nov 2026 17:30:05 +0100 Subject: [PATCH 02/20] per thread state --- sentry_sdk/_lock_free_metrics_batcher.py | 171 +++++++++++++---------- tests/test_metrics.py | 33 +++-- 2 files changed, 113 insertions(+), 91 deletions(-) diff --git a/sentry_sdk/_lock_free_metrics_batcher.py b/sentry_sdk/_lock_free_metrics_batcher.py index d83b29989c..ff757992e5 100644 --- a/sentry_sdk/_lock_free_metrics_batcher.py +++ b/sentry_sdk/_lock_free_metrics_batcher.py @@ -1,4 +1,3 @@ -import itertools import math import os import random @@ -24,35 +23,37 @@ DEFAULT_SAMPLE_PER_SECOND = 2 # Matches FLUSH_WAIT_TIME, so each window goes out in one flush WINDOW_SECONDS = 5.0 -# Marks a window's writers once it's retired -_RETIRED = "retired" class _Reservoir: - __slots__ = ("kept", "seen", "overflowed") + __slots__ = ("kept", "seen") def __init__(self) -> None: self.kept: "list[Metric]" = [] - self.seen = itertools.count() - # Metrics that were seen but couldn't get a slot because the window ran - # out of memory budget. They're reported as queue_overflow, so they must - # not also count towards the sample rate - self.overflowed = itertools.count() + self.seen = 0 + + +class _ThreadState: + """One thread's reservoirs in a window. Only that thread writes to it, so + it needs no synchronization.""" + + __slots__ = ("reservoirs", "held") + + def __init__(self) -> None: + self.reservoirs: "dict[str, _Reservoir]" = {} + # Slots across this thread's reservoirs, to bound memory + self.held = 0 class _Window: - __slots__ = ("start", "reservoirs", "held", "retire_claims", "writers") + __slots__ = ("start", "threads", "retire_claims") def __init__(self) -> None: self.start = time.monotonic() - self.reservoirs: "dict[str, _Reservoir]" = {} - # Slots handed out across all reservoirs, to bound memory - self.held = itertools.count() + # Thread ident -> that thread's reservoirs. Belongs to the window, so + # threads that are gone disappear with it + self.threads: "dict[int, _ThreadState]" = {} self.retire_claims: "list[object]" = [] - # Adds currently writing to this window, plus the _RETIRED mark once - # it's retired. Both live in one dict, so registering and checking for - # the mark can't be reordered against setting it. - self.writers: "dict[object, bool]" = {} class LockFreeMetricsBatcher: @@ -119,6 +120,11 @@ def _parse_limit(self, value: float) -> int: # extrapolate from return min(max(1, int(rate * WINDOW_SECONDS)), self.MAX_BEFORE_DROP) + def _limit(self, name: str) -> "Optional[int]": + if not self._sampling: + return None + return self._name_limits.get(name, self._default_limit) + def _ensure_thread(self) -> bool: """For forking processes we might need to restart this thread. This ensures that our process actually has that thread running. @@ -159,11 +165,7 @@ def add(self, item: "Metric") -> None: if not self._ensure_thread(): return None - limit = ( - self._name_limits.get(item["name"], self._default_limit) - if self._sampling - else None - ) + limit = self._limit(item["name"]) if limit is None: kept = self._add_to_queue(item) else: @@ -188,49 +190,34 @@ def _add_to_window(self, item: "Metric", limit: int) -> bool: if time.monotonic() - self._window.start >= WINDOW_SECONDS: self._retire_window(force=False) - # Register before touching the window. A consumer harvests a retired - # window only once every registered add has left, and an add that - # registers after the window was retired sees the mark and moves on to - # the new window. So every metric is either harvested or in a later - # window. - token = object() - while True: - window = self._window - writers = window.writers - writers[token] = True - if _RETIRED not in writers: - break - del writers[token] - - try: - name = item["name"] - reservoir = window.reservoirs.get(name) - if reservoir is None: - reservoir = window.reservoirs.setdefault(name, _Reservoir()) - - seen = next(reservoir.seen) - if seen < limit: - # Replacing a slot in a full reservoir doesn't need more memory, - # only a new slot does - if next(window.held) >= self.MAX_BEFORE_DROP: - next(reservoir.overflowed) - return False - reservoir.kept.append(item) - return True - - index = random.randrange(seen + 1) + # A thread that's still writing when its window is harvested loses + # that metric. It has almost always been counted already, so it's + # reported as sampled out. + window = self._window + thread_id = threading.get_ident() + state = window.threads.get(thread_id) + if state is None: + state = window.threads.setdefault(thread_id, _ThreadState()) + + # Reservoir sampling (Algorithm R) on this thread's own reservoir + name = item["name"] + reservoir = state.reservoirs.get(name) + if reservoir is None: + reservoir = state.reservoirs[name] = _Reservoir() + if len(reservoir.kept) < limit: + if state.held >= self.MAX_BEFORE_DROP: + # Out of memory budget. Not seen, so it doesn't count towards + # the sample rate + return False + state.held += 1 + reservoir.seen += 1 + reservoir.kept.append(item) + else: + reservoir.seen += 1 + index = random.randrange(reservoir.seen) if index < limit: - try: - reservoir.kept[index] = item - except IndexError: - # The slot hasn't been filled yet by a concurrent add, or - # never will be because that add overflowed. Either way this - # metric is sampled out, which keeps the reservoir within - # its memory budget - pass - return True - finally: - del writers[token] + reservoir.kept[index] = item + return True def _retire_window(self, force: bool) -> None: window = self._window @@ -245,10 +232,7 @@ def _retire_window(self, force: bool) -> None: # Someone else is retiring this window return - # Swap before marking, so that adds turned away by the mark find the - # new window right away self._window = _Window() - window.writers[_RETIRED] = True self._retired.append(window) def kill(self) -> None: @@ -284,10 +268,6 @@ def _flush(self, force: bool = False) -> "Optional[Envelope]": except IndexError: # Another consumer took it break - # Adds that registered before the window was retired are only - # a few bytecodes from leaving - while len(window.writers) > 1: - time.sleep(0) sampled_out.extend(self._harvest(window, items)) if not items: @@ -320,17 +300,24 @@ def _flush(self, force: bool = False) -> "Optional[Envelope]": def _harvest( self, window: "_Window", items: "list[Metric]" ) -> "list[tuple[Metric, int]]": + # Copies, since a thread that's still writing may add to them + by_name: "dict[str, list[_Reservoir]]" = {} + for state in window.threads.copy().values(): + for name, reservoir in state.reservoirs.copy().items(): + by_name.setdefault(name, []).append(reservoir) + sampled_out = [] - for reservoir in window.reservoirs.values(): - kept = reservoir.kept + for name, reservoirs in by_name.items(): + limit = self._limit(name) + if limit is None: + continue + + seen = sum(reservoir.seen for reservoir in reservoirs) + kept = self._merge(reservoirs, limit, seen) if not kept: continue items.extend(kept) - - # Nothing writes to a harvested window, so reading the counters by - # advancing them is fine - seen = next(reservoir.seen) - next(reservoir.overflowed) if seen <= len(kept): # Nothing was sampled out continue @@ -347,6 +334,36 @@ def _harvest( return sampled_out + @staticmethod + def _merge(reservoirs: "list[_Reservoir]", limit: int, seen: int) -> "list[Metric]": + """Merge per-thread samples of one name into one uniform sample of at + most ``limit`` metrics out of all ``seen``.""" + # Split the slots across threads like drawing without replacement from + # all seen metrics, so each thread gets slots in proportion to how much + # it saw + left = [reservoir.seen for reservoir in reservoirs] + remaining = seen + slots = [0] * len(reservoirs) + for _ in range(min(limit, seen)): + pick = random.randrange(remaining) + i = 0 + for i, count in enumerate(left): + if pick < count: + break + pick -= count + slots[i] += 1 + left[i] -= 1 + remaining -= 1 + + # A random subset of a uniform sample is still a uniform sample + kept = [] + for reservoir, count in zip(reservoirs, slots): + thread_kept = reservoir.kept[:] + # A thread caught writing can have counted a metric it hasn't + # stored yet + kept.extend(random.sample(thread_kept, min(count, len(thread_kept)))) + return kept + def _record_lost(self, item: "Metric") -> None: envelope_item = None with capture_internal_exceptions(): diff --git a/tests/test_metrics.py b/tests/test_metrics.py index ff8cfccf87..40c1feafb2 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -551,8 +551,8 @@ def before_send_metric(metric, _): reason="requires POSIX fork and os.register_at_fork (Python 3.7+)", ) def test_metrics_window_sampling_state_reset_in_child_after_fork(sentry_init): - """The child must not inherit the parent's queued metrics, sampling window - or flusher thread, and must be able to start its own flusher.""" + """The child must not inherit the parent's queued metrics or flusher + thread, and must be able to start its own flusher.""" sentry_init( _experiments={"metrics_window_sampling": {"per_name": {"test.sampled": 2}}} ) @@ -561,8 +561,6 @@ def test_metrics_window_sampling_state_reset_in_child_after_fork(sentry_init): sentry_sdk.metrics.count("test.sampled", 1) sentry_sdk.metrics.count("test.unsampled", 1) - original_window = batcher._window - batcher._retire_window(force=True) batcher._flush_event.set() batcher._running = False @@ -572,9 +570,6 @@ def test_metrics_window_sampling_state_reset_in_child_after_fork(sentry_init): batcher._flusher is None and batcher._flusher_pid is None and len(batcher._queue) == 0 - and len(batcher._retired) == 0 - and batcher._window is not original_window - and not batcher._window.reservoirs and not batcher._flush_event.is_set() and batcher._running is True ) @@ -623,25 +618,35 @@ def add_metrics(): total = num_threads * per_thread - def lost(reason): + def sampled_out(): return sum( quantity - for lost_reason, data_category, _, quantity in record_lost_event_calls - if lost_reason == reason and data_category == "trace_metric" + for reason, data_category, _, quantity in record_lost_event_calls + if reason == "sample_rate" and data_category == "trace_metric" + ) + + def overflowed(name): + return sum( + quantity + for reason, data_category, item, quantity in record_lost_event_calls + if reason == "queue_overflow" + and data_category == "trace_metric" + and item.payload.json["items"][0]["name"] == name ) def sent(name): return [item.payload for item in items if item.payload["name"] == name] # Every unsampled metric is either sent or reported as overflow, exactly once - assert len(sent("test.cold")) + lost("queue_overflow") == total + assert len(sent("test.cold")) + overflowed("test.cold") == total - # Every sampled metric is either kept or reported as sampled out + # Every sampled metric is either kept, sampled out or reported as overflow hot = sent("test.hot") + seen = total - overflowed("test.hot") assert 1 <= len(hot) <= 10 - assert len(hot) + lost("sample_rate") == total + assert len(hot) + sampled_out() == seen for metric in hot: - assert metric["attributes"]["sentry.client_sample_rate"] == len(hot) / total + assert metric["attributes"]["sentry.client_sample_rate"] == len(hot) / seen def test_metrics_not_sampled_by_default(sentry_init, capture_items): From 343bef67f6699ec9f0b174840a97d645debe9288 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 18 Nov 2026 17:56:57 +0100 Subject: [PATCH 03/20] simplify more --- sentry_sdk/_lock_free_metrics_batcher.py | 67 ++---- sentry_sdk/client.py | 34 +++ tests/test_metrics.py | 290 ++++++++++------------- 3 files changed, 180 insertions(+), 211 deletions(-) diff --git a/sentry_sdk/_lock_free_metrics_batcher.py b/sentry_sdk/_lock_free_metrics_batcher.py index ff757992e5..181c893b52 100644 --- a/sentry_sdk/_lock_free_metrics_batcher.py +++ b/sentry_sdk/_lock_free_metrics_batcher.py @@ -1,4 +1,3 @@ -import math import os import random import threading @@ -11,7 +10,6 @@ from sentry_sdk.utils import ( capture_internal_exceptions, format_timestamp, - logger, serialize_attribute, ) @@ -33,26 +31,18 @@ def __init__(self) -> None: self.seen = 0 -class _ThreadState: - """One thread's reservoirs in a window. Only that thread writes to it, so - it needs no synchronization.""" - - __slots__ = ("reservoirs", "held") - - def __init__(self) -> None: - self.reservoirs: "dict[str, _Reservoir]" = {} - # Slots across this thread's reservoirs, to bound memory - self.held = 0 - - class _Window: - __slots__ = ("start", "threads", "retire_claims") + __slots__ = ("start", "threads", "slots", "retire_claims") def __init__(self) -> None: self.start = time.monotonic() - # Thread ident -> that thread's reservoirs. Belongs to the window, so - # threads that are gone disappear with it - self.threads: "dict[int, _ThreadState]" = {} + # Thread ident -> that thread's reservoirs by name. Only that thread + # writes to its reservoirs, so they need no synchronization. Belongs to + # the window, so threads that are gone disappear with it + self.threads: "dict[int, dict[str, _Reservoir]]" = {} + # One entry per reservoir slot taken in this window, across all + # threads. len() and append() are atomic, so this counts exactly + self.slots: "list[None]" = [] self.retire_claims: "list[object]" = [] @@ -102,20 +92,7 @@ def _reset_thread_state(self) -> None: # Holds the claim of the one thread that gets to start the flusher. self._flusher_claim: "dict[str, object]" = {} - def _parse_limit(self, value: float) -> int: - try: - rate = float(value) - except (TypeError, ValueError): - rate = math.nan - - if not (math.isfinite(rate) and rate > 0): - logger.warning( - "[Metrics] Given window sampling rate is invalid. It must be a positive number. Got %r, using %s instead.", - value, - DEFAULT_SAMPLE_PER_SECOND, - ) - rate = DEFAULT_SAMPLE_PER_SECOND - + def _parse_limit(self, rate: float) -> int: # Never go below 1, so we always keep at least one metric per name to # extrapolate from return min(max(1, int(rate * WINDOW_SECONDS)), self.MAX_BEFORE_DROP) @@ -178,7 +155,7 @@ def _add_to_queue(self, item: "Metric") -> bool: queue = self._queue # Checking and appending isn't atomic. # The queue can go over the limit by at most one metric per concurrently adding thread. - if len(queue) >= self.MAX_BEFORE_DROP: + if len(queue) + len(self._window.slots) >= self.MAX_BEFORE_DROP: return False queue.append(item) @@ -195,21 +172,23 @@ def _add_to_window(self, item: "Metric", limit: int) -> bool: # reported as sampled out. window = self._window thread_id = threading.get_ident() - state = window.threads.get(thread_id) - if state is None: - state = window.threads.setdefault(thread_id, _ThreadState()) + reservoirs = window.threads.get(thread_id) + if reservoirs is None: + reservoirs = window.threads.setdefault(thread_id, {}) # Reservoir sampling (Algorithm R) on this thread's own reservoir name = item["name"] - reservoir = state.reservoirs.get(name) + reservoir = reservoirs.get(name) if reservoir is None: - reservoir = state.reservoirs[name] = _Reservoir() + reservoir = reservoirs[name] = _Reservoir() if len(reservoir.kept) < limit: - if state.held >= self.MAX_BEFORE_DROP: - # Out of memory budget. Not seen, so it doesn't count towards - # the sample rate + # Queued metrics and reservoir slots share one memory budget. Like + # the queue, this can go over by one slot per concurrently adding + # thread + if len(self._queue) + len(window.slots) >= self.MAX_BEFORE_DROP: + # Not seen, so it doesn't count towards the sample rate return False - state.held += 1 + window.slots.append(None) reservoir.seen += 1 reservoir.kept.append(item) else: @@ -302,8 +281,8 @@ def _harvest( ) -> "list[tuple[Metric, int]]": # Copies, since a thread that's still writing may add to them by_name: "dict[str, list[_Reservoir]]" = {} - for state in window.threads.copy().values(): - for name, reservoir in state.reservoirs.copy().items(): + for reservoirs in window.threads.copy().values(): + for name, reservoir in reservoirs.copy().items(): by_name.setdefault(name, []).append(reservoir) sampled_out = [] diff --git a/sentry_sdk/client.py b/sentry_sdk/client.py index 8ac1f68b31..bea5c83c06 100644 --- a/sentry_sdk/client.py +++ b/sentry_sdk/client.py @@ -1,3 +1,4 @@ +import math import os import random import socket @@ -130,6 +131,39 @@ def _get_options(*args: "Optional[str]", **kwargs: "Any") -> "Dict[str, Any]": rv["data_collection"] = _resolve_data_collection(rv) + window_sampling = rv["_experiments"].get("metrics_window_sampling") + if isinstance(window_sampling, dict): + # Copy, so the options the user passed aren't changed + window_sampling = dict(window_sampling) + rates: "list[tuple[Dict[str, Any], str]]" = [ + (window_sampling, "sample_per_second") + ] + if isinstance(window_sampling.get("per_name"), dict): + per_name = window_sampling["per_name"] = dict(window_sampling["per_name"]) + rates.extend((per_name, name) for name in per_name) + + for options, key in rates: + if key not in options: + continue + rate = options[key] + if not ( + isinstance(rate, (int, float)) + and not isinstance(rate, bool) + and math.isfinite(rate) + and rate > 0 + ): + logger.warning( + "[Metrics] Given window sampling rate is invalid. It must be a positive number. Got %r, ignoring it.", + rate, + ) + del options[key] + + rv["_experiments"] = { + **rv["_experiments"], + # Dropping every invalid rate must not turn sampling off + "metrics_window_sampling": window_sampling or True, + } + if rv["socket_options"] and not isinstance(rv["socket_options"], list): logger.warning( "Ignoring socket_options because of unexpected format. See urllib3.HTTPConnection.socket_options for the expected format." diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 40c1feafb2..557269fe9c 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -1,6 +1,3 @@ -import os -import sys -import threading import time from unittest import mock @@ -13,28 +10,14 @@ from sentry_sdk.consts import SPANDATA, VERSION -@pytest.fixture(params=["locked", "lock_free"]) -def sentry_init(request, sentry_init): - """Run every test in this file against both metrics batchers.""" - - def inner(*a, **kw): - sentry_init(*a, **kw) - client = sentry_sdk.get_client() - batcher = client.metrics_batcher - if request.param == "lock_free" and isinstance(batcher, MetricsBatcher): - client.metrics_batcher = LockFreeMetricsBatcher( - capture_func=batcher._capture_func, - record_lost_func=batcher._record_lost_func, - ) - batcher.kill() - elif request.param == "locked" and isinstance(batcher, LockFreeMetricsBatcher): - pytest.skip("Window sampling always uses the lock-free batcher") - - return inner - - -def test_metrics_basics(sentry_init, capture_items): +@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) +def test_metrics_basics(sentry_init, batcher_cls, capture_items): sentry_init() + client = get_client() + client.metrics_batcher = batcher_cls( + capture_func=client.metrics_batcher._capture_func, + record_lost_func=client.metrics_batcher._record_lost_func, + ) items = capture_items("trace_metric") sentry_sdk.metrics.count("test.counter", 1) @@ -64,8 +47,14 @@ def test_metrics_basics(sentry_init, capture_items): assert metrics[2]["unit"] == "second" -def test_metrics_with_attributes(sentry_init, capture_items): +@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) +def test_metrics_with_attributes(sentry_init, batcher_cls, capture_items): sentry_init(release="1.0.0", environment="test", server_name="test-server") + client = get_client() + client.metrics_batcher = batcher_cls( + capture_func=client.metrics_batcher._capture_func, + record_lost_func=client.metrics_batcher._record_lost_func, + ) items = capture_items("trace_metric") sentry_sdk.metrics.count( @@ -87,8 +76,14 @@ def test_metrics_with_attributes(sentry_init, capture_items): assert metrics[0]["attributes"]["sentry.sdk.version"] == VERSION -def test_metrics_with_user(sentry_init, capture_items): +@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) +def test_metrics_with_user(sentry_init, batcher_cls, capture_items): sentry_init(data_collection={"user_info": True}) + client = get_client() + client.metrics_batcher = batcher_cls( + capture_func=client.metrics_batcher._capture_func, + record_lost_func=client.metrics_batcher._record_lost_func, + ) items = capture_items("trace_metric") sentry_sdk.set_user( @@ -106,8 +101,14 @@ def test_metrics_with_user(sentry_init, capture_items): assert metrics[0]["attributes"]["user.name"] == "testuser" -def test_metrics_no_user_if_user_info_off(sentry_init, capture_items): +@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) +def test_metrics_no_user_if_user_info_off(sentry_init, batcher_cls, capture_items): sentry_init(data_collection={"user_info": False}) + client = get_client() + client.metrics_batcher = batcher_cls( + capture_func=client.metrics_batcher._capture_func, + record_lost_func=client.metrics_batcher._record_lost_func, + ) items = capture_items("trace_metric") sentry_sdk.set_user( @@ -125,8 +126,14 @@ def test_metrics_no_user_if_user_info_off(sentry_init, capture_items): assert "user.name" not in metrics[0]["attributes"] -def test_metrics_with_span(sentry_init, capture_items): +@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) +def test_metrics_with_span(sentry_init, batcher_cls, capture_items): sentry_init(traces_sample_rate=1.0) + client = get_client() + client.metrics_batcher = batcher_cls( + capture_func=client.metrics_batcher._capture_func, + record_lost_func=client.metrics_batcher._record_lost_func, + ) items = capture_items("trace_metric") with sentry_sdk.start_span(name="test-span") as segment: @@ -142,8 +149,14 @@ def test_metrics_with_span(sentry_init, capture_items): assert metrics[0]["span_id"] == segment.span_id -def test_metrics_tracing_without_performance(sentry_init, capture_items): +@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) +def test_metrics_tracing_without_performance(sentry_init, batcher_cls, capture_items): sentry_init() + client = get_client() + client.metrics_batcher = batcher_cls( + capture_func=client.metrics_batcher._capture_func, + record_lost_func=client.metrics_batcher._record_lost_func, + ) items = capture_items("trace_metric") with sentry_sdk.isolation_scope() as isolation_scope: @@ -163,7 +176,8 @@ def test_metrics_tracing_without_performance(sentry_init, capture_items): assert "span_id" not in metrics[0] -def test_metrics_before_send(sentry_init, capture_items): +@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) +def test_metrics_before_send(sentry_init, batcher_cls, capture_items): before_metric_called = False def _before_metric(record, hint): @@ -189,6 +203,11 @@ def _before_metric(record, hint): sentry_init( before_send_metric=_before_metric, ) + client = get_client() + client.metrics_batcher = batcher_cls( + capture_func=client.metrics_batcher._capture_func, + record_lost_func=client.metrics_batcher._record_lost_func, + ) items = capture_items("trace_metric") sentry_sdk.metrics.count("test.skip", 1) @@ -203,8 +222,9 @@ def _before_metric(record, hint): @pytest.mark.tests_internal_exceptions +@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) def test_metrics_before_send_raises_does_not_crash_application( - sentry_init, capture_items + sentry_init, batcher_cls, capture_items ): def _before_metric(record, hint): raise ValueError("before_send_metric error") @@ -212,6 +232,11 @@ def _before_metric(record, hint): sentry_init( before_send_metric=_before_metric, ) + client = get_client() + client.metrics_batcher = batcher_cls( + capture_func=client.metrics_batcher._capture_func, + record_lost_func=client.metrics_batcher._record_lost_func, + ) items = capture_items("trace_metric") sentry_sdk.metrics.count("test.keep", 1) @@ -225,13 +250,19 @@ def _before_metric(record, hint): @pytest.mark.tests_internal_exceptions +@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) def test_metrics_before_send_raises_records_callback_error( - sentry_init, capture_items, capture_record_lost_event_calls + sentry_init, batcher_cls, capture_items, capture_record_lost_event_calls ): def _before_metric(record, hint): raise ValueError("before_send_metric error") sentry_init(before_send_metric=_before_metric) + client = get_client() + client.metrics_batcher = batcher_cls( + capture_func=client.metrics_batcher._capture_func, + record_lost_func=client.metrics_batcher._record_lost_func, + ) items = capture_items("trace_metric") record_lost_event_calls = capture_record_lost_event_calls() @@ -251,13 +282,19 @@ def _before_metric(record, hint): assert item.payload.json["items"][0]["name"] == "test.keep" +@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) def test_metrics_before_send_returns_none_records_before_send( - sentry_init, capture_items, capture_record_lost_event_calls + sentry_init, batcher_cls, capture_items, capture_record_lost_event_calls ): def _before_metric(record, hint): return None sentry_init(before_send_metric=_before_metric) + client = get_client() + client.metrics_batcher = batcher_cls( + capture_func=client.metrics_batcher._capture_func, + record_lost_func=client.metrics_batcher._record_lost_func, + ) items = capture_items("trace_metric") record_lost_event_calls = capture_record_lost_event_calls() @@ -277,8 +314,14 @@ def _before_metric(record, hint): assert item.payload.json["items"][0]["name"] == "test.drop" -def test_transport_format(sentry_init, capture_envelopes): +@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) +def test_transport_format(sentry_init, batcher_cls, capture_envelopes): sentry_init(server_name="test-server", release="1.0.0") + client = get_client() + client.metrics_batcher = batcher_cls( + capture_func=client.metrics_batcher._capture_func, + record_lost_func=client.metrics_batcher._record_lost_func, + ) envelopes = capture_envelopes() @@ -340,9 +383,14 @@ def test_transport_format(sentry_init, capture_envelopes): } -def test_batcher_drops_metrics(sentry_init, monkeypatch): +@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) +def test_batcher_drops_metrics(sentry_init, batcher_cls, monkeypatch): sentry_init() - client = sentry_sdk.get_client() + client = get_client() + client.metrics_batcher = batcher_cls( + capture_func=client.metrics_batcher._capture_func, + record_lost_func=client.metrics_batcher._record_lost_func, + ) def no_op_flush(*args, **kwargs): pass @@ -376,8 +424,14 @@ def record_lost_event(reason, data_category=None, item=None, *, quantity=1): assert item.payload.json["items"][0]["name"] == "test.counter" -def test_metric_gets_attributes_from_scopes(sentry_init, capture_items): +@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) +def test_metric_gets_attributes_from_scopes(sentry_init, batcher_cls, capture_items): sentry_init() + client = get_client() + client.metrics_batcher = batcher_cls( + capture_func=client.metrics_batcher._capture_func, + record_lost_func=client.metrics_batcher._record_lost_func, + ) items = capture_items("trace_metric") @@ -402,8 +456,16 @@ def test_metric_gets_attributes_from_scopes(sentry_init, capture_items): assert "current.attribute" not in metric2["attributes"] -def test_metric_attributes_override_scope_attributes(sentry_init, capture_items): +@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) +def test_metric_attributes_override_scope_attributes( + sentry_init, batcher_cls, capture_items +): sentry_init() + client = get_client() + client.metrics_batcher = batcher_cls( + capture_func=client.metrics_batcher._capture_func, + record_lost_func=client.metrics_batcher._record_lost_func, + ) items = capture_items("trace_metric") @@ -421,10 +483,16 @@ def test_metric_attributes_override_scope_attributes(sentry_init, capture_items) assert metric["attributes"]["temp.attribute"] == "value2" -def test_log_array_attributes(sentry_init, capture_envelopes): +@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) +def test_log_array_attributes(sentry_init, batcher_cls, capture_envelopes): """Test homogeneous list and tuple attributes, and fallback for inhomogeneous collections.""" sentry_init() + client = get_client() + client.metrics_batcher = batcher_cls( + capture_func=client.metrics_batcher._capture_func, + record_lost_func=client.metrics_batcher._record_lost_func, + ) envelopes = capture_envelopes() @@ -477,7 +545,10 @@ def test_log_array_attributes(sentry_init, capture_envelopes): } -def test_attributes_preserialized_in_before_send(sentry_init, capture_items): +@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) +def test_attributes_preserialized_in_before_send( + sentry_init, batcher_cls, capture_items +): """We don't surface user-held references to objects in attributes.""" def before_send_metric(metric, _): @@ -489,6 +560,11 @@ def before_send_metric(metric, _): return metric sentry_init(before_send_metric=before_send_metric) + client = get_client() + client.metrics_batcher = batcher_cls( + capture_func=client.metrics_batcher._capture_func, + record_lost_func=client.metrics_batcher._record_lost_func, + ) items = capture_items("trace_metric") @@ -518,7 +594,8 @@ class Cat: assert isinstance(metric["attributes"]["dictionary"], str) -def test_array_attributes_deep_copied_in_before_send(sentry_init): +@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) +def test_array_attributes_deep_copied_in_before_send(sentry_init, batcher_cls): """We don't surface user-held references to objects in attributes.""" strings = ["value1", "value2"] @@ -531,6 +608,11 @@ def before_send_metric(metric, _): return metric sentry_init(before_send_metric=before_send_metric) + client = get_client() + client.metrics_batcher = batcher_cls( + capture_func=client.metrics_batcher._capture_func, + record_lost_func=client.metrics_batcher._record_lost_func, + ) sentry_sdk.metrics.count( "test.counter", @@ -544,127 +626,6 @@ def before_send_metric(metric, _): get_client().flush() -@pytest.mark.skipif( - sys.platform == "win32" - or not hasattr(os, "fork") - or not hasattr(os, "register_at_fork"), - reason="requires POSIX fork and os.register_at_fork (Python 3.7+)", -) -def test_metrics_window_sampling_state_reset_in_child_after_fork(sentry_init): - """The child must not inherit the parent's queued metrics or flusher - thread, and must be able to start its own flusher.""" - sentry_init( - _experiments={"metrics_window_sampling": {"per_name": {"test.sampled": 2}}} - ) - batcher = sentry_sdk.get_client().metrics_batcher - assert isinstance(batcher, LockFreeMetricsBatcher) - - sentry_sdk.metrics.count("test.sampled", 1) - sentry_sdk.metrics.count("test.unsampled", 1) - batcher._flush_event.set() - batcher._running = False - - pid = os.fork() - if pid == 0: - reset = ( - batcher._flusher is None - and batcher._flusher_pid is None - and len(batcher._queue) == 0 - and not batcher._flush_event.is_set() - and batcher._running is True - ) - restarted = batcher._ensure_thread() and batcher._flusher_pid == os.getpid() - os._exit(0 if reset and restarted else 1) - - _, status = os.waitpid(pid, 0) - assert os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0 - - -def test_metrics_window_sampling_concurrent_adds( - sentry_init, capture_items, capture_record_lost_event_calls, monkeypatch -): - # Keep everything in one window, however slow the interleaved adds are - monkeypatch.setattr(sentry_sdk._lock_free_metrics_batcher, "WINDOW_SECONDS", 60.0) - # Small enough that the queue overflows, so both paths race - monkeypatch.setattr(LockFreeMetricsBatcher, "MAX_BEFORE_DROP", 2000) - monkeypatch.setattr(LockFreeMetricsBatcher, "MAX_BEFORE_FLUSH", 500) - - sentry_init( - _experiments={"metrics_window_sampling": {"per_name": {"test.hot": 1 / 6}}} - ) - items = capture_items("trace_metric") - record_lost_event_calls = capture_record_lost_event_calls() - - num_threads, per_thread = 8, 1000 - - def add_metrics(): - for _ in range(per_thread): - sentry_sdk.metrics.count("test.hot", 1) - sentry_sdk.metrics.count("test.cold", 1) - - # Switch threads as often as possible to interleave adds - original_interval = sys.getswitchinterval() - sys.setswitchinterval(1e-6) - try: - threads = [threading.Thread(target=add_metrics) for _ in range(num_threads)] - for thread in threads: - thread.start() - for thread in threads: - thread.join() - finally: - sys.setswitchinterval(original_interval) - - get_client().flush() - - total = num_threads * per_thread - - def sampled_out(): - return sum( - quantity - for reason, data_category, _, quantity in record_lost_event_calls - if reason == "sample_rate" and data_category == "trace_metric" - ) - - def overflowed(name): - return sum( - quantity - for reason, data_category, item, quantity in record_lost_event_calls - if reason == "queue_overflow" - and data_category == "trace_metric" - and item.payload.json["items"][0]["name"] == name - ) - - def sent(name): - return [item.payload for item in items if item.payload["name"] == name] - - # Every unsampled metric is either sent or reported as overflow, exactly once - assert len(sent("test.cold")) + overflowed("test.cold") == total - - # Every sampled metric is either kept, sampled out or reported as overflow - hot = sent("test.hot") - seen = total - overflowed("test.hot") - assert 1 <= len(hot) <= 10 - assert len(hot) + sampled_out() == seen - for metric in hot: - assert metric["attributes"]["sentry.client_sample_rate"] == len(hot) / seen - - -def test_metrics_not_sampled_by_default(sentry_init, capture_items): - sentry_init() - items = capture_items("trace_metric") - - for _ in range(100): - sentry_sdk.metrics.count("test.counter", 1) - - get_client().flush() - - metrics = [item.payload for item in items] - assert len(metrics) == 100 - - for metric in metrics: - assert "sentry.client_sample_rate" not in metric["attributes"] - - def test_metrics_window_sampling_basics( sentry_init, capture_items, capture_record_lost_event_calls ): @@ -704,11 +665,6 @@ def test_metrics_window_sampling_basics( (4, 20), (0.2, 1), (0.05, 1), - (float("inf"), 10), - (float("nan"), 10), - (0, 10), - (-1, 10), - ("fast", 10), ], ) def test_metrics_window_sampling_sample_per_second( From 7e4983629c47b0a93e1a18b958c4eaabe7328315 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 18 Nov 2026 18:09:48 +0100 Subject: [PATCH 04/20] only use class if option set --- sentry_sdk/_lock_free_metrics_batcher.py | 53 +++--- tests/test_metrics.py | 200 +++++++++-------------- 2 files changed, 101 insertions(+), 152 deletions(-) diff --git a/sentry_sdk/_lock_free_metrics_batcher.py b/sentry_sdk/_lock_free_metrics_batcher.py index a5dea62d52..cf7d5da24e 100644 --- a/sentry_sdk/_lock_free_metrics_batcher.py +++ b/sentry_sdk/_lock_free_metrics_batcher.py @@ -59,25 +59,23 @@ def __init__( self, capture_func: "Callable[[Envelope], None]", record_lost_func: "Callable[..., None]", - window_sampling: "Union[bool, MetricWindowSamplingOptions, None]" = None, + window_sampling: "Union[bool, MetricWindowSamplingOptions]", ) -> None: self._capture_func = capture_func self._record_lost_func = record_lost_func self._default_limit: "Optional[int]" = None self._name_limits: "dict[str, int]" = {} - if window_sampling: - options = window_sampling if isinstance(window_sampling, dict) else {} - per_name = options.get("per_name") - if isinstance(per_name, dict): - self._name_limits = { - name: self._parse_limit(rate) for name, rate in per_name.items() - } - if "sample_per_second" in options or not self._name_limits: - self._default_limit = self._parse_limit( - options.get("sample_per_second", DEFAULT_SAMPLE_PER_SECOND) - ) - self._sampling = bool(self._name_limits) or self._default_limit is not None + options = window_sampling if isinstance(window_sampling, dict) else {} + per_name = options.get("per_name") + if isinstance(per_name, dict): + self._name_limits = { + name: self._parse_limit(rate) for name, rate in per_name.items() + } + if "sample_per_second" in options or not self._name_limits: + self._default_limit = self._parse_limit( + options.get("sample_per_second", DEFAULT_SAMPLE_PER_SECOND) + ) self._reset_thread_state() @@ -98,8 +96,6 @@ def _parse_limit(self, rate: float) -> int: return min(max(1, int(rate * WINDOW_SECONDS)), self.MAX_BEFORE_DROP) def _limit(self, name: str) -> "Optional[int]": - if not self._sampling: - return None return self._name_limits.get(name, self._default_limit) def _ensure_thread(self) -> bool: @@ -237,17 +233,16 @@ def _flush(self, force: bool = False) -> "Optional[Envelope]": break sampled_out: "list[tuple[Metric, int]]" = [] - if self._sampling: - # Retire before taking the retired windows, so a flush() sends the - # window it just retired - self._retire_window(force) - for _ in range(len(self._retired)): - try: - window = self._retired.popleft() - except IndexError: - # Another consumer took it - break - sampled_out.extend(self._harvest(window, items)) + # Retire before taking the retired windows, so a flush() sends the + # window it just retired + self._retire_window(force) + for _ in range(len(self._retired)): + try: + window = self._retired.popleft() + except IndexError: + # Another consumer took it + break + sampled_out.extend(self._harvest(window, items)) if not items: return None @@ -340,7 +335,11 @@ def _merge(reservoirs: "list[_Reservoir]", limit: int, seen: int) -> "list[Metri thread_kept = reservoir.kept[:] # A thread caught writing can have counted a metric it hasn't # stored yet - kept.extend(random.sample(thread_kept, min(count, len(thread_kept)))) + if count >= len(thread_kept): + # Keep the order metrics were added in when all of them stay + kept.extend(thread_kept) + else: + kept.extend(random.sample(thread_kept, count)) return kept def _record_lost(self, item: "Metric") -> None: diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 557269fe9c..356b2873a5 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -6,18 +6,12 @@ import sentry_sdk from sentry_sdk import get_client from sentry_sdk._lock_free_metrics_batcher import LockFreeMetricsBatcher -from sentry_sdk._metrics_batcher import MetricsBatcher from sentry_sdk.consts import SPANDATA, VERSION -@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) -def test_metrics_basics(sentry_init, batcher_cls, capture_items): - sentry_init() - client = get_client() - client.metrics_batcher = batcher_cls( - capture_func=client.metrics_batcher._capture_func, - record_lost_func=client.metrics_batcher._record_lost_func, - ) +@pytest.mark.parametrize("window_sampling", [None, True]) +def test_metrics_basics(sentry_init, window_sampling, capture_items): + sentry_init(_experiments={"metrics_window_sampling": window_sampling}) items = capture_items("trace_metric") sentry_sdk.metrics.count("test.counter", 1) @@ -47,13 +41,13 @@ def test_metrics_basics(sentry_init, batcher_cls, capture_items): assert metrics[2]["unit"] == "second" -@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) -def test_metrics_with_attributes(sentry_init, batcher_cls, capture_items): - sentry_init(release="1.0.0", environment="test", server_name="test-server") - client = get_client() - client.metrics_batcher = batcher_cls( - capture_func=client.metrics_batcher._capture_func, - record_lost_func=client.metrics_batcher._record_lost_func, +@pytest.mark.parametrize("window_sampling", [None, True]) +def test_metrics_with_attributes(sentry_init, window_sampling, capture_items): + sentry_init( + release="1.0.0", + environment="test", + server_name="test-server", + _experiments={"metrics_window_sampling": window_sampling}, ) items = capture_items("trace_metric") @@ -76,13 +70,11 @@ def test_metrics_with_attributes(sentry_init, batcher_cls, capture_items): assert metrics[0]["attributes"]["sentry.sdk.version"] == VERSION -@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) -def test_metrics_with_user(sentry_init, batcher_cls, capture_items): - sentry_init(data_collection={"user_info": True}) - client = get_client() - client.metrics_batcher = batcher_cls( - capture_func=client.metrics_batcher._capture_func, - record_lost_func=client.metrics_batcher._record_lost_func, +@pytest.mark.parametrize("window_sampling", [None, True]) +def test_metrics_with_user(sentry_init, window_sampling, capture_items): + sentry_init( + data_collection={"user_info": True}, + _experiments={"metrics_window_sampling": window_sampling}, ) items = capture_items("trace_metric") @@ -101,13 +93,11 @@ def test_metrics_with_user(sentry_init, batcher_cls, capture_items): assert metrics[0]["attributes"]["user.name"] == "testuser" -@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) -def test_metrics_no_user_if_user_info_off(sentry_init, batcher_cls, capture_items): - sentry_init(data_collection={"user_info": False}) - client = get_client() - client.metrics_batcher = batcher_cls( - capture_func=client.metrics_batcher._capture_func, - record_lost_func=client.metrics_batcher._record_lost_func, +@pytest.mark.parametrize("window_sampling", [None, True]) +def test_metrics_no_user_if_user_info_off(sentry_init, window_sampling, capture_items): + sentry_init( + data_collection={"user_info": False}, + _experiments={"metrics_window_sampling": window_sampling}, ) items = capture_items("trace_metric") @@ -126,13 +116,11 @@ def test_metrics_no_user_if_user_info_off(sentry_init, batcher_cls, capture_item assert "user.name" not in metrics[0]["attributes"] -@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) -def test_metrics_with_span(sentry_init, batcher_cls, capture_items): - sentry_init(traces_sample_rate=1.0) - client = get_client() - client.metrics_batcher = batcher_cls( - capture_func=client.metrics_batcher._capture_func, - record_lost_func=client.metrics_batcher._record_lost_func, +@pytest.mark.parametrize("window_sampling", [None, True]) +def test_metrics_with_span(sentry_init, window_sampling, capture_items): + sentry_init( + traces_sample_rate=1.0, + _experiments={"metrics_window_sampling": window_sampling}, ) items = capture_items("trace_metric") @@ -149,14 +137,11 @@ def test_metrics_with_span(sentry_init, batcher_cls, capture_items): assert metrics[0]["span_id"] == segment.span_id -@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) -def test_metrics_tracing_without_performance(sentry_init, batcher_cls, capture_items): - sentry_init() - client = get_client() - client.metrics_batcher = batcher_cls( - capture_func=client.metrics_batcher._capture_func, - record_lost_func=client.metrics_batcher._record_lost_func, - ) +@pytest.mark.parametrize("window_sampling", [None, True]) +def test_metrics_tracing_without_performance( + sentry_init, window_sampling, capture_items +): + sentry_init(_experiments={"metrics_window_sampling": window_sampling}) items = capture_items("trace_metric") with sentry_sdk.isolation_scope() as isolation_scope: @@ -176,8 +161,8 @@ def test_metrics_tracing_without_performance(sentry_init, batcher_cls, capture_i assert "span_id" not in metrics[0] -@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) -def test_metrics_before_send(sentry_init, batcher_cls, capture_items): +@pytest.mark.parametrize("window_sampling", [None, True]) +def test_metrics_before_send(sentry_init, window_sampling, capture_items): before_metric_called = False def _before_metric(record, hint): @@ -202,11 +187,7 @@ def _before_metric(record, hint): sentry_init( before_send_metric=_before_metric, - ) - client = get_client() - client.metrics_batcher = batcher_cls( - capture_func=client.metrics_batcher._capture_func, - record_lost_func=client.metrics_batcher._record_lost_func, + _experiments={"metrics_window_sampling": window_sampling}, ) items = capture_items("trace_metric") @@ -222,20 +203,16 @@ def _before_metric(record, hint): @pytest.mark.tests_internal_exceptions -@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) +@pytest.mark.parametrize("window_sampling", [None, True]) def test_metrics_before_send_raises_does_not_crash_application( - sentry_init, batcher_cls, capture_items + sentry_init, window_sampling, capture_items ): def _before_metric(record, hint): raise ValueError("before_send_metric error") sentry_init( before_send_metric=_before_metric, - ) - client = get_client() - client.metrics_batcher = batcher_cls( - capture_func=client.metrics_batcher._capture_func, - record_lost_func=client.metrics_batcher._record_lost_func, + _experiments={"metrics_window_sampling": window_sampling}, ) items = capture_items("trace_metric") @@ -250,18 +227,16 @@ def _before_metric(record, hint): @pytest.mark.tests_internal_exceptions -@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) +@pytest.mark.parametrize("window_sampling", [None, True]) def test_metrics_before_send_raises_records_callback_error( - sentry_init, batcher_cls, capture_items, capture_record_lost_event_calls + sentry_init, window_sampling, capture_items, capture_record_lost_event_calls ): def _before_metric(record, hint): raise ValueError("before_send_metric error") - sentry_init(before_send_metric=_before_metric) - client = get_client() - client.metrics_batcher = batcher_cls( - capture_func=client.metrics_batcher._capture_func, - record_lost_func=client.metrics_batcher._record_lost_func, + sentry_init( + before_send_metric=_before_metric, + _experiments={"metrics_window_sampling": window_sampling}, ) items = capture_items("trace_metric") record_lost_event_calls = capture_record_lost_event_calls() @@ -282,18 +257,16 @@ def _before_metric(record, hint): assert item.payload.json["items"][0]["name"] == "test.keep" -@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) +@pytest.mark.parametrize("window_sampling", [None, True]) def test_metrics_before_send_returns_none_records_before_send( - sentry_init, batcher_cls, capture_items, capture_record_lost_event_calls + sentry_init, window_sampling, capture_items, capture_record_lost_event_calls ): def _before_metric(record, hint): return None - sentry_init(before_send_metric=_before_metric) - client = get_client() - client.metrics_batcher = batcher_cls( - capture_func=client.metrics_batcher._capture_func, - record_lost_func=client.metrics_batcher._record_lost_func, + sentry_init( + before_send_metric=_before_metric, + _experiments={"metrics_window_sampling": window_sampling}, ) items = capture_items("trace_metric") record_lost_event_calls = capture_record_lost_event_calls() @@ -314,13 +287,12 @@ def _before_metric(record, hint): assert item.payload.json["items"][0]["name"] == "test.drop" -@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) -def test_transport_format(sentry_init, batcher_cls, capture_envelopes): - sentry_init(server_name="test-server", release="1.0.0") - client = get_client() - client.metrics_batcher = batcher_cls( - capture_func=client.metrics_batcher._capture_func, - record_lost_func=client.metrics_batcher._record_lost_func, +@pytest.mark.parametrize("window_sampling", [None, True]) +def test_transport_format(sentry_init, window_sampling, capture_envelopes): + sentry_init( + server_name="test-server", + release="1.0.0", + _experiments={"metrics_window_sampling": window_sampling}, ) envelopes = capture_envelopes() @@ -383,14 +355,9 @@ def test_transport_format(sentry_init, batcher_cls, capture_envelopes): } -@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) -def test_batcher_drops_metrics(sentry_init, batcher_cls, monkeypatch): +def test_batcher_drops_metrics(sentry_init, monkeypatch): sentry_init() - client = get_client() - client.metrics_batcher = batcher_cls( - capture_func=client.metrics_batcher._capture_func, - record_lost_func=client.metrics_batcher._record_lost_func, - ) + client = sentry_sdk.get_client() def no_op_flush(*args, **kwargs): pass @@ -424,14 +391,11 @@ def record_lost_event(reason, data_category=None, item=None, *, quantity=1): assert item.payload.json["items"][0]["name"] == "test.counter" -@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) -def test_metric_gets_attributes_from_scopes(sentry_init, batcher_cls, capture_items): - sentry_init() - client = get_client() - client.metrics_batcher = batcher_cls( - capture_func=client.metrics_batcher._capture_func, - record_lost_func=client.metrics_batcher._record_lost_func, - ) +@pytest.mark.parametrize("window_sampling", [None, True]) +def test_metric_gets_attributes_from_scopes( + sentry_init, window_sampling, capture_items +): + sentry_init(_experiments={"metrics_window_sampling": window_sampling}) items = capture_items("trace_metric") @@ -456,16 +420,11 @@ def test_metric_gets_attributes_from_scopes(sentry_init, batcher_cls, capture_it assert "current.attribute" not in metric2["attributes"] -@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) +@pytest.mark.parametrize("window_sampling", [None, True]) def test_metric_attributes_override_scope_attributes( - sentry_init, batcher_cls, capture_items + sentry_init, window_sampling, capture_items ): - sentry_init() - client = get_client() - client.metrics_batcher = batcher_cls( - capture_func=client.metrics_batcher._capture_func, - record_lost_func=client.metrics_batcher._record_lost_func, - ) + sentry_init(_experiments={"metrics_window_sampling": window_sampling}) items = capture_items("trace_metric") @@ -483,16 +442,11 @@ def test_metric_attributes_override_scope_attributes( assert metric["attributes"]["temp.attribute"] == "value2" -@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) -def test_log_array_attributes(sentry_init, batcher_cls, capture_envelopes): +@pytest.mark.parametrize("window_sampling", [None, True]) +def test_log_array_attributes(sentry_init, window_sampling, capture_envelopes): """Test homogeneous list and tuple attributes, and fallback for inhomogeneous collections.""" - sentry_init() - client = get_client() - client.metrics_batcher = batcher_cls( - capture_func=client.metrics_batcher._capture_func, - record_lost_func=client.metrics_batcher._record_lost_func, - ) + sentry_init(_experiments={"metrics_window_sampling": window_sampling}) envelopes = capture_envelopes() @@ -545,9 +499,9 @@ def test_log_array_attributes(sentry_init, batcher_cls, capture_envelopes): } -@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) +@pytest.mark.parametrize("window_sampling", [None, True]) def test_attributes_preserialized_in_before_send( - sentry_init, batcher_cls, capture_items + sentry_init, window_sampling, capture_items ): """We don't surface user-held references to objects in attributes.""" @@ -559,11 +513,9 @@ def before_send_metric(metric, _): return metric - sentry_init(before_send_metric=before_send_metric) - client = get_client() - client.metrics_batcher = batcher_cls( - capture_func=client.metrics_batcher._capture_func, - record_lost_func=client.metrics_batcher._record_lost_func, + sentry_init( + before_send_metric=before_send_metric, + _experiments={"metrics_window_sampling": window_sampling}, ) items = capture_items("trace_metric") @@ -594,8 +546,8 @@ class Cat: assert isinstance(metric["attributes"]["dictionary"], str) -@pytest.mark.parametrize("batcher_cls", [MetricsBatcher, LockFreeMetricsBatcher]) -def test_array_attributes_deep_copied_in_before_send(sentry_init, batcher_cls): +@pytest.mark.parametrize("window_sampling", [None, True]) +def test_array_attributes_deep_copied_in_before_send(sentry_init, window_sampling): """We don't surface user-held references to objects in attributes.""" strings = ["value1", "value2"] @@ -607,11 +559,9 @@ def before_send_metric(metric, _): return metric - sentry_init(before_send_metric=before_send_metric) - client = get_client() - client.metrics_batcher = batcher_cls( - capture_func=client.metrics_batcher._capture_func, - record_lost_func=client.metrics_batcher._record_lost_func, + sentry_init( + before_send_metric=before_send_metric, + _experiments={"metrics_window_sampling": window_sampling}, ) sentry_sdk.metrics.count( From 14893a6e48b44fd6b2a09427d2b83564bf64d632 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 18 Nov 2026 20:11:07 +0100 Subject: [PATCH 05/20] simplify --- sentry_sdk/_lock_free_metrics_batcher.py | 58 +++++++++---------- sentry_sdk/_types.py | 6 -- sentry_sdk/client.py | 50 ++++++++--------- sentry_sdk/consts.py | 7 +-- tests/test_metrics.py | 71 ++++++++++++------------ 5 files changed, 86 insertions(+), 106 deletions(-) diff --git a/sentry_sdk/_lock_free_metrics_batcher.py b/sentry_sdk/_lock_free_metrics_batcher.py index cf7d5da24e..3d9f29337c 100644 --- a/sentry_sdk/_lock_free_metrics_batcher.py +++ b/sentry_sdk/_lock_free_metrics_batcher.py @@ -14,11 +14,10 @@ ) if TYPE_CHECKING: - from typing import Any, Callable, Optional, Union + from typing import Any, Callable, Optional - from sentry_sdk._types import Metric, MetricWindowSamplingOptions + from sentry_sdk._types import Metric -DEFAULT_SAMPLE_PER_SECOND = 2 # Matches FLUSH_WAIT_TIME, so each window goes out in one flush WINDOW_SECONDS = 5.0 @@ -32,17 +31,17 @@ def __init__(self) -> None: class _Window: - __slots__ = ("start", "threads", "slots", "retire_claims") + __slots__ = ("start", "threads", "size", "size_lock", "retire_claims") def __init__(self) -> None: self.start = time.monotonic() - # Thread ident -> that thread's reservoirs by name. Only that thread - # writes to its reservoirs, so they need no synchronization. Belongs to - # the window, so threads that are gone disappear with it + # Thread identifier mapping to the thread's reservoir. self.threads: "dict[int, dict[str, _Reservoir]]" = {} - # One entry per reservoir slot taken in this window, across all - # threads. len() and append() are atomic, so this counts exactly - self.slots: "list[None]" = [] + # Metrics stored in reservoirs in this window, across all threads + self.size = 0 + # Keeps concurrent increments of size from getting lost. The only lock + # adds take, and only when a reservoir takes a new slot + self.size_lock = threading.Lock() self.retire_claims: "list[object]" = [] @@ -59,23 +58,16 @@ def __init__( self, capture_func: "Callable[[Envelope], None]", record_lost_func: "Callable[..., None]", - window_sampling: "Union[bool, MetricWindowSamplingOptions]", + window_sampling: "dict[str, float]", ) -> None: self._capture_func = capture_func self._record_lost_func = record_lost_func - self._default_limit: "Optional[int]" = None - self._name_limits: "dict[str, int]" = {} - options = window_sampling if isinstance(window_sampling, dict) else {} - per_name = options.get("per_name") - if isinstance(per_name, dict): - self._name_limits = { - name: self._parse_limit(rate) for name, rate in per_name.items() - } - if "sample_per_second" in options or not self._name_limits: - self._default_limit = self._parse_limit( - options.get("sample_per_second", DEFAULT_SAMPLE_PER_SECOND) - ) + # Metric name -> metrics kept per window. Names not in here aren't + # sampled + self._limits = { + name: self._parse_limit(rate) for name, rate in window_sampling.items() + } self._reset_thread_state() @@ -96,7 +88,7 @@ def _parse_limit(self, rate: float) -> int: return min(max(1, int(rate * WINDOW_SECONDS)), self.MAX_BEFORE_DROP) def _limit(self, name: str) -> "Optional[int]": - return self._name_limits.get(name, self._default_limit) + return self._limits.get(name) def _ensure_thread(self) -> bool: """For forking processes we might need to restart this thread. @@ -148,14 +140,13 @@ def add(self, item: "Metric") -> None: self._record_lost(item) def _add_to_queue(self, item: "Metric") -> bool: - queue = self._queue # Checking and appending isn't atomic. # The queue can go over the limit by at most one metric per concurrently adding thread. - if len(queue) + len(self._window.slots) >= self.MAX_BEFORE_DROP: + if len(self._queue) + self._window.size >= self.MAX_BEFORE_DROP: return False - queue.append(item) - if len(queue) >= self.MAX_BEFORE_FLUSH and not self._flush_event.is_set(): + self._queue.append(item) + if len(self._queue) >= self.MAX_BEFORE_FLUSH and not self._flush_event.is_set(): self._flush_event.set() return True @@ -178,13 +169,14 @@ def _add_to_window(self, item: "Metric", limit: int) -> bool: if reservoir is None: reservoir = reservoirs[name] = _Reservoir() if len(reservoir.kept) < limit: - # Queued metrics and reservoir slots share one memory budget. Like - # the queue, this can go over by one slot per concurrently adding - # thread - if len(self._queue) + len(window.slots) >= self.MAX_BEFORE_DROP: + # Queued metrics and reservoir metrics share one memory budget. + # Checking and incrementing isn't atomic, so this can go over by + # one slot per concurrently adding thread + if len(self._queue) + window.size >= self.MAX_BEFORE_DROP: # Not seen, so it doesn't count towards the sample rate return False - window.slots.append(None) + with window.size_lock: + window.size += 1 reservoir.seen += 1 reservoir.kept.append(item) else: diff --git a/sentry_sdk/_types.py b/sentry_sdk/_types.py index d77bee2b24..40efa2d228 100644 --- a/sentry_sdk/_types.py +++ b/sentry_sdk/_types.py @@ -359,12 +359,6 @@ class DataCollection(TypedDict): }, ) - MetricWindowSamplingOptions = TypedDict( - "MetricWindowSamplingOptions", - {"sample_per_second": float, "per_name": Dict[str, float]}, - total=False, - ) - SpanJSON = TypedDict( "SpanJSON", { diff --git a/sentry_sdk/client.py b/sentry_sdk/client.py index bea5c83c06..a2ce2bb57b 100644 --- a/sentry_sdk/client.py +++ b/sentry_sdk/client.py @@ -132,36 +132,32 @@ def _get_options(*args: "Optional[str]", **kwargs: "Any") -> "Dict[str, Any]": rv["data_collection"] = _resolve_data_collection(rv) window_sampling = rv["_experiments"].get("metrics_window_sampling") - if isinstance(window_sampling, dict): + if window_sampling is not None: # Copy, so the options the user passed aren't changed - window_sampling = dict(window_sampling) - rates: "list[tuple[Dict[str, Any], str]]" = [ - (window_sampling, "sample_per_second") - ] - if isinstance(window_sampling.get("per_name"), dict): - per_name = window_sampling["per_name"] = dict(window_sampling["per_name"]) - rates.extend((per_name, name) for name in per_name) - - for options, key in rates: - if key not in options: - continue - rate = options[key] - if not ( - isinstance(rate, (int, float)) - and not isinstance(rate, bool) - and math.isfinite(rate) - and rate > 0 - ): - logger.warning( - "[Metrics] Given window sampling rate is invalid. It must be a positive number. Got %r, ignoring it.", - rate, - ) - del options[key] - + valid_rates = {} + if isinstance(window_sampling, dict): + for name, rate in window_sampling.items(): + if ( + isinstance(rate, (int, float)) + and not isinstance(rate, bool) + and math.isfinite(rate) + and rate > 0 + ): + valid_rates[name] = rate + else: + logger.warning( + "[Metrics] Given window sampling rate for %r is invalid. It must be a positive number. Got %r, not sampling it.", + name, + rate, + ) + else: + logger.warning( + "[Metrics] metrics_window_sampling must map metric names to rates. Got %r, not sampling.", + window_sampling, + ) rv["_experiments"] = { **rv["_experiments"], - # Dropping every invalid rate must not turn sampling off - "metrics_window_sampling": window_sampling or True, + "metrics_window_sampling": valid_rates, } if rv["socket_options"] and not isinstance(rv["socket_options"], list): diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index 83ff4d161d..f6113192f1 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -54,7 +54,6 @@ class CompressionAlgo(Enum): IgnoreSpansConfig, Log, Metric, - MetricWindowSamplingOptions, SpanJSON, TracesSampler, ) @@ -74,9 +73,9 @@ class CompressionAlgo(Enum): "transport_num_pools": Optional[int], "transport_http2": Optional[bool], "transport_async": Optional[bool], - "metrics_window_sampling": Optional[ - Union[bool, MetricWindowSamplingOptions] - ], + # Metric name -> metrics per second to keep. Names not in here + # aren't sampled + "metrics_window_sampling": Optional[Dict[str, float]], }, total=False, ) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 356b2873a5..3d629df52b 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -9,7 +9,7 @@ from sentry_sdk.consts import SPANDATA, VERSION -@pytest.mark.parametrize("window_sampling", [None, True]) +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) def test_metrics_basics(sentry_init, window_sampling, capture_items): sentry_init(_experiments={"metrics_window_sampling": window_sampling}) items = capture_items("trace_metric") @@ -41,7 +41,7 @@ def test_metrics_basics(sentry_init, window_sampling, capture_items): assert metrics[2]["unit"] == "second" -@pytest.mark.parametrize("window_sampling", [None, True]) +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) def test_metrics_with_attributes(sentry_init, window_sampling, capture_items): sentry_init( release="1.0.0", @@ -70,7 +70,7 @@ def test_metrics_with_attributes(sentry_init, window_sampling, capture_items): assert metrics[0]["attributes"]["sentry.sdk.version"] == VERSION -@pytest.mark.parametrize("window_sampling", [None, True]) +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) def test_metrics_with_user(sentry_init, window_sampling, capture_items): sentry_init( data_collection={"user_info": True}, @@ -93,7 +93,7 @@ def test_metrics_with_user(sentry_init, window_sampling, capture_items): assert metrics[0]["attributes"]["user.name"] == "testuser" -@pytest.mark.parametrize("window_sampling", [None, True]) +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) def test_metrics_no_user_if_user_info_off(sentry_init, window_sampling, capture_items): sentry_init( data_collection={"user_info": False}, @@ -116,7 +116,7 @@ def test_metrics_no_user_if_user_info_off(sentry_init, window_sampling, capture_ assert "user.name" not in metrics[0]["attributes"] -@pytest.mark.parametrize("window_sampling", [None, True]) +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) def test_metrics_with_span(sentry_init, window_sampling, capture_items): sentry_init( traces_sample_rate=1.0, @@ -137,7 +137,7 @@ def test_metrics_with_span(sentry_init, window_sampling, capture_items): assert metrics[0]["span_id"] == segment.span_id -@pytest.mark.parametrize("window_sampling", [None, True]) +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) def test_metrics_tracing_without_performance( sentry_init, window_sampling, capture_items ): @@ -161,7 +161,7 @@ def test_metrics_tracing_without_performance( assert "span_id" not in metrics[0] -@pytest.mark.parametrize("window_sampling", [None, True]) +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) def test_metrics_before_send(sentry_init, window_sampling, capture_items): before_metric_called = False @@ -203,7 +203,7 @@ def _before_metric(record, hint): @pytest.mark.tests_internal_exceptions -@pytest.mark.parametrize("window_sampling", [None, True]) +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) def test_metrics_before_send_raises_does_not_crash_application( sentry_init, window_sampling, capture_items ): @@ -227,7 +227,7 @@ def _before_metric(record, hint): @pytest.mark.tests_internal_exceptions -@pytest.mark.parametrize("window_sampling", [None, True]) +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) def test_metrics_before_send_raises_records_callback_error( sentry_init, window_sampling, capture_items, capture_record_lost_event_calls ): @@ -257,7 +257,7 @@ def _before_metric(record, hint): assert item.payload.json["items"][0]["name"] == "test.keep" -@pytest.mark.parametrize("window_sampling", [None, True]) +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) def test_metrics_before_send_returns_none_records_before_send( sentry_init, window_sampling, capture_items, capture_record_lost_event_calls ): @@ -287,7 +287,7 @@ def _before_metric(record, hint): assert item.payload.json["items"][0]["name"] == "test.drop" -@pytest.mark.parametrize("window_sampling", [None, True]) +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) def test_transport_format(sentry_init, window_sampling, capture_envelopes): sentry_init( server_name="test-server", @@ -391,7 +391,7 @@ def record_lost_event(reason, data_category=None, item=None, *, quantity=1): assert item.payload.json["items"][0]["name"] == "test.counter" -@pytest.mark.parametrize("window_sampling", [None, True]) +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) def test_metric_gets_attributes_from_scopes( sentry_init, window_sampling, capture_items ): @@ -420,7 +420,7 @@ def test_metric_gets_attributes_from_scopes( assert "current.attribute" not in metric2["attributes"] -@pytest.mark.parametrize("window_sampling", [None, True]) +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) def test_metric_attributes_override_scope_attributes( sentry_init, window_sampling, capture_items ): @@ -442,7 +442,7 @@ def test_metric_attributes_override_scope_attributes( assert metric["attributes"]["temp.attribute"] == "value2" -@pytest.mark.parametrize("window_sampling", [None, True]) +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) def test_log_array_attributes(sentry_init, window_sampling, capture_envelopes): """Test homogeneous list and tuple attributes, and fallback for inhomogeneous collections.""" @@ -499,7 +499,7 @@ def test_log_array_attributes(sentry_init, window_sampling, capture_envelopes): } -@pytest.mark.parametrize("window_sampling", [None, True]) +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) def test_attributes_preserialized_in_before_send( sentry_init, window_sampling, capture_items ): @@ -546,7 +546,7 @@ class Cat: assert isinstance(metric["attributes"]["dictionary"], str) -@pytest.mark.parametrize("window_sampling", [None, True]) +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) def test_array_attributes_deep_copied_in_before_send(sentry_init, window_sampling): """We don't surface user-held references to objects in attributes.""" @@ -579,7 +579,9 @@ def before_send_metric(metric, _): def test_metrics_window_sampling_basics( sentry_init, capture_items, capture_record_lost_event_calls ): - sentry_init(_experiments={"metrics_window_sampling": True}) + sentry_init( + _experiments={"metrics_window_sampling": {"test.hot": 2, "test.cold": 2}} + ) items = capture_items("trace_metric") record_lost_event_calls = capture_record_lost_event_calls() @@ -610,21 +612,15 @@ def test_metrics_window_sampling_basics( @pytest.mark.parametrize( - "sample_per_second,expected_count", + "rate,expected_count", [ (4, 20), (0.2, 1), (0.05, 1), ], ) -def test_metrics_window_sampling_sample_per_second( - sentry_init, capture_items, sample_per_second, expected_count -): - sentry_init( - _experiments={ - "metrics_window_sampling": {"sample_per_second": sample_per_second} - } - ) +def test_metrics_window_sampling_rate(sentry_init, capture_items, rate, expected_count): + sentry_init(_experiments={"metrics_window_sampling": {"test.counter": rate}}) items = capture_items("trace_metric") for _ in range(200): @@ -635,10 +631,8 @@ def test_metrics_window_sampling_sample_per_second( assert len(items) == expected_count -def test_metrics_window_sampling_per_name(sentry_init, capture_items): - sentry_init( - _experiments={"metrics_window_sampling": {"per_name": {"test.hot": 0.2}}} - ) +def test_metrics_window_sampling_unlisted_names(sentry_init, capture_items): + sentry_init(_experiments={"metrics_window_sampling": {"test.hot": 0.2}}) items = capture_items("trace_metric") for _ in range(200): @@ -672,7 +666,7 @@ def test_metrics_window_sampling_per_name(sentry_init, capture_items): def test_metrics_window_sampling_multiplies_client_sample_rate( sentry_init, capture_items, client_sample_rate, expected_sample_rate ): - sentry_init(_experiments={"metrics_window_sampling": True}) + sentry_init(_experiments={"metrics_window_sampling": {"test.counter": 2}}) items = capture_items("trace_metric") for _ in range(100): @@ -696,7 +690,7 @@ def test_metrics_window_sampling_new_window(sentry_init, capture_items, monkeypa # to close monkeypatch.setattr(sentry_sdk._lock_free_metrics_batcher, "WINDOW_SECONDS", 1.0) - sentry_init(_experiments={"metrics_window_sampling": {"sample_per_second": 10}}) + sentry_init(_experiments={"metrics_window_sampling": {"test.counter": 10}}) items = capture_items("trace_metric") for _ in range(30): @@ -715,7 +709,11 @@ def test_metrics_window_sampling_new_window(sentry_init, capture_items, monkeypa def test_metrics_window_sampling_does_not_cap_names(sentry_init, capture_items): - sentry_init(_experiments={"metrics_window_sampling": True}) + sentry_init( + _experiments={ + "metrics_window_sampling": {f"test.counter{i}": 2 for i in range(600)} + } + ) items = capture_items("trace_metric") for i in range(600): @@ -732,11 +730,12 @@ def test_metrics_window_sampling_drops_after_max_reached( ): monkeypatch.setattr(LockFreeMetricsBatcher, "MAX_BEFORE_DROP", 25) - sentry_init(_experiments={"metrics_window_sampling": True}) + names = ("test.a", "test.b", "test.c") + sentry_init(_experiments={"metrics_window_sampling": dict.fromkeys(names, 2)}) items = capture_items("trace_metric") record_lost_event_calls = capture_record_lost_event_calls() - for name in ("test.a", "test.b", "test.c"): + for name in names: for _ in range(10): sentry_sdk.metrics.count(name, 1) @@ -770,7 +769,7 @@ def _before_metric(record, hint): sentry_init( before_send_metric=_before_metric, - _experiments={"metrics_window_sampling": True}, + _experiments={"metrics_window_sampling": {"test.counter": 2}}, ) items = capture_items("trace_metric") From 183d1a16b83e58046806d9c39ac511f06e68ac1a Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 18 Nov 2026 20:56:16 +0100 Subject: [PATCH 06/20] simplify again --- sentry_sdk/_lock_free_metrics_batcher.py | 144 ++++++++--------------- tests/test_metrics.py | 35 +----- 2 files changed, 51 insertions(+), 128 deletions(-) diff --git a/sentry_sdk/_lock_free_metrics_batcher.py b/sentry_sdk/_lock_free_metrics_batcher.py index 3d9f29337c..11d1760492 100644 --- a/sentry_sdk/_lock_free_metrics_batcher.py +++ b/sentry_sdk/_lock_free_metrics_batcher.py @@ -1,8 +1,10 @@ +import bisect +import itertools import os import random import threading import time -from collections import deque +from collections import defaultdict, deque from datetime import datetime, timezone from typing import TYPE_CHECKING @@ -18,9 +20,6 @@ from sentry_sdk._types import Metric -# Matches FLUSH_WAIT_TIME, so each window goes out in one flush -WINDOW_SECONDS = 5.0 - class _Reservoir: __slots__ = ("kept", "seen") @@ -30,25 +29,39 @@ def __init__(self) -> None: self.seen = 0 +class _ThreadState: + """The metrics collected by one thread in a given window and their count.""" + + __slots__ = ("reservoirs", "size") + + def __init__(self) -> None: + self.reservoirs: "defaultdict[str, _Reservoir]" = defaultdict(_Reservoir) + self.size = 0 + + class _Window: - __slots__ = ("start", "threads", "size", "size_lock", "retire_claims") + """ + One time window in which multiple threads concurrently capture metrics. + Only the thread with the given identifier should change it's local state. + """ + + __slots__ = ("start", "threads", "retire_claims") def __init__(self) -> None: self.start = time.monotonic() - # Thread identifier mapping to the thread's reservoir. - self.threads: "dict[int, dict[str, _Reservoir]]" = {} - # Metrics stored in reservoirs in this window, across all threads - self.size = 0 - # Keeps concurrent increments of size from getting lost. The only lock - # adds take, and only when a reservoir takes a new slot - self.size_lock = threading.Lock() + self.threads: "defaultdict[int, _ThreadState]" = defaultdict(_ThreadState) self.retire_claims: "list[object]" = [] + def size(self) -> int: + return sum(state.size for state in self.threads.copy().values()) + class LockFreeMetricsBatcher: MAX_BEFORE_FLUSH = 1000 MAX_BEFORE_DROP = 10_000 FLUSH_WAIT_TIME = 5.0 + # Matches FLUSH_WAIT_TIME, so each window goes out in one flush + WINDOW_SECONDS = 5.0 TYPE = "trace_metric" CLIENT_REPORT_TYPE = "trace_metric" @@ -63,8 +76,6 @@ def __init__( self._capture_func = capture_func self._record_lost_func = record_lost_func - # Metric name -> metrics kept per window. Names not in here aren't - # sampled self._limits = { name: self._parse_limit(rate) for name, rate in window_sampling.items() } @@ -85,7 +96,7 @@ def _reset_thread_state(self) -> None: def _parse_limit(self, rate: float) -> int: # Never go below 1, so we always keep at least one metric per name to # extrapolate from - return min(max(1, int(rate * WINDOW_SECONDS)), self.MAX_BEFORE_DROP) + return min(max(1, int(rate * self.WINDOW_SECONDS)), self.MAX_BEFORE_DROP) def _limit(self, name: str) -> "Optional[int]": return self._limits.get(name) @@ -142,7 +153,7 @@ def add(self, item: "Metric") -> None: def _add_to_queue(self, item: "Metric") -> bool: # Checking and appending isn't atomic. # The queue can go over the limit by at most one metric per concurrently adding thread. - if len(self._queue) + self._window.size >= self.MAX_BEFORE_DROP: + if len(self._queue) + self._window.size() >= self.MAX_BEFORE_DROP: return False self._queue.append(item) @@ -151,32 +162,18 @@ def _add_to_queue(self, item: "Metric") -> bool: return True def _add_to_window(self, item: "Metric", limit: int) -> bool: - if time.monotonic() - self._window.start >= WINDOW_SECONDS: - self._retire_window(force=False) - - # A thread that's still writing when its window is harvested loses - # that metric. It has almost always been counted already, so it's - # reported as sampled out. window = self._window - thread_id = threading.get_ident() - reservoirs = window.threads.get(thread_id) - if reservoirs is None: - reservoirs = window.threads.setdefault(thread_id, {}) - - # Reservoir sampling (Algorithm R) on this thread's own reservoir - name = item["name"] - reservoir = reservoirs.get(name) - if reservoir is None: - reservoir = reservoirs[name] = _Reservoir() + if time.monotonic() - window.start >= self.WINDOW_SECONDS: + self._retire_window(window) + window = self._window + + state = window.threads[threading.get_ident()] + reservoir = state.reservoirs[item["name"]] if len(reservoir.kept) < limit: - # Queued metrics and reservoir metrics share one memory budget. - # Checking and incrementing isn't atomic, so this can go over by - # one slot per concurrently adding thread - if len(self._queue) + window.size >= self.MAX_BEFORE_DROP: - # Not seen, so it doesn't count towards the sample rate + # The size is off by at most one per adding thread. + if len(self._queue) + window.size() >= self.MAX_BEFORE_DROP: return False - with window.size_lock: - window.size += 1 + state.size += 1 reservoir.seen += 1 reservoir.kept.append(item) else: @@ -186,13 +183,7 @@ def _add_to_window(self, item: "Metric", limit: int) -> bool: reservoir.kept[index] = item return True - def _retire_window(self, force: bool) -> None: - window = self._window - if not force and time.monotonic() - window.start < WINDOW_SECONDS: - # Early flushes leave the window open, so a name can't go over its - # rate - return - + def _retire_window(self, window: "_Window") -> None: claim = object() window.retire_claims.append(claim) if window.retire_claims[0] is not claim: @@ -225,14 +216,14 @@ def _flush(self, force: bool = False) -> "Optional[Envelope]": break sampled_out: "list[tuple[Metric, int]]" = [] - # Retire before taking the retired windows, so a flush() sends the - # window it just retired - self._retire_window(force) + + window = self._window + if force or time.monotonic() - window.start >= self.WINDOW_SECONDS: + self._retire_window(window) for _ in range(len(self._retired)): try: window = self._retired.popleft() except IndexError: - # Another consumer took it break sampled_out.extend(self._harvest(window, items)) @@ -252,24 +243,15 @@ def _flush(self, force: bool = False) -> "Optional[Envelope]": data_category=self.CLIENT_REPORT_TYPE, quantity=count, ) - # Serializing every sampled out metric would cost as much as we saved - # by sampling, so we estimate from one - with capture_internal_exceptions(): - self._record_lost_func( - reason="sample_rate", - data_category="trace_metric_byte", - quantity=count * len(self._to_envelope_item([item]).get_bytes()), - ) return envelope def _harvest( self, window: "_Window", items: "list[Metric]" ) -> "list[tuple[Metric, int]]": - # Copies, since a thread that's still writing may add to them by_name: "dict[str, list[_Reservoir]]" = {} - for reservoirs in window.threads.copy().values(): - for name, reservoir in reservoirs.copy().items(): + for state in window.threads.copy().values(): + for name, reservoir in state.reservoirs.copy().items(): by_name.setdefault(name, []).append(reservoir) sampled_out = [] @@ -285,53 +267,27 @@ def _harvest( items.extend(kept) if seen <= len(kept): - # Nothing was sampled out continue rate = len(kept) / seen for item in kept: - attributes = item["attributes"] - client_rate = attributes.get("sentry.client_sample_rate") - if isinstance(client_rate, (int, float)) and 0 < client_rate <= 1: - attributes["sentry.client_sample_rate"] = rate * client_rate - else: - attributes["sentry.client_sample_rate"] = rate + item["attributes"]["sentry.client_sample_rate"] = rate sampled_out.append((kept[0], seen - len(kept))) return sampled_out @staticmethod def _merge(reservoirs: "list[_Reservoir]", limit: int, seen: int) -> "list[Metric]": - """Merge per-thread samples of one name into one uniform sample of at - most ``limit`` metrics out of all ``seen``.""" - # Split the slots across threads like drawing without replacement from - # all seen metrics, so each thread gets slots in proportion to how much - # it saw - left = [reservoir.seen for reservoir in reservoirs] - remaining = seen + """Draw uniform sample of at most `limit` metrics out of all `seen`.""" + bounds = list(itertools.accumulate(reservoir.seen for reservoir in reservoirs)) slots = [0] * len(reservoirs) - for _ in range(min(limit, seen)): - pick = random.randrange(remaining) - i = 0 - for i, count in enumerate(left): - if pick < count: - break - pick -= count - slots[i] += 1 - left[i] -= 1 - remaining -= 1 - - # A random subset of a uniform sample is still a uniform sample + for position in random.sample(range(seen), min(limit, seen)): + slots[bisect.bisect_right(bounds, position)] += 1 + kept = [] for reservoir, count in zip(reservoirs, slots): thread_kept = reservoir.kept[:] - # A thread caught writing can have counted a metric it hasn't - # stored yet - if count >= len(thread_kept): - # Keep the order metrics were added in when all of them stay - kept.extend(thread_kept) - else: - kept.extend(random.sample(thread_kept, count)) + kept.extend(random.sample(thread_kept, min(count, len(thread_kept)))) return kept def _record_lost(self, item: "Metric") -> None: diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 3d629df52b..8d09f226bb 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -652,43 +652,10 @@ def test_metrics_window_sampling_unlisted_names(sentry_init, capture_items): assert "sentry.client_sample_rate" not in metric["attributes"] -@pytest.mark.parametrize( - "client_sample_rate,expected_sample_rate", - [ - (0.5, 0.05), - (5, 0.1), - (0, 0.1), - (-1, 0.1), - (float("nan"), 0.1), - ("0.5", 0.1), - ], -) -def test_metrics_window_sampling_multiplies_client_sample_rate( - sentry_init, capture_items, client_sample_rate, expected_sample_rate -): - sentry_init(_experiments={"metrics_window_sampling": {"test.counter": 2}}) - items = capture_items("trace_metric") - - for _ in range(100): - sentry_sdk.metrics.count( - "test.counter", - 1, - attributes={"sentry.client_sample_rate": client_sample_rate}, - ) - - get_client().flush() - - metrics = [item.payload for item in items] - assert len(metrics) == 10 - - for metric in metrics: - assert metric["attributes"]["sentry.client_sample_rate"] == expected_sample_rate - - def test_metrics_window_sampling_new_window(sentry_init, capture_items, monkeypatch): # Shorten the window so that the test doesn't have to wait 5 seconds for it # to close - monkeypatch.setattr(sentry_sdk._lock_free_metrics_batcher, "WINDOW_SECONDS", 1.0) + monkeypatch.setattr(LockFreeMetricsBatcher, "WINDOW_SECONDS", 1.0) sentry_init(_experiments={"metrics_window_sampling": {"test.counter": 10}}) items = capture_items("trace_metric") From 51819038518660ee7b6350f26bee7f831e7a3c31 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 18 Nov 2026 21:03:13 +0100 Subject: [PATCH 07/20] remove some tests --- sentry_sdk/_lock_free_metrics_batcher.py | 8 ++- tests/test_metrics.py | 66 +----------------------- 2 files changed, 4 insertions(+), 70 deletions(-) diff --git a/sentry_sdk/_lock_free_metrics_batcher.py b/sentry_sdk/_lock_free_metrics_batcher.py index 11d1760492..137715e41c 100644 --- a/sentry_sdk/_lock_free_metrics_batcher.py +++ b/sentry_sdk/_lock_free_metrics_batcher.py @@ -60,8 +60,6 @@ class LockFreeMetricsBatcher: MAX_BEFORE_FLUSH = 1000 MAX_BEFORE_DROP = 10_000 FLUSH_WAIT_TIME = 5.0 - # Matches FLUSH_WAIT_TIME, so each window goes out in one flush - WINDOW_SECONDS = 5.0 TYPE = "trace_metric" CLIENT_REPORT_TYPE = "trace_metric" @@ -96,7 +94,7 @@ def _reset_thread_state(self) -> None: def _parse_limit(self, rate: float) -> int: # Never go below 1, so we always keep at least one metric per name to # extrapolate from - return min(max(1, int(rate * self.WINDOW_SECONDS)), self.MAX_BEFORE_DROP) + return min(max(1, int(rate * self.FLUSH_WAIT_TIME)), self.MAX_BEFORE_DROP) def _limit(self, name: str) -> "Optional[int]": return self._limits.get(name) @@ -163,7 +161,7 @@ def _add_to_queue(self, item: "Metric") -> bool: def _add_to_window(self, item: "Metric", limit: int) -> bool: window = self._window - if time.monotonic() - window.start >= self.WINDOW_SECONDS: + if time.monotonic() - window.start >= self.FLUSH_WAIT_TIME: self._retire_window(window) window = self._window @@ -218,7 +216,7 @@ def _flush(self, force: bool = False) -> "Optional[Envelope]": sampled_out: "list[tuple[Metric, int]]" = [] window = self._window - if force or time.monotonic() - window.start >= self.WINDOW_SECONDS: + if force or time.monotonic() - window.start >= self.FLUSH_WAIT_TIME: self._retire_window(window) for _ in range(len(self._retired)): try: diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 8d09f226bb..ff283b3dfd 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -631,31 +631,10 @@ def test_metrics_window_sampling_rate(sentry_init, capture_items, rate, expected assert len(items) == expected_count -def test_metrics_window_sampling_unlisted_names(sentry_init, capture_items): - sentry_init(_experiments={"metrics_window_sampling": {"test.hot": 0.2}}) - items = capture_items("trace_metric") - - for _ in range(200): - sentry_sdk.metrics.count("test.hot", 1) - sentry_sdk.metrics.count("test.other", 1) - - get_client().flush() - - metrics = [item.payload for item in items] - assert len(metrics) == 201 - - (hot,) = [metric for metric in metrics if metric["name"] == "test.hot"] - assert hot["attributes"]["sentry.client_sample_rate"] == 1 / 200 - - other = [metric for metric in metrics if metric["name"] == "test.other"] - for metric in other: - assert "sentry.client_sample_rate" not in metric["attributes"] - - def test_metrics_window_sampling_new_window(sentry_init, capture_items, monkeypatch): # Shorten the window so that the test doesn't have to wait 5 seconds for it # to close - monkeypatch.setattr(LockFreeMetricsBatcher, "WINDOW_SECONDS", 1.0) + monkeypatch.setattr(LockFreeMetricsBatcher, "FLUSH_WAIT_TIME", 1.0) sentry_init(_experiments={"metrics_window_sampling": {"test.counter": 10}}) items = capture_items("trace_metric") @@ -675,23 +654,6 @@ def test_metrics_window_sampling_new_window(sentry_init, capture_items, monkeypa assert metric["attributes"]["sentry.client_sample_rate"] == 1 / 3 -def test_metrics_window_sampling_does_not_cap_names(sentry_init, capture_items): - sentry_init( - _experiments={ - "metrics_window_sampling": {f"test.counter{i}": 2 for i in range(600)} - } - ) - items = capture_items("trace_metric") - - for i in range(600): - sentry_sdk.metrics.count(f"test.counter{i}", 1) - sentry_sdk.metrics.count(f"test.counter{i}", 1) - - get_client().flush() - - assert len(items) == 1200 - - def test_metrics_window_sampling_drops_after_max_reached( sentry_init, capture_items, capture_record_lost_event_calls, monkeypatch ): @@ -724,29 +686,3 @@ def test_metrics_window_sampling_drops_after_max_reached( assert data_category == "trace_metric" assert quantity == 1 assert item.payload.json["items"][0]["name"] == "test.c" - - -def test_metrics_window_sampling_after_before_send_metric(sentry_init, capture_items): - before_send_metric_calls = 0 - - def _before_metric(record, hint): - nonlocal before_send_metric_calls - before_send_metric_calls += 1 - return record - - sentry_init( - before_send_metric=_before_metric, - _experiments={"metrics_window_sampling": {"test.counter": 2}}, - ) - items = capture_items("trace_metric") - - for _ in range(200): - sentry_sdk.metrics.count("test.counter", 1) - - get_client().flush() - - # Sampling happens after before_send_metric, so it still sees every metric - assert before_send_metric_calls == 200 - - metrics = [item.payload for item in items] - assert len(metrics) == 10 From c626961bcfcb3ec908f213071767555ce0f01759 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 18 Nov 2026 21:06:51 +0100 Subject: [PATCH 08/20] remove indirection and tests --- sentry_sdk/_lock_free_metrics_batcher.py | 17 +++----- tests/test_metrics.py | 55 ++++++++---------------- 2 files changed, 25 insertions(+), 47 deletions(-) diff --git a/sentry_sdk/_lock_free_metrics_batcher.py b/sentry_sdk/_lock_free_metrics_batcher.py index 137715e41c..418884c8e8 100644 --- a/sentry_sdk/_lock_free_metrics_batcher.py +++ b/sentry_sdk/_lock_free_metrics_batcher.py @@ -74,8 +74,11 @@ def __init__( self._capture_func = capture_func self._record_lost_func = record_lost_func + # Metric name -> metrics kept per window. Never below 1, so there's + # always at least one metric per name to extrapolate from self._limits = { - name: self._parse_limit(rate) for name, rate in window_sampling.items() + name: min(max(1, int(rate * self.FLUSH_WAIT_TIME)), self.MAX_BEFORE_DROP) + for name, rate in window_sampling.items() } self._reset_thread_state() @@ -91,14 +94,6 @@ def _reset_thread_state(self) -> None: # Holds the claim of the one thread that gets to start the flusher. self._flusher_claim: "dict[str, object]" = {} - def _parse_limit(self, rate: float) -> int: - # Never go below 1, so we always keep at least one metric per name to - # extrapolate from - return min(max(1, int(rate * self.FLUSH_WAIT_TIME)), self.MAX_BEFORE_DROP) - - def _limit(self, name: str) -> "Optional[int]": - return self._limits.get(name) - def _ensure_thread(self) -> bool: """For forking processes we might need to restart this thread. This ensures that our process actually has that thread running. @@ -139,7 +134,7 @@ def add(self, item: "Metric") -> None: if not self._ensure_thread(): return None - limit = self._limit(item["name"]) + limit = self._limits.get(item["name"]) if limit is None: kept = self._add_to_queue(item) else: @@ -254,7 +249,7 @@ def _harvest( sampled_out = [] for name, reservoirs in by_name.items(): - limit = self._limit(name) + limit = self._limits.get(name) if limit is None: continue diff --git a/tests/test_metrics.py b/tests/test_metrics.py index ff283b3dfd..0b233b0744 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -1,4 +1,3 @@ -import time from unittest import mock import pytest @@ -576,11 +575,20 @@ def before_send_metric(metric, _): get_client().flush() +@pytest.mark.parametrize( + "rate,expected_count", + [ + (2, 10), + (4, 20), + (0.2, 1), + (0.05, 1), + ], +) def test_metrics_window_sampling_basics( - sentry_init, capture_items, capture_record_lost_event_calls + sentry_init, capture_items, capture_record_lost_event_calls, rate, expected_count ): sentry_init( - _experiments={"metrics_window_sampling": {"test.hot": 2, "test.cold": 2}} + _experiments={"metrics_window_sampling": {"test.hot": rate, "test.cold": 2}} ) items = capture_items("trace_metric") record_lost_event_calls = capture_record_lost_event_calls() @@ -593,58 +601,33 @@ def test_metrics_window_sampling_basics( get_client().flush() metrics = [item.payload for item in items] - assert len(metrics) == 13 hot = [metric for metric in metrics if metric["name"] == "test.hot"] - assert len(hot) == 10 + assert len(hot) == expected_count for metric in hot: - assert metric["attributes"]["sentry.client_sample_rate"] == 0.05 + assert metric["attributes"]["sentry.client_sample_rate"] == expected_count / 200 + # Under its limit, so nothing is sampled out cold = [metric for metric in metrics if metric["name"] == "test.cold"] assert len(cold) == 3 for metric in cold: assert "sentry.client_sample_rate" not in metric["attributes"] assert record_lost_event_calls == [ - ("sample_rate", "trace_metric", None, 190), - ("sample_rate", "trace_metric_byte", None, mock.ANY), + ("sample_rate", "trace_metric", None, 200 - expected_count), ] -@pytest.mark.parametrize( - "rate,expected_count", - [ - (4, 20), - (0.2, 1), - (0.05, 1), - ], -) -def test_metrics_window_sampling_rate(sentry_init, capture_items, rate, expected_count): - sentry_init(_experiments={"metrics_window_sampling": {"test.counter": rate}}) - items = capture_items("trace_metric") - - for _ in range(200): - sentry_sdk.metrics.count("test.counter", 1) - - get_client().flush() - - assert len(items) == expected_count - - -def test_metrics_window_sampling_new_window(sentry_init, capture_items, monkeypatch): - # Shorten the window so that the test doesn't have to wait 5 seconds for it - # to close - monkeypatch.setattr(LockFreeMetricsBatcher, "FLUSH_WAIT_TIME", 1.0) - - sentry_init(_experiments={"metrics_window_sampling": {"test.counter": 10}}) +def test_metrics_window_sampling_new_window(sentry_init, capture_items): + sentry_init(_experiments={"metrics_window_sampling": {"test.counter": 2}}) items = capture_items("trace_metric") + # flush() ends the window, so each batch of 30 is sampled on its own for _ in range(30): sentry_sdk.metrics.count("test.counter", 1) - time.sleep(1.1) + get_client().flush() for _ in range(30): sentry_sdk.metrics.count("test.counter", 1) - get_client().flush() metrics = [item.payload for item in items] From 33aa0d760a2990bec0b9a4c22dcb52b8960e5dd1 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 18 Nov 2026 21:42:51 +0100 Subject: [PATCH 09/20] . --- sentry_sdk/_lock_free_metrics_batcher.py | 13 ++-- tests/test_metrics.py | 95 ++++++++++++++++++------ 2 files changed, 78 insertions(+), 30 deletions(-) diff --git a/sentry_sdk/_lock_free_metrics_batcher.py b/sentry_sdk/_lock_free_metrics_batcher.py index 418884c8e8..2669feeb68 100644 --- a/sentry_sdk/_lock_free_metrics_batcher.py +++ b/sentry_sdk/_lock_free_metrics_batcher.py @@ -144,8 +144,6 @@ def add(self, item: "Metric") -> None: self._record_lost(item) def _add_to_queue(self, item: "Metric") -> bool: - # Checking and appending isn't atomic. - # The queue can go over the limit by at most one metric per concurrently adding thread. if len(self._queue) + self._window.size() >= self.MAX_BEFORE_DROP: return False @@ -160,17 +158,18 @@ def _add_to_window(self, item: "Metric", limit: int) -> bool: self._retire_window(window) window = self._window + window_size = window.size() state = window.threads[threading.get_ident()] reservoir = state.reservoirs[item["name"]] + + if len(self._queue) + window_size >= self.MAX_BEFORE_DROP: + return False + + reservoir.seen += 1 if len(reservoir.kept) < limit: - # The size is off by at most one per adding thread. - if len(self._queue) + window.size() >= self.MAX_BEFORE_DROP: - return False state.size += 1 - reservoir.seen += 1 reservoir.kept.append(item) else: - reservoir.seen += 1 index = random.randrange(reservoir.seen) if index < limit: reservoir.kept[index] = item diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 0b233b0744..dbf0b16613 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -1,3 +1,4 @@ +import time from unittest import mock import pytest @@ -637,35 +638,83 @@ def test_metrics_window_sampling_new_window(sentry_init, capture_items): assert metric["attributes"]["sentry.client_sample_rate"] == 1 / 3 +def test_metrics_window_sampling_flushes_queue_at_max_before_flush( + sentry_init, capture_items, monkeypatch +): + """Reaching MAX_BEFORE_FLUSH sends the unsampled metrics, but leaves the + window open, so sampled metrics wait for it to end.""" + monkeypatch.setattr(LockFreeMetricsBatcher, "MAX_BEFORE_FLUSH", 5) + # Set the time-based flush and the window to something huge, so that + # neither ends during the test + monkeypatch.setattr(LockFreeMetricsBatcher, "FLUSH_WAIT_TIME", 100000) + + # The huge window would make the limit huge too, so use a tiny rate to + # keep it at 1 + sentry_init(_experiments={"metrics_window_sampling": {"test.sampled": 1e-6}}) + items = capture_items("trace_metric") + + for _ in range(3): + for _ in range(20): + sentry_sdk.metrics.count("test.sampled", 1) + for _ in range(5): + sentry_sdk.metrics.count("test.unsampled", 1) + + time.sleep(0.1) + + assert [item.payload["name"] for item in items] == ["test.unsampled"] * 5 + + # Ends the window + get_client().flush() + + (sampled,) = [ + item.payload for item in items if item.payload["name"] == "test.sampled" + ] + assert sampled["attributes"]["sentry.client_sample_rate"] == 1 / 20 + + items.clear() + + def test_metrics_window_sampling_drops_after_max_reached( sentry_init, capture_items, capture_record_lost_event_calls, monkeypatch ): - monkeypatch.setattr(LockFreeMetricsBatcher, "MAX_BEFORE_DROP", 25) + """Sampled and unsampled metrics share MAX_BEFORE_DROP, which starts over + in every window.""" + monkeypatch.setattr(LockFreeMetricsBatcher, "MAX_BEFORE_DROP", 15) - names = ("test.a", "test.b", "test.c") - sentry_init(_experiments={"metrics_window_sampling": dict.fromkeys(names, 2)}) + sentry_init(_experiments={"metrics_window_sampling": {"test.sampled": 2}}) items = capture_items("trace_metric") record_lost_event_calls = capture_record_lost_event_calls() - for name in names: + for _ in range(3): for _ in range(10): - sentry_sdk.metrics.count(name, 1) - - get_client().flush() - - metrics = [item.payload for item in items] - assert len(metrics) == 25 - - # No name went over its limit, so nothing was sampled and the only loss is - # the overflow - for metric in metrics: - assert "sentry.client_sample_rate" not in metric["attributes"] - - assert len(record_lost_event_calls) == 5 - for lost_event_call in record_lost_event_calls: - reason, data_category, item, quantity = lost_event_call + sentry_sdk.metrics.count("test.unsampled", 1) + # Gets the 5 left in the budget, the other 15 are dropped + for _ in range(20): + sentry_sdk.metrics.count("test.sampled", 1) + # The budget is full, so these are dropped + for _ in range(5): + sentry_sdk.metrics.count("test.unsampled", 1) + + # Ends the window + get_client().flush() + + metrics = [item.payload for item in items] + names = sorted(metric["name"] for metric in metrics) + assert names == ["test.sampled"] * 5 + ["test.unsampled"] * 10 + + # Dropped metrics aren't counted as seen, so nothing was sampled out + for metric in metrics: + assert "sentry.client_sample_rate" not in metric["attributes"] + + dropped = sorted( + item.payload.json["items"][0]["name"] + for reason, data_category, item, quantity in record_lost_event_calls + if reason == "queue_overflow" and data_category == "trace_metric" + ) + assert dropped == ["test.sampled"] * 15 + ["test.unsampled"] * 5 + assert not [ + call for call in record_lost_event_calls if call[0] == "sample_rate" + ] - assert reason == "queue_overflow" - assert data_category == "trace_metric" - assert quantity == 1 - assert item.payload.json["items"][0]["name"] == "test.c" + items.clear() + record_lost_event_calls.clear() From 4fdf6823e228cebe8cf79121d42fe857244cde25 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 18 Nov 2026 21:54:51 +0100 Subject: [PATCH 10/20] fix typo --- sentry_sdk/_lock_free_metrics_batcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/_lock_free_metrics_batcher.py b/sentry_sdk/_lock_free_metrics_batcher.py index f272989cbf..f39092496c 100644 --- a/sentry_sdk/_lock_free_metrics_batcher.py +++ b/sentry_sdk/_lock_free_metrics_batcher.py @@ -42,7 +42,7 @@ def __init__(self) -> None: class _Window: """ One time window in which multiple threads concurrently capture metrics. - Only the thread with the given identifier should change it's local state. + Only the thread with the given identifier should change its local state. """ __slots__ = ("start", "threads", "retire_claims") From 7ed73d55287df769bbb002f6c74e8a524e64de8b Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 18 Nov 2026 21:55:34 +0100 Subject: [PATCH 11/20] iteration clean up --- sentry_sdk/_lock_free_metrics_batcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/_lock_free_metrics_batcher.py b/sentry_sdk/_lock_free_metrics_batcher.py index f39092496c..19dab8c57c 100644 --- a/sentry_sdk/_lock_free_metrics_batcher.py +++ b/sentry_sdk/_lock_free_metrics_batcher.py @@ -223,7 +223,7 @@ def _flush(self, force: bool = False) -> "Optional[Envelope]": envelope.add_item(self._to_envelope_item(items)) self._capture_func(envelope) - for item, count in sampled_out: + for count in sampled_out.values(): self._record_lost_func( reason="sample_rate", data_category=self.CLIENT_REPORT_TYPE, From c7cc2dd163388ebb30e124396fac9009285d35e3 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 18 Nov 2026 21:58:37 +0100 Subject: [PATCH 12/20] remove indirection --- sentry_sdk/_lock_free_metrics_batcher.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/sentry_sdk/_lock_free_metrics_batcher.py b/sentry_sdk/_lock_free_metrics_batcher.py index 19dab8c57c..108b04d4e7 100644 --- a/sentry_sdk/_lock_free_metrics_batcher.py +++ b/sentry_sdk/_lock_free_metrics_batcher.py @@ -145,16 +145,14 @@ def add(self, item: "Metric") -> None: limit = self._limits.get(item["name"]) if limit is None: - self._add_to_queue(item) - else: - self._add_to_window(item, window, limit) + self._queue.append(item) + if len(self._queue) < self.MAX_BEFORE_FLUSH or self._flush_event.is_set(): + return - def _add_to_queue(self, item: "Metric") -> None: - self._queue.append(item) - if len(self._queue) < self.MAX_BEFORE_FLUSH or self._flush_event.is_set(): + self._flush_event.set() return - self._flush_event.set() + self._add_to_window(item, window, limit) def _add_to_window(self, item: "Metric", window: "_Window", limit: int) -> None: state = window.threads[threading.get_ident()] From 3119a602fbeb64dcee4d2568d3253c97af9cd5c9 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 18 Nov 2026 22:06:03 +0100 Subject: [PATCH 13/20] fix --- sentry_sdk/_lock_free_metrics_batcher.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/sentry_sdk/_lock_free_metrics_batcher.py b/sentry_sdk/_lock_free_metrics_batcher.py index 108b04d4e7..6c2279f3f2 100644 --- a/sentry_sdk/_lock_free_metrics_batcher.py +++ b/sentry_sdk/_lock_free_metrics_batcher.py @@ -199,7 +199,7 @@ def _flush(self, force: bool = False) -> "Optional[Envelope]": except IndexError: break - sampled_out: "list[tuple[Metric, int]]" = [] + sampled_out = 0 window = self._window if force or time.monotonic() - window.start >= self.FLUSH_WAIT_TIME: @@ -209,7 +209,7 @@ def _flush(self, force: bool = False) -> "Optional[Envelope]": window = self._retired.popleft() except IndexError: break - sampled_out.extend(self._harvest(window, items)) + sampled_out += self._harvest(window, items) if not items: return None @@ -221,24 +221,23 @@ def _flush(self, force: bool = False) -> "Optional[Envelope]": envelope.add_item(self._to_envelope_item(items)) self._capture_func(envelope) - for count in sampled_out.values(): + if sampled_out: self._record_lost_func( reason="sample_rate", data_category=self.CLIENT_REPORT_TYPE, - quantity=count, + quantity=sampled_out, ) return envelope - def _harvest( - self, window: "_Window", items: "list[Metric]" - ) -> "list[tuple[Metric, int]]": + def _harvest(self, window: "_Window", items: "list[Metric]") -> int: + """Uniformly gather metrics from the window and append to `items`.""" by_name: "dict[str, list[_Reservoir]]" = {} for state in window.threads.copy().values(): for name, reservoir in state.reservoirs.copy().items(): by_name.setdefault(name, []).append(reservoir) - sampled_out = [] + sampled_out = 0 for name, reservoirs in by_name.items(): limit = self._limits.get(name) if limit is None: @@ -256,7 +255,7 @@ def _harvest( rate = len(kept) / seen for item in kept: item["attributes"]["sentry.client_sample_rate"] = rate - sampled_out.append((kept[0], seen - len(kept))) + sampled_out += seen - len(kept) return sampled_out From 44d2d61d4496197a39990e6e976d78a4d003f1be Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 19 Nov 2026 09:42:38 +0100 Subject: [PATCH 14/20] iterate on tests --- tests/test_metrics.py | 102 ++++++++++++------------------------------ 1 file changed, 28 insertions(+), 74 deletions(-) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index dbf0b16613..9040635565 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -1,11 +1,10 @@ -import time +from collections import Counter from unittest import mock import pytest import sentry_sdk from sentry_sdk import get_client -from sentry_sdk._lock_free_metrics_batcher import LockFreeMetricsBatcher from sentry_sdk.consts import SPANDATA, VERSION @@ -576,20 +575,11 @@ def before_send_metric(metric, _): get_client().flush() -@pytest.mark.parametrize( - "rate,expected_count", - [ - (2, 10), - (4, 20), - (0.2, 1), - (0.05, 1), - ], -) -def test_metrics_window_sampling_basics( - sentry_init, capture_items, capture_record_lost_event_calls, rate, expected_count +def test_metrics_window_sampling( + sentry_init, capture_items, capture_record_lost_event_calls ): sentry_init( - _experiments={"metrics_window_sampling": {"test.hot": rate, "test.cold": 2}} + _experiments={"metrics_window_sampling": {"test.hot": 1.5, "test.cold": 2}} ) items = capture_items("trace_metric") record_lost_event_calls = capture_record_lost_event_calls() @@ -604,18 +594,17 @@ def test_metrics_window_sampling_basics( metrics = [item.payload for item in items] hot = [metric for metric in metrics if metric["name"] == "test.hot"] - assert len(hot) == expected_count + assert len(hot) == 7 for metric in hot: - assert metric["attributes"]["sentry.client_sample_rate"] == expected_count / 200 + assert metric["attributes"]["sentry.client_sample_rate"] == 7 / 200 - # Under its limit, so nothing is sampled out cold = [metric for metric in metrics if metric["name"] == "test.cold"] assert len(cold) == 3 for metric in cold: assert "sentry.client_sample_rate" not in metric["attributes"] assert record_lost_event_calls == [ - ("sample_rate", "trace_metric", None, 200 - expected_count), + ("sample_rate", "trace_metric", None, 193), ] @@ -623,10 +612,10 @@ def test_metrics_window_sampling_new_window(sentry_init, capture_items): sentry_init(_experiments={"metrics_window_sampling": {"test.counter": 2}}) items = capture_items("trace_metric") - # flush() ends the window, so each batch of 30 is sampled on its own for _ in range(30): sentry_sdk.metrics.count("test.counter", 1) get_client().flush() + for _ in range(30): sentry_sdk.metrics.count("test.counter", 1) get_client().flush() @@ -638,83 +627,48 @@ def test_metrics_window_sampling_new_window(sentry_init, capture_items): assert metric["attributes"]["sentry.client_sample_rate"] == 1 / 3 -def test_metrics_window_sampling_flushes_queue_at_max_before_flush( +def test_metrics_window_sampling_drops_after_max_reached( sentry_init, capture_items, monkeypatch ): - """Reaching MAX_BEFORE_FLUSH sends the unsampled metrics, but leaves the - window open, so sampled metrics wait for it to end.""" - monkeypatch.setattr(LockFreeMetricsBatcher, "MAX_BEFORE_FLUSH", 5) - # Set the time-based flush and the window to something huge, so that - # neither ends during the test - monkeypatch.setattr(LockFreeMetricsBatcher, "FLUSH_WAIT_TIME", 100000) - - # The huge window would make the limit huge too, so use a tiny rate to - # keep it at 1 - sentry_init(_experiments={"metrics_window_sampling": {"test.sampled": 1e-6}}) - items = capture_items("trace_metric") - - for _ in range(3): - for _ in range(20): - sentry_sdk.metrics.count("test.sampled", 1) - for _ in range(5): - sentry_sdk.metrics.count("test.unsampled", 1) - - time.sleep(0.1) - - assert [item.payload["name"] for item in items] == ["test.unsampled"] * 5 + sentry_init(_experiments={"metrics_window_sampling": {"test.sampled": 2}}) + batcher = sentry_sdk.get_client().metrics_batcher - # Ends the window - get_client().flush() + def no_op_flush(*args, **kwargs): + pass - (sampled,) = [ - item.payload for item in items if item.payload["name"] == "test.sampled" - ] - assert sampled["attributes"]["sentry.client_sample_rate"] == 1 / 20 + monkeypatch.setattr(batcher, "_flush", no_op_flush) - items.clear() + lost_event_calls = [] + def record_lost_event(reason, data_category=None, item=None, *, quantity=1): + lost_event_calls.append((reason, data_category, item, quantity)) -def test_metrics_window_sampling_drops_after_max_reached( - sentry_init, capture_items, capture_record_lost_event_calls, monkeypatch -): - """Sampled and unsampled metrics share MAX_BEFORE_DROP, which starts over - in every window.""" - monkeypatch.setattr(LockFreeMetricsBatcher, "MAX_BEFORE_DROP", 15) + monkeypatch.setattr(batcher, "_record_lost_func", record_lost_event) - sentry_init(_experiments={"metrics_window_sampling": {"test.sampled": 2}}) items = capture_items("trace_metric") - record_lost_event_calls = capture_record_lost_event_calls() for _ in range(3): - for _ in range(10): + for _ in range(9_995): sentry_sdk.metrics.count("test.unsampled", 1) - # Gets the 5 left in the budget, the other 15 are dropped + for _ in range(20): sentry_sdk.metrics.count("test.sampled", 1) - # The budget is full, so these are dropped + for _ in range(5): sentry_sdk.metrics.count("test.unsampled", 1) - # Ends the window - get_client().flush() + sentry_sdk.flush() metrics = [item.payload for item in items] - names = sorted(metric["name"] for metric in metrics) - assert names == ["test.sampled"] * 5 + ["test.unsampled"] * 10 - - # Dropped metrics aren't counted as seen, so nothing was sampled out - for metric in metrics: - assert "sentry.client_sample_rate" not in metric["attributes"] + names = Counter(metric["name"] for metric in metrics) + assert names == {"test.unsampled": 9_995, "test.sampled": 5} - dropped = sorted( + dropped = Counter( item.payload.json["items"][0]["name"] - for reason, data_category, item, quantity in record_lost_event_calls + for reason, data_category, item, quantity in lost_event_calls if reason == "queue_overflow" and data_category == "trace_metric" ) - assert dropped == ["test.sampled"] * 15 + ["test.unsampled"] * 5 - assert not [ - call for call in record_lost_event_calls if call[0] == "sample_rate" - ] + assert dropped == {"test.sampled": 15, "test.unsampled": 5} items.clear() - record_lost_event_calls.clear() + lost_event_calls.clear() From eb22208408bf36695ba3cbe006bb0591dec4f7f8 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 19 Nov 2026 09:45:06 +0100 Subject: [PATCH 15/20] make test intersting --- tests/test_metrics.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 9040635565..70a4bc82fd 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -612,19 +612,17 @@ def test_metrics_window_sampling_new_window(sentry_init, capture_items): sentry_init(_experiments={"metrics_window_sampling": {"test.counter": 2}}) items = capture_items("trace_metric") - for _ in range(30): - sentry_sdk.metrics.count("test.counter", 1) - get_client().flush() - - for _ in range(30): - sentry_sdk.metrics.count("test.counter", 1) - get_client().flush() + for count in (30, 50): + for _ in range(count): + sentry_sdk.metrics.count("test.counter", 1) + get_client().flush() - metrics = [item.payload for item in items] - assert len(metrics) == 20 + metrics = [item.payload for item in items] + assert len(metrics) == 10 + for metric in metrics: + assert metric["attributes"]["sentry.client_sample_rate"] == 10 / count - for metric in metrics: - assert metric["attributes"]["sentry.client_sample_rate"] == 1 / 3 + items.clear() def test_metrics_window_sampling_drops_after_max_reached( From eaf3ab69b3ee6e7642c274e1bcbf9be4d18ee220 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 19 Nov 2026 09:49:50 +0100 Subject: [PATCH 16/20] fix client options --- sentry_sdk/client.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/sentry_sdk/client.py b/sentry_sdk/client.py index a2ce2bb57b..7b830521ed 100644 --- a/sentry_sdk/client.py +++ b/sentry_sdk/client.py @@ -133,26 +133,20 @@ def _get_options(*args: "Optional[str]", **kwargs: "Any") -> "Dict[str, Any]": window_sampling = rv["_experiments"].get("metrics_window_sampling") if window_sampling is not None: - # Copy, so the options the user passed aren't changed valid_rates = {} if isinstance(window_sampling, dict): for name, rate in window_sampling.items(): - if ( - isinstance(rate, (int, float)) - and not isinstance(rate, bool) - and math.isfinite(rate) - and rate > 0 - ): + if isinstance(rate, (int, float)) and math.isfinite(rate) and rate > 0: valid_rates[name] = rate else: logger.warning( - "[Metrics] Given window sampling rate for %r is invalid. It must be a positive number. Got %r, not sampling it.", + "[Metrics] Given window sampling rate for %s is invalid. It must be a positive number. Got %s, not sampling it.", name, rate, ) else: logger.warning( - "[Metrics] metrics_window_sampling must map metric names to rates. Got %r, not sampling.", + "[Metrics] metrics_window_sampling must map metric names to rates. Got %s, not sampling.", window_sampling, ) rv["_experiments"] = { From 3f45ee54c26a12c855d257f5ddba66bff00531fb Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 19 Nov 2026 10:09:02 +0100 Subject: [PATCH 17/20] . --- tests/test_metrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 70a4bc82fd..524e727367 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -631,10 +631,10 @@ def test_metrics_window_sampling_drops_after_max_reached( sentry_init(_experiments={"metrics_window_sampling": {"test.sampled": 2}}) batcher = sentry_sdk.get_client().metrics_batcher - def no_op_flush(*args, **kwargs): + def no_op_flush_loop(*args, **kwargs): pass - monkeypatch.setattr(batcher, "_flush", no_op_flush) + monkeypatch.setattr(batcher, "_flush_loop", no_op_flush_loop) lost_event_calls = [] From cb36693de6039e09d37c3ad8d03d417d6fcef754 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 19 Nov 2026 10:12:39 +0100 Subject: [PATCH 18/20] simplify test --- tests/test_metrics.py | 45 ++++++++++++++++--------------------------- 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 524e727367..ac62a1f7be 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -625,16 +625,14 @@ def test_metrics_window_sampling_new_window(sentry_init, capture_items): items.clear() -def test_metrics_window_sampling_drops_after_max_reached( - sentry_init, capture_items, monkeypatch -): +def test_metrics_window_sampling_drops_after_max_reached(sentry_init, monkeypatch): sentry_init(_experiments={"metrics_window_sampling": {"test.sampled": 2}}) batcher = sentry_sdk.get_client().metrics_batcher - def no_op_flush_loop(*args, **kwargs): + def no_op_flush(*args, **kwargs): pass - monkeypatch.setattr(batcher, "_flush_loop", no_op_flush_loop) + monkeypatch.setattr(batcher, "_flush", no_op_flush) lost_event_calls = [] @@ -643,30 +641,21 @@ def record_lost_event(reason, data_category=None, item=None, *, quantity=1): monkeypatch.setattr(batcher, "_record_lost_func", record_lost_event) - items = capture_items("trace_metric") - - for _ in range(3): - for _ in range(9_995): - sentry_sdk.metrics.count("test.unsampled", 1) - - for _ in range(20): - sentry_sdk.metrics.count("test.sampled", 1) - - for _ in range(5): - sentry_sdk.metrics.count("test.unsampled", 1) + for _ in range(9_995): + sentry_sdk.metrics.count("test.unsampled", 1) - sentry_sdk.flush() + for _ in range(20): + sentry_sdk.metrics.count("test.sampled", 1) - metrics = [item.payload for item in items] - names = Counter(metric["name"] for metric in metrics) - assert names == {"test.unsampled": 9_995, "test.sampled": 5} + for _ in range(5): + sentry_sdk.metrics.count("test.unsampled", 1) - dropped = Counter( - item.payload.json["items"][0]["name"] - for reason, data_category, item, quantity in lost_event_calls - if reason == "queue_overflow" and data_category == "trace_metric" - ) - assert dropped == {"test.sampled": 15, "test.unsampled": 5} + for reason, data_category, _, quantity in lost_event_calls: + assert reason == "queue_overflow" + assert data_category == "trace_metric" + assert quantity == 1 - items.clear() - lost_event_calls.clear() + dropped = Counter( + item.payload.json["items"][0]["name"] for _, _, item, _ in lost_event_calls + ) + assert dropped == {"test.sampled": 15, "test.unsampled": 5} From c89e37dd7475d995e4f90fbe98a7f935272f9dcb Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 19 Nov 2026 16:43:15 +0100 Subject: [PATCH 19/20] inline _merge() --- sentry_sdk/_lock_free_metrics_batcher.py | 34 ++++++++++-------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/sentry_sdk/_lock_free_metrics_batcher.py b/sentry_sdk/_lock_free_metrics_batcher.py index e7432dcaf2..ada0dca511 100644 --- a/sentry_sdk/_lock_free_metrics_batcher.py +++ b/sentry_sdk/_lock_free_metrics_batcher.py @@ -173,7 +173,6 @@ def _retire_window(self, window: "_Window") -> None: self._retired.append(window) def flush(self) -> None: - # Explicit flushes send the current window regardless of its age. self._retire_window(self._window) super().flush() @@ -231,36 +230,31 @@ def _harvest(self, window: "_Window", items: "list[Metric]") -> int: if limit is None: continue - seen = sum(reservoir.seen for reservoir in reservoirs) - kept = self._merge(reservoirs, limit, seen) + bounds = list( + itertools.accumulate(reservoir.seen for reservoir in reservoirs) + ) + slots = [0] * len(reservoirs) + for position in random.sample(range(bounds[-1]), min(limit, bounds[-1])): + slots[bisect.bisect_right(bounds, position)] += 1 + + kept: "list[Metric]" = [] + for reservoir, count in zip(reservoirs, slots): + thread_kept = reservoir.kept[:] + kept.extend(random.sample(thread_kept, min(count, len(thread_kept)))) if not kept: continue items.extend(kept) - if seen <= len(kept): + if bounds[-1] <= len(kept): continue - rate = len(kept) / seen + rate = len(kept) / bounds[-1] for item in kept: item["attributes"]["sentry.client_sample_rate"] = rate - sampled_out += seen - len(kept) + sampled_out += bounds[-1] - len(kept) return sampled_out - @staticmethod - def _merge(reservoirs: "list[_Reservoir]", limit: int, seen: int) -> "list[Metric]": - """Draw uniform sample of at most `limit` metrics out of all `seen`.""" - bounds = list(itertools.accumulate(reservoir.seen for reservoir in reservoirs)) - slots = [0] * len(reservoirs) - for position in random.sample(range(seen), min(limit, seen)): - slots[bisect.bisect_right(bounds, position)] += 1 - - kept = [] - for reservoir, count in zip(reservoirs, slots): - thread_kept = reservoir.kept[:] - kept.extend(random.sample(thread_kept, min(count, len(thread_kept)))) - return kept - @staticmethod def _to_transport_format(item: "Metric") -> "Any": res = { From 7967d7f3754a41c4597cf2a560a9f294aa67f5c2 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 19 Nov 2026 21:53:43 +0100 Subject: [PATCH 20/20] use list.copy() for consistency --- sentry_sdk/_lock_free_metrics_batcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/_lock_free_metrics_batcher.py b/sentry_sdk/_lock_free_metrics_batcher.py index ada0dca511..c6cb51a66e 100644 --- a/sentry_sdk/_lock_free_metrics_batcher.py +++ b/sentry_sdk/_lock_free_metrics_batcher.py @@ -239,7 +239,7 @@ def _harvest(self, window: "_Window", items: "list[Metric]") -> int: kept: "list[Metric]" = [] for reservoir, count in zip(reservoirs, slots): - thread_kept = reservoir.kept[:] + thread_kept = reservoir.kept.copy() kept.extend(random.sample(thread_kept, min(count, len(thread_kept)))) if not kept: continue