diff --git a/sentry_sdk/_lock_free_metrics_batcher.py b/sentry_sdk/_lock_free_metrics_batcher.py index e7508dbe24..c6cb51a66e 100644 --- a/sentry_sdk/_lock_free_metrics_batcher.py +++ b/sentry_sdk/_lock_free_metrics_batcher.py @@ -1,6 +1,10 @@ +import bisect +import itertools import os +import random import threading -from collections import deque +import time +from collections import defaultdict, deque from datetime import datetime, timezone from typing import TYPE_CHECKING @@ -18,6 +22,41 @@ from sentry_sdk._types import Metric +class _Reservoir: + __slots__ = ("kept", "seen") + + def __init__(self) -> None: + self.kept: "list[Metric]" = [] + self.seen = 0 + + +class _ThreadState: + """The metrics collected by one thread in a given window and their count.""" + + __slots__ = ("reservoirs", "size") + + def __init__(self) -> None: + self.reservoirs: "defaultdict[str, _Reservoir]" = defaultdict(_Reservoir) + self.size = 0 + + +class _Window: + """ + One time window in which multiple threads concurrently capture metrics. + Only the thread with the given identifier should change its local state. + """ + + __slots__ = ("start", "threads", "retire_claims") + + def __init__(self) -> None: + self.start = time.monotonic() + self.threads: "defaultdict[int, _ThreadState]" = defaultdict(_ThreadState) + self.retire_claims: "list[object]" = [] + + def size(self) -> int: + return sum(state.size for state in self.threads.copy().values()) + + class LockFreeMetricsBatcher(Batcher["Metric"]): MAX_BEFORE_FLUSH = 1000 MAX_BEFORE_DROP = 10_000 @@ -31,15 +70,28 @@ def __init__( self, capture_func: "Callable[[Envelope], None]", record_lost_func: "Callable[..., None]", + window_sampling: "dict[str, float]", ) -> None: super().__init__(capture_func, record_lost_func) + + # Metric name -> metrics kept per window. Never below 1, so there's + # always at least one metric per name to extrapolate from + self._limits = { + name: min(max(1, int(rate * self.FLUSH_WAIT_TIME)), self.MAX_BEFORE_DROP) + for name, rate in window_sampling.items() + } + self._queue: "deque[Metric]" = deque() + self._window = _Window() + self._retired: "deque[_Window]" = deque() # Holds the claim of the one thread that gets to start the flusher. self._flusher_claim: "dict[str, object]" = {} def _reset_thread_state(self) -> None: super()._reset_thread_state() self._queue = deque() + self._window = _Window() + self._retired = deque() self._flusher_claim = {} def _ensure_thread(self) -> bool: @@ -76,16 +128,53 @@ def add(self, item: "Metric") -> None: if not self._ensure_thread(): return None - queue = self._queue - if len(queue) >= self.MAX_BEFORE_DROP: + window = self._window + if time.monotonic() - window.start >= self.FLUSH_WAIT_TIME: + self._retire_window(window) + window = self._window + + if len(self._queue) + window.size() >= self.MAX_BEFORE_DROP: self._record_lost(item) return None - queue.append(item) - if len(queue) < self.MAX_BEFORE_FLUSH or self._flush_event.is_set(): + limit = self._limits.get(item["name"]) + if limit is None: + self._queue.append(item) + if len(self._queue) < self.MAX_BEFORE_FLUSH or self._flush_event.is_set(): + return + + self._flush_event.set() + return + + self._add_to_window(item, window, limit) + + def _add_to_window(self, item: "Metric", window: "_Window", limit: int) -> None: + state = window.threads[threading.get_ident()] + reservoir = state.reservoirs[item["name"]] + reservoir.seen += 1 + if len(reservoir.kept) < limit: + state.size += 1 + reservoir.kept.append(item) return - self._flush_event.set() + index = random.randrange(reservoir.seen) + if index >= limit: + return + + reservoir.kept[index] = item + + def _retire_window(self, window: "_Window") -> None: + claim = object() + window.retire_claims.append(claim) + if window.retire_claims[0] is not claim: + return + + self._window = _Window() + self._retired.append(window) + + def flush(self) -> None: + self._retire_window(self._window) + super().flush() def _flush(self) -> "Optional[Envelope]": queue = self._queue @@ -97,6 +186,18 @@ def _flush(self) -> "Optional[Envelope]": except IndexError: break + sampled_out = 0 + + window = self._window + if time.monotonic() - window.start >= self.FLUSH_WAIT_TIME: + self._retire_window(window) + for _ in range(len(self._retired)): + try: + window = self._retired.popleft() + except IndexError: + break + sampled_out += self._harvest(window, items) + if not items: return None @@ -106,8 +207,54 @@ def _flush(self) -> "Optional[Envelope]": with capture_internal_exceptions(): envelope.add_item(self._to_envelope_item(items)) self._capture_func(envelope) + + if sampled_out: + self._record_lost_func( + reason="sample_rate", + data_category=self.CLIENT_REPORT_TYPE, + quantity=sampled_out, + ) + return envelope + def _harvest(self, window: "_Window", items: "list[Metric]") -> int: + """Uniformly gather metrics from the window and append to `items`.""" + by_name: "dict[str, list[_Reservoir]]" = {} + for state in window.threads.copy().values(): + for name, reservoir in state.reservoirs.copy().items(): + by_name.setdefault(name, []).append(reservoir) + + sampled_out = 0 + for name, reservoirs in by_name.items(): + limit = self._limits.get(name) + if limit is None: + continue + + bounds = list( + itertools.accumulate(reservoir.seen for reservoir in reservoirs) + ) + slots = [0] * len(reservoirs) + for position in random.sample(range(bounds[-1]), min(limit, bounds[-1])): + slots[bisect.bisect_right(bounds, position)] += 1 + + kept: "list[Metric]" = [] + for reservoir, count in zip(reservoirs, slots): + thread_kept = reservoir.kept.copy() + kept.extend(random.sample(thread_kept, min(count, len(thread_kept)))) + if not kept: + continue + + items.extend(kept) + if bounds[-1] <= len(kept): + continue + + rate = len(kept) / bounds[-1] + for item in kept: + item["attributes"]["sentry.client_sample_rate"] = rate + sampled_out += bounds[-1] - len(kept) + + return sampled_out + @staticmethod def _to_transport_format(item: "Metric") -> "Any": res = { diff --git a/sentry_sdk/client.py b/sentry_sdk/client.py index 7f84cea140..36aa114c47 100644 --- a/sentry_sdk/client.py +++ b/sentry_sdk/client.py @@ -1,3 +1,4 @@ +import math import os import random import socket @@ -9,6 +10,7 @@ from typing import TYPE_CHECKING, Dict, cast, overload from sentry_sdk._compat import check_uwsgi_thread_support +from sentry_sdk._lock_free_metrics_batcher import LockFreeMetricsBatcher from sentry_sdk._log_batcher import LogBatcher from sentry_sdk._metrics_batcher import MetricsBatcher from sentry_sdk._span_batcher import SpanBatcher @@ -128,6 +130,29 @@ def _get_options(*args: "Optional[str]", **kwargs: "Any") -> "Dict[str, Any]": rv["data_collection"] = _resolve_data_collection(rv) + window_sampling = rv["_experiments"].get("metrics_window_sampling") + if window_sampling is not None: + valid_rates = {} + if isinstance(window_sampling, dict): + for name, rate in window_sampling.items(): + if isinstance(rate, (int, float)) and math.isfinite(rate) and rate > 0: + valid_rates[name] = rate + else: + logger.warning( + "[Metrics] Given window sampling rate for %s is invalid. It must be a positive number. Got %s, not sampling it.", + name, + rate, + ) + else: + logger.warning( + "[Metrics] metrics_window_sampling must map metric names to rates. Got %s, not sampling.", + window_sampling, + ) + rv["_experiments"] = { + **rv["_experiments"], + "metrics_window_sampling": valid_rates, + } + if rv["socket_options"] and not isinstance(rv["socket_options"], list): logger.warning( "Ignoring socket_options because of unexpected format. See urllib3.HTTPConnection.socket_options for the expected format." @@ -160,7 +185,7 @@ def __init__(self, options: "Optional[Dict[str, Any]]" = None) -> None: self.transport: "Optional[Transport]" = None self.monitor: "Optional[Monitor]" = None self.log_batcher: "Optional[LogBatcher]" = None - self.metrics_batcher: "Optional[MetricsBatcher]" = None + self.metrics_batcher: "Optional[Union[MetricsBatcher, LockFreeMetricsBatcher]]" = None self.span_batcher: "Optional[SpanBatcher]" = None self.integrations: "dict[str, Integration]" = {} @@ -379,10 +404,20 @@ def _record_lost_event( record_lost_func=_record_lost_event, ) - self.metrics_batcher = MetricsBatcher( - capture_func=_capture_envelope, - record_lost_func=_record_lost_event, + window_sampling = self.options["_experiments"].get( + "metrics_window_sampling" ) + if window_sampling: + self.metrics_batcher = LockFreeMetricsBatcher( + capture_func=_capture_envelope, + record_lost_func=_record_lost_event, + window_sampling=window_sampling, + ) + else: + self.metrics_batcher = MetricsBatcher( + capture_func=_capture_envelope, + record_lost_func=_record_lost_event, + ) self.span_batcher = SpanBatcher( capture_func=_capture_envelope, diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index 953b693744..f6113192f1 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -73,6 +73,9 @@ class CompressionAlgo(Enum): "transport_num_pools": Optional[int], "transport_http2": Optional[bool], "transport_async": Optional[bool], + # Metric name -> metrics per second to keep. Names not in here + # aren't sampled + "metrics_window_sampling": Optional[Dict[str, float]], }, total=False, ) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index efc458c32a..fcf16e41a2 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -1,5 +1,6 @@ import os import sys +from collections import Counter from unittest import mock import pytest @@ -9,8 +10,9 @@ from sentry_sdk.consts import SPANDATA, VERSION -def test_metrics_basics(sentry_init, capture_items): - sentry_init() +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) +def test_metrics_basics(sentry_init, window_sampling, capture_items): + sentry_init(_experiments={"metrics_window_sampling": window_sampling}) items = capture_items("trace_metric") sentry_sdk.metrics.count("test.counter", 1) @@ -40,8 +42,14 @@ def test_metrics_basics(sentry_init, capture_items): assert metrics[2]["unit"] == "second" -def test_metrics_with_attributes(sentry_init, capture_items): - sentry_init(release="1.0.0", environment="test", server_name="test-server") +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) +def test_metrics_with_attributes(sentry_init, window_sampling, capture_items): + sentry_init( + release="1.0.0", + environment="test", + server_name="test-server", + _experiments={"metrics_window_sampling": window_sampling}, + ) items = capture_items("trace_metric") sentry_sdk.metrics.count( @@ -63,8 +71,12 @@ def test_metrics_with_attributes(sentry_init, capture_items): assert metrics[0]["attributes"]["sentry.sdk.version"] == VERSION -def test_metrics_with_user(sentry_init, capture_items): - sentry_init(data_collection={"user_info": True}) +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) +def test_metrics_with_user(sentry_init, window_sampling, capture_items): + sentry_init( + data_collection={"user_info": True}, + _experiments={"metrics_window_sampling": window_sampling}, + ) items = capture_items("trace_metric") sentry_sdk.set_user( @@ -82,8 +94,12 @@ def test_metrics_with_user(sentry_init, capture_items): assert metrics[0]["attributes"]["user.name"] == "testuser" -def test_metrics_no_user_if_user_info_off(sentry_init, capture_items): - sentry_init(data_collection={"user_info": False}) +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) +def test_metrics_no_user_if_user_info_off(sentry_init, window_sampling, capture_items): + sentry_init( + data_collection={"user_info": False}, + _experiments={"metrics_window_sampling": window_sampling}, + ) items = capture_items("trace_metric") sentry_sdk.set_user( @@ -101,8 +117,12 @@ def test_metrics_no_user_if_user_info_off(sentry_init, capture_items): assert "user.name" not in metrics[0]["attributes"] -def test_metrics_with_span(sentry_init, capture_items): - sentry_init(traces_sample_rate=1.0) +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) +def test_metrics_with_span(sentry_init, window_sampling, capture_items): + sentry_init( + traces_sample_rate=1.0, + _experiments={"metrics_window_sampling": window_sampling}, + ) items = capture_items("trace_metric") with sentry_sdk.start_span(name="test-span") as segment: @@ -118,8 +138,11 @@ def test_metrics_with_span(sentry_init, capture_items): assert metrics[0]["span_id"] == segment.span_id -def test_metrics_tracing_without_performance(sentry_init, capture_items): - sentry_init() +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) +def test_metrics_tracing_without_performance( + sentry_init, window_sampling, capture_items +): + sentry_init(_experiments={"metrics_window_sampling": window_sampling}) items = capture_items("trace_metric") with sentry_sdk.isolation_scope() as isolation_scope: @@ -139,7 +162,8 @@ def test_metrics_tracing_without_performance(sentry_init, capture_items): assert "span_id" not in metrics[0] -def test_metrics_before_send(sentry_init, capture_items): +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) +def test_metrics_before_send(sentry_init, window_sampling, capture_items): before_metric_called = False def _before_metric(record, hint): @@ -164,6 +188,7 @@ def _before_metric(record, hint): sentry_init( before_send_metric=_before_metric, + _experiments={"metrics_window_sampling": window_sampling}, ) items = capture_items("trace_metric") @@ -179,14 +204,16 @@ def _before_metric(record, hint): @pytest.mark.tests_internal_exceptions +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) def test_metrics_before_send_raises_does_not_crash_application( - sentry_init, capture_items + sentry_init, window_sampling, capture_items ): def _before_metric(record, hint): raise ValueError("before_send_metric error") sentry_init( before_send_metric=_before_metric, + _experiments={"metrics_window_sampling": window_sampling}, ) items = capture_items("trace_metric") @@ -201,13 +228,17 @@ def _before_metric(record, hint): @pytest.mark.tests_internal_exceptions +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) def test_metrics_before_send_raises_records_callback_error( - sentry_init, capture_items, capture_record_lost_event_calls + sentry_init, window_sampling, capture_items, capture_record_lost_event_calls ): def _before_metric(record, hint): raise ValueError("before_send_metric error") - sentry_init(before_send_metric=_before_metric) + sentry_init( + before_send_metric=_before_metric, + _experiments={"metrics_window_sampling": window_sampling}, + ) items = capture_items("trace_metric") record_lost_event_calls = capture_record_lost_event_calls() @@ -227,13 +258,17 @@ def _before_metric(record, hint): assert item.payload.json["items"][0]["name"] == "test.keep" +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) def test_metrics_before_send_returns_none_records_before_send( - sentry_init, capture_items, capture_record_lost_event_calls + sentry_init, window_sampling, capture_items, capture_record_lost_event_calls ): def _before_metric(record, hint): return None - sentry_init(before_send_metric=_before_metric) + sentry_init( + before_send_metric=_before_metric, + _experiments={"metrics_window_sampling": window_sampling}, + ) items = capture_items("trace_metric") record_lost_event_calls = capture_record_lost_event_calls() @@ -253,8 +288,13 @@ def _before_metric(record, hint): assert item.payload.json["items"][0]["name"] == "test.drop" -def test_transport_format(sentry_init, capture_envelopes): - sentry_init(server_name="test-server", release="1.0.0") +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) +def test_transport_format(sentry_init, window_sampling, capture_envelopes): + sentry_init( + server_name="test-server", + release="1.0.0", + _experiments={"metrics_window_sampling": window_sampling}, + ) envelopes = capture_envelopes() @@ -320,7 +360,7 @@ def test_batcher_drops_metrics(sentry_init, monkeypatch): sentry_init() client = sentry_sdk.get_client() - def no_op_flush(): + def no_op_flush(*args, **kwargs): pass monkeypatch.setattr(client.metrics_batcher, "_flush", no_op_flush) @@ -352,8 +392,11 @@ def record_lost_event(reason, data_category=None, item=None, *, quantity=1): assert item.payload.json["items"][0]["name"] == "test.counter" -def test_metric_gets_attributes_from_scopes(sentry_init, capture_items): - sentry_init() +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) +def test_metric_gets_attributes_from_scopes( + sentry_init, window_sampling, capture_items +): + sentry_init(_experiments={"metrics_window_sampling": window_sampling}) items = capture_items("trace_metric") @@ -378,8 +421,11 @@ def test_metric_gets_attributes_from_scopes(sentry_init, capture_items): assert "current.attribute" not in metric2["attributes"] -def test_metric_attributes_override_scope_attributes(sentry_init, capture_items): - sentry_init() +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) +def test_metric_attributes_override_scope_attributes( + sentry_init, window_sampling, capture_items +): + sentry_init(_experiments={"metrics_window_sampling": window_sampling}) items = capture_items("trace_metric") @@ -397,10 +443,11 @@ def test_metric_attributes_override_scope_attributes(sentry_init, capture_items) assert metric["attributes"]["temp.attribute"] == "value2" -def test_log_array_attributes(sentry_init, capture_envelopes): +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) +def test_log_array_attributes(sentry_init, window_sampling, capture_envelopes): """Test homogeneous list and tuple attributes, and fallback for inhomogeneous collections.""" - sentry_init() + sentry_init(_experiments={"metrics_window_sampling": window_sampling}) envelopes = capture_envelopes() @@ -453,7 +500,10 @@ def test_log_array_attributes(sentry_init, capture_envelopes): } -def test_attributes_preserialized_in_before_send(sentry_init, capture_items): +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) +def test_attributes_preserialized_in_before_send( + sentry_init, window_sampling, capture_items +): """We don't surface user-held references to objects in attributes.""" def before_send_metric(metric, _): @@ -464,7 +514,10 @@ def before_send_metric(metric, _): return metric - sentry_init(before_send_metric=before_send_metric) + sentry_init( + before_send_metric=before_send_metric, + _experiments={"metrics_window_sampling": window_sampling}, + ) items = capture_items("trace_metric") @@ -494,7 +547,8 @@ class Cat: assert isinstance(metric["attributes"]["dictionary"], str) -def test_array_attributes_deep_copied_in_before_send(sentry_init): +@pytest.mark.parametrize("window_sampling", [None, {"test.sampled": 2}]) +def test_array_attributes_deep_copied_in_before_send(sentry_init, window_sampling): """We don't surface user-held references to objects in attributes.""" strings = ["value1", "value2"] @@ -506,7 +560,10 @@ def before_send_metric(metric, _): return metric - sentry_init(before_send_metric=before_send_metric) + sentry_init( + before_send_metric=before_send_metric, + _experiments={"metrics_window_sampling": window_sampling}, + ) sentry_sdk.metrics.count( "test.counter", @@ -520,6 +577,92 @@ def before_send_metric(metric, _): get_client().flush() +def test_metrics_window_sampling( + sentry_init, capture_items, capture_record_lost_event_calls +): + sentry_init( + _experiments={"metrics_window_sampling": {"test.hot": 1.5, "test.cold": 2}} + ) + items = capture_items("trace_metric") + record_lost_event_calls = capture_record_lost_event_calls() + + for _ in range(200): + sentry_sdk.metrics.count("test.hot", 1) + for _ in range(3): + sentry_sdk.metrics.count("test.cold", 1) + + get_client().flush() + + metrics = [item.payload for item in items] + + hot = [metric for metric in metrics if metric["name"] == "test.hot"] + assert len(hot) == 7 + for metric in hot: + assert metric["attributes"]["sentry.client_sample_rate"] == 7 / 200 + + cold = [metric for metric in metrics if metric["name"] == "test.cold"] + assert len(cold) == 3 + for metric in cold: + assert "sentry.client_sample_rate" not in metric["attributes"] + + assert record_lost_event_calls == [ + ("sample_rate", "trace_metric", None, 193), + ] + + +def test_metrics_window_sampling_new_window(sentry_init, capture_items): + sentry_init(_experiments={"metrics_window_sampling": {"test.counter": 2}}) + items = capture_items("trace_metric") + + for count in (30, 50): + for _ in range(count): + sentry_sdk.metrics.count("test.counter", 1) + get_client().flush() + + metrics = [item.payload for item in items] + assert len(metrics) == 10 + for metric in metrics: + assert metric["attributes"]["sentry.client_sample_rate"] == 10 / count + + items.clear() + + +def test_metrics_window_sampling_drops_after_max_reached(sentry_init, monkeypatch): + sentry_init(_experiments={"metrics_window_sampling": {"test.sampled": 2}}) + batcher = sentry_sdk.get_client().metrics_batcher + + def no_op_flush(*args, **kwargs): + pass + + monkeypatch.setattr(batcher, "_flush", no_op_flush) + + lost_event_calls = [] + + def record_lost_event(reason, data_category=None, item=None, *, quantity=1): + lost_event_calls.append((reason, data_category, item, quantity)) + + monkeypatch.setattr(batcher, "_record_lost_func", record_lost_event) + + for _ in range(9_995): + sentry_sdk.metrics.count("test.unsampled", 1) + + for _ in range(20): + sentry_sdk.metrics.count("test.sampled", 1) + + for _ in range(5): + sentry_sdk.metrics.count("test.unsampled", 1) + + for reason, data_category, _, quantity in lost_event_calls: + assert reason == "queue_overflow" + assert data_category == "trace_metric" + assert quantity == 1 + + dropped = Counter( + item.payload.json["items"][0]["name"] for _, _, item, _ in lost_event_calls + ) + assert dropped == {"test.sampled": 15, "test.unsampled": 5} + + @pytest.mark.skipif( sys.platform == "win32" or not hasattr(os, "fork")