diff --git a/fixtures/stubs-for-mypy/rapidjson.pyi b/fixtures/stubs-for-mypy/rapidjson.pyi index f40ce19c07c3..453507e36ba2 100644 --- a/fixtures/stubs-for-mypy/rapidjson.pyi +++ b/fixtures/stubs-for-mypy/rapidjson.pyi @@ -2,5 +2,5 @@ from typing import Any class JSONDecodeError(ValueError): ... -def dumps(o: object) -> str: ... -def loads(s: str | bytes) -> Any: ... +def dumps(o: object, *, ensure_ascii: bool = True, allow_nan: bool = True) -> str: ... +def loads(s: str | bytes, *, allow_nan: bool = True) -> Any: ... diff --git a/src/sentry/features/temporary.py b/src/sentry/features/temporary.py index 890883ec9b9b..f1f2e2b70212 100644 --- a/src/sentry/features/temporary.py +++ b/src/sentry/features/temporary.py @@ -446,6 +446,8 @@ def register_temporary_features(manager: FeatureManager) -> None: manager.add("organizations:unified-table-controls", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=True) # Enables view hierarchy attachment scrubbing manager.add("organizations:view-hierarchy-scrubbing", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False) + # Enables symbolication for flamegraph attachments + manager.add("organizations:flamegraph-attachments", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=True) # Enable AI-powered assertion suggestions for uptime monitors manager.add("organizations:uptime-ai-assertion-suggestions", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=True) # Enable auto spam classification at User Feedback ingest time diff --git a/src/sentry/ingest/consumer/__init__.py b/src/sentry/ingest/consumer/__init__.py index e69de29bb2d1..15881956a0e1 100644 --- a/src/sentry/ingest/consumer/__init__.py +++ b/src/sentry/ingest/consumer/__init__.py @@ -0,0 +1 @@ +CACHE_TIMEOUT = 3600 diff --git a/src/sentry/ingest/consumer/processors.py b/src/sentry/ingest/consumer/processors.py index 1d633ac8160c..64ea751e3902 100644 --- a/src/sentry/ingest/consumer/processors.py +++ b/src/sentry/ingest/consumer/processors.py @@ -18,6 +18,7 @@ from sentry.event_manager import save_attachment from sentry.feedback.lib.utils import FeedbackCreationSource, is_in_feedback_denylist from sentry.feedback.usecases.ingest.userreport import Conflict, save_userreport +from sentry.ingest.consumer import CACHE_TIMEOUT from sentry.ingest.event_payload import get_event_payload_transport from sentry.ingest.types import ConsumerType from sentry.killswitches import killswitch_matches_context @@ -41,8 +42,6 @@ logger = logging.getLogger(__name__) -CACHE_TIMEOUT = 3600 - IngestMessage = Mapping[str, Any] diff --git a/src/sentry/lang/java/view_hierarchies.py b/src/sentry/lang/java/view_hierarchies.py index 92d4158c4da8..6d23c7c27809 100644 --- a/src/sentry/lang/java/view_hierarchies.py +++ b/src/sentry/lang/java/view_hierarchies.py @@ -7,7 +7,7 @@ get_attachments_for_event, store_attachments_for_event, ) -from sentry.ingest.consumer.processors import CACHE_TIMEOUT +from sentry.ingest.consumer import CACHE_TIMEOUT from sentry.models.project import Project diff --git a/src/sentry/lang/native/flamegraphs.py b/src/sentry/lang/native/flamegraphs.py new file mode 100644 index 000000000000..73dd3a5d1afb --- /dev/null +++ b/src/sentry/lang/native/flamegraphs.py @@ -0,0 +1,159 @@ +from collections.abc import Generator +from typing import Any + +import rapidjson + +from sentry import features +from sentry.attachments import ( + CachedAttachment, + MissingAttachmentChunks, + get_attachments_for_event, + store_attachments_for_event, +) +from sentry.ingest.consumer import CACHE_TIMEOUT +from sentry.models.project import Project + +# See: https://develop.sentry.dev/sdk/telemetry/attachments/#flame-graph-validation-and-limits +MAX_FLAMEGRAPH_SIZE = 10 * 1024 * 1024 +MAX_SYMBOLICATION_FRAMES = 2500 +MAX_SAMPLE_COUNT = 9007199254740991 + + +def _is_valid_flamegraph(payload: Any) -> bool: + if not isinstance(payload, dict) or payload.get("version") != "1": + return False + if not isinstance(payload.get("platform"), str): + return False + frames = payload.get("frames") + trees = payload.get("trees") + if not isinstance(frames, list) or not frames or not isinstance(trees, list) or not trees: + return False + for frame in frames: + if not isinstance(frame, dict) or not any( + isinstance(frame.get(field), str) and frame[field] + for field in ("filename", "function", "instruction_addr") + ): + return False + if "instruction_addr" in frame and not isinstance(frame["instruction_addr"], str): + return False + if "addr_mode" in frame and not isinstance(frame["addr_mode"], str): + return False + if "platform" in frame and not isinstance(frame["platform"], str): + return False + if "data" in frame and not isinstance(frame["data"], dict): + return False + debug_meta = payload.get("debug_meta", {}) + if not isinstance(debug_meta, dict): + return False + images = debug_meta.get("images", []) + if not isinstance(images, list) or not all(isinstance(image, dict) for image in images): + return False + for image in images: + if any( + image.get(field) is not None and not isinstance(image[field], str) + for field in ("debug_id", "id", "uuid") + ): + return False + for tree in trees: + if ( + not isinstance(tree, dict) + or not isinstance(tree.get("roots"), list) + or not tree["roots"] + ): + return False + nodes = list(tree["roots"]) + while nodes: + node = nodes.pop() + if not isinstance(node, dict): + return False + frame_id = node.get("frame_id") + count = node.get("sample_count") + if type(frame_id) is not int or not 0 <= frame_id < len(frames): + return False + if type(count) is not int or not 0 < count <= MAX_SAMPLE_COUNT: + return False + children = node.get("children", []) + if not isinstance(children, list) or not all( + isinstance(child, dict) for child in children + ): + return False + child_counts = [child.get("sample_count") for child in children] + if any(type(value) is not int or value <= 0 for value in child_counts): + return False + if sum(child_counts) > count: + return False + nodes.extend(children) + return True + + +def expand_inline_frames( + trees: list[dict[str, Any]], chains: dict[int, list[int]] +) -> list[dict[str, Any]]: + output = [{**tree, "roots": []} for tree in trees] + nodes = [ + (root, new_tree["roots"]) + for tree, new_tree in zip(trees, output) + for root in reversed(tree["roots"]) + ] + while nodes: + original, siblings = nodes.pop() + node = dict(original) + siblings.append(node) + children = node.pop("children", None) + current = node + for frame_id in chains.get(node["frame_id"], [])[1:]: + child = {"frame_id": frame_id, "sample_count": node["sample_count"]} + current["children"] = [child] + current = child + if children is not None: + current["children"] = [] + nodes.extend((child, current["children"]) for child in reversed(children)) + return output + + +class Flamegraphs: + def __init__(self, project: Project, data: Any): + self._project = project + self._event = data + self._attachments = list(get_attachments_for_event(data)) + + def get_flamegraphs(self) -> Generator[tuple[int, dict[str, Any]]]: + if not any(attachment.type == "event.flamegraph" for attachment in self._attachments): + return + if not features.has("organizations:flamegraph-attachments", self._project.organization): + return + for index, attachment in enumerate(self._attachments): + if attachment.type != "event.flamegraph": + continue + try: + raw = attachment.load_data(self._project) + except (MissingAttachmentChunks, FileNotFoundError): + continue + if len(raw) > MAX_FLAMEGRAPH_SIZE: + continue + try: + payload = rapidjson.loads(raw, allow_nan=False) + except (rapidjson.JSONDecodeError, UnicodeDecodeError): + continue + if not isinstance(payload, dict): + continue + state = payload.get("_symbolication") + if isinstance(state, dict) and state.get("status") == "completed": + continue + if _is_valid_flamegraph(payload): + yield index, payload + + def save(self, attachment_index: int, payload: dict[str, Any]) -> bool: + serialized = rapidjson.dumps(payload, ensure_ascii=False, allow_nan=False).encode() + if len(serialized) > MAX_FLAMEGRAPH_SIZE: + return False + attachment = self._attachments[attachment_index] + metadata = attachment.meta() + metadata.update(data=serialized, chunks=None, size=len(serialized)) + # Checkpoint only this attachment, without reading or rewriting unrelated payloads. + checkpoint = dict(self._event) + store_attachments_for_event( + self._project, checkpoint, [CachedAttachment(**metadata)], timeout=CACHE_TIMEOUT + ) + self._event["_attachments"][attachment_index] = checkpoint["_attachments"][0] + return True diff --git a/src/sentry/lang/native/processing.py b/src/sentry/lang/native/processing.py index fd3f6595c488..1db6a97aaf32 100644 --- a/src/sentry/lang/native/processing.py +++ b/src/sentry/lang/native/processing.py @@ -6,11 +6,17 @@ from typing import Any import sentry_sdk +from requests.exceptions import RequestException from symbolic.debuginfo import normalize_debug_id from symbolic.exceptions import ParseDebugIdError from sentry import options from sentry.lang.native.error import SymbolicationFailed, write_error +from sentry.lang.native.flamegraphs import ( + MAX_SYMBOLICATION_FRAMES, + Flamegraphs, + expand_inline_frames, +) from sentry.lang.native.symbolicator import FrameOrder, Symbolicator, SymbolicatorFunction from sentry.lang.native.utils import ( get_event_attachment, @@ -20,6 +26,7 @@ is_applecrashreport_event, is_minidump_event, is_native_event, + is_native_image, is_native_platform, native_images_from_data, signal_from_data, @@ -447,7 +454,102 @@ def get_frames_for_symbolication( return rv +def _get_flamegraph_stacktraces( + payload: dict[str, Any], +) -> tuple[list[dict[str, Any]], list[int], list[dict[str, Any]]]: + frame_ids = [] + stacktraces = [] + original_modules = payload.get("debug_meta", {}).get("images", []) + modules: list[dict[str, Any]] = [] + module_indices: dict[int, int] = {} + for index, image in enumerate(original_modules): + if is_native_image(image): + module_indices[index] = len(modules) + modules.append(image) + if not modules: + return [], [], [] + for frame_id, frame in enumerate(payload["frames"][:MAX_SYMBOLICATION_FRAMES]): + frames = get_frames_for_symbolication([frame], payload, original_modules, adjustment="none") + if not frames: + continue + # Resolve against the captured image table before remapping to the filtered request. + addr_mode = frames[0].get("addr_mode") + if frame.get("addr_mode", "").startswith("rel:") and addr_mode is None: + continue + if addr_mode is not None: + module_index = module_indices.get(int(addr_mode[4:])) + if module_index is None: + continue + frames[0]["addr_mode"] = f"rel:{module_index}" + frame_ids.append(frame_id) + stacktraces.append({"frames": frames}) + return modules, frame_ids, stacktraces + + +def _merge_flamegraph_frames( + payload: dict[str, Any], frame_ids: list[int], stacktraces: list[dict[str, Any]] +) -> dict[str, Any]: + output = {**payload, "frames": list(payload["frames"])} + chains = {} + for frame_id, stacktrace in zip(frame_ids, stacktraces): + original = payload["frames"][frame_id] + resolved = stacktrace.get("frames", []) + if not resolved or any( + frame.get("status") not in (None, "symbolicated") for frame in resolved + ): + continue + chain = [] + for position, frame in enumerate(resolved): + merged = dict(original) if position == 0 else {} + if "platform" in original: + merged["platform"] = original["platform"] + if "data" in merged: + merged["data"] = dict(merged["data"]) + _merge_frame(merged, frame, original.get("platform", payload["platform"])) + # Symbolicator may adjust addresses or normalize relative addressing modes. + # The diagnostic must retain its captured symbolication inputs. + merged["instruction_addr"] = original["instruction_addr"] + if "addr_mode" in original: + merged["addr_mode"] = original["addr_mode"] + else: + merged.pop("addr_mode", None) + if position == 0: + output["frames"][frame_id] = merged + chain.append(frame_id) + else: + chain.append(len(output["frames"])) + output["frames"].append(merged) + chains[frame_id] = chain + output["trees"] = expand_inline_frames(payload["trees"], chains) + output["_symbolication"] = {"status": "completed"} + return output + + def process_native_stacktraces(symbolicator: Symbolicator, data: Any) -> Any: + flamegraphs = Flamegraphs(symbolicator.project, data) + attachments_changed = False + for attachment_index, payload in flamegraphs.get_flamegraphs(): + modules, frame_ids, stacktraces = _get_flamegraph_stacktraces(payload) + if not stacktraces: + continue + try: + response = symbolicator.process_payload( + platform=payload["platform"], + stacktraces=stacktraces, + modules=modules, + frame_order=FrameOrder.caller_first, + apply_source_context=False, + ) + except (OSError, RequestException) as error: + sentry_sdk.capture_exception(error) + continue + if not response or response.get("status") != "completed": + continue + resolved_stacktraces = response.get("stacktraces", []) + if len(resolved_stacktraces) != len(frame_ids): + continue + output = _merge_flamegraph_frames(payload, frame_ids, resolved_stacktraces) + attachments_changed |= flamegraphs.save(attachment_index, output) stacktrace_infos = [ stacktrace for stacktrace in find_stacktraces_in_data(data) @@ -470,7 +572,7 @@ def process_native_stacktraces(symbolicator: Symbolicator, data: Any) -> Any: ] if not any(stacktrace["frames"] for stacktrace in stacktraces): - return + return data if attachments_changed else None signal = signal_from_data(data) @@ -639,7 +741,9 @@ def get_native_symbolication_functions( functions.append(SymbolicatorFunction.native) return functions # early return to prevent duplicates - if has_native_stacktraces(stacktraces): + if has_native_stacktraces(stacktraces) or any( + attachment.get("type") == "event.flamegraph" for attachment in data.get("_attachments", []) + ): # Run native symbolication first, to prevent sending minidump / ACR events to # symbolicator twice (after symbolication a minidump event will have native stacktraces). functions.insert(0, SymbolicatorFunction.native) diff --git a/src/sentry/reprocessing2.py b/src/sentry/reprocessing2.py index bd3bab59fa35..b6f4b8ba63dc 100644 --- a/src/sentry/reprocessing2.py +++ b/src/sentry/reprocessing2.py @@ -236,7 +236,7 @@ def pull_event_data(project_id: int, event_id: str) -> ReprocessableEvent: def reprocess_event(project_id: int, event_id: str, start_time: float) -> None: - from sentry.ingest.consumer.processors import CACHE_TIMEOUT + from sentry.ingest.consumer import CACHE_TIMEOUT from sentry.tasks.store import preprocess_event_from_reprocessing reprocessable_event = pull_event_data(project_id, event_id) diff --git a/tests/sentry/api/serializers/test_organization.py b/tests/sentry/api/serializers/test_organization.py index 05d7d5250e96..0acad0fcc50a 100644 --- a/tests/sentry/api/serializers/test_organization.py +++ b/tests/sentry/api/serializers/test_organization.py @@ -88,6 +88,11 @@ def test_simple(self) -> None: "uptime", ] + @with_feature("organizations:flamegraph-attachments") + def test_flamegraph_feature_is_exposed(self) -> None: + result = serialize(self.organization, self.user) + assert "flamegraph-attachments" in result["features"] + @mock.patch("sentry.features.batch_has") def test_organization_batch_has(self, mock_batch: mock.MagicMock) -> None: user = self.create_user() diff --git a/tests/sentry/ingest/ingest_consumer/test_ingest_consumer_processing.py b/tests/sentry/ingest/ingest_consumer/test_ingest_consumer_processing.py index d02fa90b10ec..66bc236deeff 100644 --- a/tests/sentry/ingest/ingest_consumer/test_ingest_consumer_processing.py +++ b/tests/sentry/ingest/ingest_consumer/test_ingest_consumer_processing.py @@ -41,7 +41,7 @@ from sentry.ingest.types import ConsumerType from sentry.lang.native.utils import STORE_CRASH_REPORTS_ALL from sentry.models.debugfile import create_files_from_dif_zip -from sentry.models.eventattachment import EventAttachment, PendingEventAttachment +from sentry.models.eventattachment import V2_PREFIX, EventAttachment, PendingEventAttachment from sentry.models.userreport import UserReport from sentry.objectstore import UsecaseId, get_session from sentry.services import eventstore @@ -733,6 +733,112 @@ def test_deobfuscate_view_hierarchy_with_objectstore_attachment( do_process_view_hierarchy(default_project, task_runner, use_objectstore=True) +@debug_files_test_both_backends +class TestSymbolicateFlamegraph: + @django_db_all + @requires_symbolicator + @thread_leak_allowlist(reason="django dev server", issue=97036) + def test_symbolicate_flamegraph_attachment( + self, default_project, task_runner, live_server + ) -> None: + do_process_flamegraph(default_project, task_runner, live_server) + + @django_db_all + @requires_objectstore + @requires_symbolicator + @thread_leak_allowlist(reason="django dev server", issue=97036) + def test_symbolicate_stored_flamegraph_attachment( + self, default_project, task_runner, live_server + ) -> None: + do_process_flamegraph(default_project, task_runner, live_server, use_objectstore=True) + + +def do_process_flamegraph(default_project, task_runner, live_server, use_objectstore=False): + default_project.update_option("sentry:builtin_symbol_sources", []) + event = get_normalized_event( + {"platform": "other", "message": "Flamegraph symbolication"}, default_project + ) + with zipfile.ZipFile(BytesIO(), "w") as archive: + archive.write(get_fixture_path("native", "hello.dsym"), "dSYM/hello") + create_files_from_dif_zip(archive, project=default_project) + diagnostic = { + "version": "1", + "platform": "cocoa", + "frames": [{"instruction_addr": "0x100000fa0"}], + "debug_meta": { + "images": [ + { + "type": "macho", + "debug_id": "502fc0a5-1ec1-3e47-9998-684fa139dca7", + "image_addr": "0x100000000", + "image_size": 4096, + "arch": "x86_64", + } + ] + }, + "trees": [{"roots": [{"frame_id": 0, "sample_count": 10}]}], + } + attachment_id = "ca90fb45-6dd9-40a0-a18f-8693aa621abb" + raw = orjson.dumps(diagnostic) + attachment_metadata = { + "id": attachment_id, + "name": "flamegraph.json", + "content_type": "application/json", + "attachment_type": "event.flamegraph", + "size": len(raw), + } + if use_objectstore: + session = get_session(UsecaseId.ATTACHMENTS, default_project) + stored_id = session.put(raw) + attachment_metadata["stored_id"] = stored_id + else: + process_attachment_chunk( + { + "payload": raw, + "event_id": event["event_id"], + "project_id": default_project.id, + "id": attachment_id, + "chunk_index": 0, + } + ) + attachment_metadata["chunks"] = 1 + with ( + Feature({"organizations:flamegraph-attachments": True}), + override_options({"system.url-prefix": live_server.url}), + patch("sentry.auth.system.is_internal_ip", return_value=True), + task_runner(), + ): + process_event( + ConsumerType.Events, + { + "payload": orjson.dumps(event).decode(), + "start_time": time.time(), + "event_id": event["event_id"], + "project_id": default_project.id, + "remote_addr": "127.0.0.1", + "attachments": [attachment_metadata], + }, + project=default_project, + ) + attachment = EventAttachment.objects.get( + project_id=default_project.id, event_id=event["event_id"] + ) + assert attachment.type == "event.flamegraph" + assert attachment.content_type == "application/json" + with attachment.getfile() as file: + processed = orjson.loads(file.read()) + assert processed["frames"][0]["function"] == "main" + assert processed["frames"][0]["instruction_addr"] == "0x100000fa0" + assert processed["trees"] == diagnostic["trees"] + assert processed["_symbolication"]["status"] == "completed" + assert attachment.size == len(orjson.dumps(processed)) + if use_objectstore: + assert attachment.blob_path == V2_PREFIX + stored_id + stored = session.get(stored_id) + assert stored is not None + assert orjson.loads(stored.payload.read()) == processed + + def do_process_view_hierarchy(project, task_runner, use_objectstore=False): payload = get_normalized_event( { diff --git a/tests/sentry/lang/native/test_processing.py b/tests/sentry/lang/native/test_processing.py index d6b002de0b58..453261d788a6 100644 --- a/tests/sentry/lang/native/test_processing.py +++ b/tests/sentry/lang/native/test_processing.py @@ -6,13 +6,26 @@ from __future__ import annotations import re +from copy import deepcopy from typing import Any from unittest import mock +import orjson import pytest - +import rapidjson +from requests.exceptions import ConnectionError, Timeout + +from sentry.attachments import ( + CachedAttachment, + MissingAttachmentChunks, + get_attachments_for_event, + store_attachments_for_event, +) +from sentry.lang.native.flamegraphs import _is_valid_flamegraph from sentry.lang.native.processing import ( ELECTRON_FIRST_MODULE_REWRITE_RULES, + _get_flamegraph_stacktraces, + _merge_flamegraph_frames, _merge_image, get_frames_for_symbolication, get_native_symbolication_functions, @@ -20,8 +33,11 @@ ) from sentry.lang.native.symbolicator import SymbolicatorFunction from sentry.models.eventerror import EventErrorType +from sentry.objectstore import UsecaseId, get_session from sentry.stacktraces.processing import find_stacktraces_in_data +from sentry.testutils.helpers.features import Feature from sentry.testutils.pytest.fixtures import django_db_all +from sentry.testutils.skips import requires_objectstore from sentry.utils.safe import get_path MINIDUMP_PLACEHOLDER = { @@ -94,6 +110,687 @@ def test_get_native_symbolication_functions( assert functions == expected_functions +@django_db_all +def test_process_native_flamegraph(cached_flamegraph) -> None: + data, _, symbolicator = cached_flamegraph + payload: dict[str, Any] = { + "version": "1", + "platform": "cocoa", + "debug_meta": { + "images": [ + { + "type": "macho", + "debug_id": "84a04d24-0e60-3810-a8c0-90a65e2df61a", + "image_addr": "0x100000000", + "image_size": 8192, + "arch": "arm64", + } + ] + }, + "frames": [ + {"instruction_addr": "0x100001000", "addr_mode": "abs", "custom": "retained"}, + {"instruction_addr": "0x100001200", "function": "unresolved"}, + ], + "trees": [ + { + "thread_attributed": True, + "roots": [ + { + "frame_id": 0, + "sample_count": 10, + "children": [ + { + "frame_id": 0, + "sample_count": 6, + "children": [{"frame_id": 1, "sample_count": 4}], + } + ], + } + ], + } + ], + "custom": "retained", + } + original = deepcopy(payload) + store_attachments_for_event( + symbolicator.project, + data, + [ + CachedAttachment( + type="event.flamegraph", + name="flamegraph.json", + content_type="application/json", + data=orjson.dumps(payload), + ) + ], + timeout=60, + ) + data["debug_meta"] = {"images": [{"type": "elf", "debug_id": "event-image"}]} + symbolicator.process_payload.return_value = { + "status": "completed", + "stacktraces": [ + { + "frames": [ + { + "function": "outer", + "status": "symbolicated", + "instruction_addr": "0x100000fff", + }, + { + "function": "inlined", + "status": "symbolicated", + "filename": "Source.swift", + "lineno": 42, + }, + ] + }, + {"frames": [{"status": "missing", "instruction_addr": "0x1000011ff"}]}, + ], + } + with Feature({"organizations:flamegraph-attachments": True}): + result = process_native_stacktraces(symbolicator, data) + symbolicator.process_payload.assert_called_once() + request = symbolicator.process_payload.call_args.kwargs + assert request["platform"] == "cocoa" + assert request["modules"] == payload["debug_meta"]["images"] + assert request["apply_source_context"] is False + assert request["stacktraces"] == [ + { + "frames": [ + { + "instruction_addr": "0x100001000", + "custom": "retained", + "adjust_instruction_addr": False, + } + ] + }, + { + "frames": [ + { + "instruction_addr": "0x100001200", + "function": "unresolved", + "adjust_instruction_addr": False, + } + ] + }, + ] + assert result is data + processed = orjson.loads(next(get_attachments_for_event(data)).load_data(symbolicator.project)) + assert processed["frames"][0]["function"] == "outer" + assert processed["frames"][0]["instruction_addr"] == "0x100001000" + assert processed["frames"][0]["addr_mode"] == "abs" + assert processed["frames"][1] == original["frames"][1] + assert processed["frames"][2]["function"] == "inlined" + root = processed["trees"][0]["roots"][0] + assert root["sample_count"] == 10 + assert root["children"][0]["frame_id"] == 2 + assert root["children"][0]["sample_count"] == 10 + recursive = root["children"][0]["children"][0] + assert recursive["frame_id"] == 0 + assert recursive["children"][0]["sample_count"] == 6 + assert recursive["children"][0]["children"] == [{"frame_id": 1, "sample_count": 4}] + assert processed["debug_meta"] == original["debug_meta"] + assert processed["custom"] == "retained" + + +@django_db_all +def test_flamegraph_symbolication_is_idempotent(cached_flamegraph) -> None: + data, _, symbolicator = cached_flamegraph + with Feature({"organizations:flamegraph-attachments": True}): + process_native_stacktraces(symbolicator, data) + before = next(get_attachments_for_event(data)).load_data(symbolicator.project) + assert process_native_stacktraces(symbolicator, data) is None + symbolicator.process_payload.assert_called_once() + assert next(get_attachments_for_event(data)).load_data(symbolicator.project) == before + + +def test_route_flamegraph_to_native_symbolication() -> None: + data = {"platform": "other", "_attachments": [{"type": "event.flamegraph"}]} + assert get_native_symbolication_functions(data, []) == [SymbolicatorFunction.native] + + +@pytest.fixture +def flamegraph_payload(): + return { + "version": "1", + "platform": "cocoa", + "frames": [{"instruction_addr": "0x100001000", "data": {"custom": "retained"}}], + "debug_meta": { + "images": [{"type": "macho", "debug_id": "84a04d24-0e60-3810-a8c0-90a65e2df61a"}] + }, + "trees": [{"roots": [{"frame_id": 0, "sample_count": 10}]}], + } + + +@pytest.fixture +def cached_flamegraph(default_project, flamegraph_payload): + payload = flamegraph_payload + data = { + "platform": "other", + "project": default_project.id, + "event_id": "cc3e6c2bb6b6498097f336d1e6979f4b", + } + store_attachments_for_event( + default_project, + data, + [ + CachedAttachment( + type="event.flamegraph", + name="flamegraph.json", + content_type="application/json", + data=orjson.dumps(payload), + ) + ], + timeout=60, + ) + symbolicator = mock.Mock(project=default_project) + symbolicator.process_payload.return_value = { + "status": "completed", + "stacktraces": [ + { + "frames": [ + {"status": "symbolicated", "function": "outer"}, + {"status": "symbolicated", "function": "inner"}, + ] + } + ], + } + return data, payload, symbolicator + + +@pytest.mark.parametrize( + ("field", "value"), + [("data", "invalid"), ("data", None), ("addr_mode", 1), ("platform", [])], +) +def test_flamegraph_rejects_invalid_frame_fields(flamegraph_payload, field, value) -> None: + flamegraph_payload["frames"][0][field] = value + assert not _is_valid_flamegraph(flamegraph_payload) + + +@pytest.mark.parametrize("field", ["debug_id", "id", "uuid"]) +def test_flamegraph_rejects_invalid_image_identifiers(flamegraph_payload, field) -> None: + flamegraph_payload["debug_meta"]["images"][0][field] = [] + assert not _is_valid_flamegraph(flamegraph_payload) + + +def test_flamegraph_inline_frames_do_not_inherit_captured_metadata(flamegraph_payload) -> None: + original = flamegraph_payload["frames"][0] + original.update( + filename="captured.cpp", + lineno=42, + function="captured", + addr_mode="rel:0", + platform="native", + ) + before = deepcopy(flamegraph_payload) + output = _merge_flamegraph_frames( + flamegraph_payload, + [0], + [{"frames": [{"status": "symbolicated"}, {"status": "symbolicated"}]}], + ) + assert output["frames"][0]["filename"] == "captured.cpp" + assert output["frames"][0]["data"]["custom"] == "retained" + inline = output["frames"][1] + assert inline["instruction_addr"] == original["instruction_addr"] + assert inline["addr_mode"] == "rel:0" + assert inline["platform"] == "native" + assert "filename" not in inline + assert "lineno" not in inline + assert "function" not in inline + assert "custom" not in inline["data"] + assert flamegraph_payload == before + + +@django_db_all +@pytest.mark.parametrize("error", [TimeoutError(), Timeout(), ConnectionError()]) +def test_flamegraph_transport_error_does_not_block_event_symbolication( + cached_flamegraph, error +) -> None: + data, payload, symbolicator = cached_flamegraph + data["platform"] = "native" + data["exception"] = {"values": [deepcopy(NATIVE_EXCEPTION)]} + symbolicator.process_payload.side_effect = [ + error, + { + "status": "completed", + "modules": [], + "stacktraces": [ + {"frames": [{"original_index": 0, "function": "main", "status": "symbolicated"}]} + ], + }, + ] + with Feature({"organizations:flamegraph-attachments": True}): + assert process_native_stacktraces(symbolicator, data) is data + assert symbolicator.process_payload.call_count == 2 + assert data["exception"]["values"][0]["stacktrace"]["frames"][0]["function"] == "main" + assert next(get_attachments_for_event(data)).load_data(symbolicator.project) == orjson.dumps( + payload + ) + + +@django_db_all +@pytest.mark.parametrize("error", [MissingAttachmentChunks(), FileNotFoundError()]) +def test_flamegraph_unavailable_data_does_not_block_event_symbolication( + cached_flamegraph, error +) -> None: + data, _, symbolicator = cached_flamegraph + data["platform"] = "native" + data["exception"] = {"values": [deepcopy(NATIVE_EXCEPTION)]} + symbolicator.process_payload.return_value = { + "status": "completed", + "modules": [], + "stacktraces": [ + {"frames": [{"original_index": 0, "function": "main", "status": "symbolicated"}]} + ], + } + with ( + Feature({"organizations:flamegraph-attachments": True}), + mock.patch.object(CachedAttachment, "load_data", side_effect=error), + ): + assert process_native_stacktraces(symbolicator, data) is data + assert data["exception"]["values"][0]["stacktrace"]["frames"][0]["function"] == "main" + + +@django_db_all +def test_flamegraph_invalid_frame_does_not_block_event_symbolication(cached_flamegraph) -> None: + data, payload, symbolicator = cached_flamegraph + payload["frames"][0]["data"] = "invalid" + raw = orjson.dumps(payload) + store_attachments_for_event( + symbolicator.project, + data, + [CachedAttachment(type="event.flamegraph", data=raw)], + timeout=60, + ) + data["platform"] = "native" + data["exception"] = {"values": [deepcopy(NATIVE_EXCEPTION)]} + symbolicator.process_payload.return_value = { + "status": "completed", + "modules": [], + "stacktraces": [ + {"frames": [{"original_index": 0, "function": "main", "status": "symbolicated"}]} + ], + } + with Feature({"organizations:flamegraph-attachments": True}): + assert process_native_stacktraces(symbolicator, data) is data + assert data["exception"]["values"][0]["stacktrace"]["frames"][0]["function"] == "main" + assert next(get_attachments_for_event(data)).load_data(symbolicator.project) == raw + + +@django_db_all +def test_flamegraph_feature_disabled(cached_flamegraph) -> None: + data, payload, symbolicator = cached_flamegraph + with Feature({"organizations:flamegraph-attachments": False}): + assert process_native_stacktraces(symbolicator, data) is None + symbolicator.process_payload.assert_not_called() + assert next(get_attachments_for_event(data)).load_data(symbolicator.project) == orjson.dumps( + payload + ) + + +@django_db_all +@pytest.mark.parametrize("depth", [100, 600]) +def test_flamegraph_deep_inline_expansion(cached_flamegraph, depth) -> None: + data, payload, symbolicator = cached_flamegraph + node = {"frame_id": 0, "sample_count": 10, "custom": "retained"} + for _ in range(depth - 1): + node = {"frame_id": 0, "sample_count": 10, "children": [node], "custom": "retained"} + payload["trees"][0]["roots"] = [node] + raw = rapidjson.dumps(payload).encode() + store_attachments_for_event( + symbolicator.project, + data, + [CachedAttachment(type="event.flamegraph", data=raw)], + timeout=60, + ) + with Feature({"organizations:flamegraph-attachments": True}): + assert process_native_stacktraces(symbolicator, data) is data + processed = rapidjson.loads( + next(get_attachments_for_event(data)).load_data(symbolicator.project) + ) + assert processed["frames"][0]["function"] == "outer" + assert processed["frames"][1]["function"] == "inner" + node = processed["trees"][0]["roots"][0] + for _ in range(depth - 1): + assert node["frame_id"] == 0 + assert node["sample_count"] == 10 + assert node["custom"] == "retained" + inline = node["children"][0] + assert inline["frame_id"] == 1 + assert inline["sample_count"] == 10 + node = inline["children"][0] + assert node == { + "frame_id": 0, + "sample_count": 10, + "custom": "retained", + "children": [{"frame_id": 1, "sample_count": 10}], + } + + +@django_db_all +def test_flamegraph_large_frame_metadata_is_not_duplicated(cached_flamegraph) -> None: + data, payload, symbolicator = cached_flamegraph + payload["frames"][0]["custom"] = "x" * (6 * 1024 * 1024) + raw = orjson.dumps(payload) + store_attachments_for_event( + symbolicator.project, + data, + [CachedAttachment(type="event.flamegraph", data=raw)], + timeout=60, + ) + with Feature({"organizations:flamegraph-attachments": True}): + assert process_native_stacktraces(symbolicator, data) is data + processed = orjson.loads(next(get_attachments_for_event(data)).load_data(symbolicator.project)) + assert processed["frames"][0]["custom"] == payload["frames"][0]["custom"] + assert "custom" not in processed["frames"][1] + assert processed["_symbolication"]["status"] == "completed" + + +@django_db_all +def test_flamegraph_oversized_rewrite_preserves_original(cached_flamegraph) -> None: + data, payload, symbolicator = cached_flamegraph + raw = orjson.dumps(payload) + with ( + Feature({"organizations:flamegraph-attachments": True}), + mock.patch("sentry.lang.native.flamegraphs.MAX_FLAMEGRAPH_SIZE", len(raw)), + ): + assert process_native_stacktraces(symbolicator, data) is None + symbolicator.process_payload.assert_called_once() + assert next(get_attachments_for_event(data)).load_data(symbolicator.project) == raw + + +@pytest.mark.parametrize("addr_mode", ["rel:1", "rel:84a04d24-0e60-3810-a8c0-90a65e2df61a"]) +def test_flamegraph_relative_address_uses_filtered_module_index( + flamegraph_payload, addr_mode +) -> None: + flamegraph_payload["debug_meta"]["images"].insert(0, {"type": "macho", "image_addr": "0x2000"}) + flamegraph_payload["frames"][0]["addr_mode"] = addr_mode + original = deepcopy(flamegraph_payload) + modules, frame_ids, stacktraces = _get_flamegraph_stacktraces(flamegraph_payload) + assert modules == original["debug_meta"]["images"][1:] + assert frame_ids == [0] + assert stacktraces[0]["frames"][0]["addr_mode"] == "rel:0" + processed = _merge_flamegraph_frames( + flamegraph_payload, + frame_ids, + [{"frames": [{"status": "symbolicated", "function": "main"}]}], + ) + assert processed["frames"][0]["function"] == "main" + assert processed["frames"][0]["addr_mode"] == addr_mode + assert processed["frames"][0]["instruction_addr"] == "0x100001000" + assert processed["debug_meta"] == original["debug_meta"] + assert flamegraph_payload == original + + +@pytest.mark.parametrize( + "addr_mode", ["rel:0", "rel:2", "rel:00000000-0000-0000-0000-000000000001"] +) +def test_flamegraph_relative_address_without_matching_image_is_preserved( + flamegraph_payload, addr_mode +) -> None: + flamegraph_payload["debug_meta"]["images"].insert(0, {"type": "macho", "image_addr": "0x2000"}) + flamegraph_payload["frames"][0]["addr_mode"] = addr_mode + original = deepcopy(flamegraph_payload) + modules, frame_ids, stacktraces = _get_flamegraph_stacktraces(flamegraph_payload) + assert modules == original["debug_meta"]["images"][1:] + assert frame_ids == [] + assert stacktraces == [] + assert flamegraph_payload == original + + +@django_db_all +def test_flamegraph_frame_limit(cached_flamegraph) -> None: + data, payload, symbolicator = cached_flamegraph + payload["frames"] *= 2501 + store_attachments_for_event( + symbolicator.project, + data, + [ + CachedAttachment( + type="event.flamegraph", + data=orjson.dumps(payload), + ) + ], + timeout=60, + ) + symbolicator.process_payload.return_value = { + "status": "completed", + "stacktraces": [{"frames": []}] * 2500, + } + with Feature({"organizations:flamegraph-attachments": True}): + process_native_stacktraces(symbolicator, data) + assert len(symbolicator.process_payload.call_args.kwargs["stacktraces"]) == 2500 + processed = orjson.loads(next(get_attachments_for_event(data)).load_data(symbolicator.project)) + assert processed["frames"] == payload["frames"] + assert processed["trees"] == payload["trees"] + + +@django_db_all +@pytest.mark.parametrize( + "response", + [{"status": "failed"}, {"status": "pending"}, {"status": "completed", "stacktraces": []}], +) +def test_flamegraph_failure_preserves_attachment(cached_flamegraph, response) -> None: + data, payload, symbolicator = cached_flamegraph + symbolicator.process_payload.return_value = response + with Feature({"organizations:flamegraph-attachments": True}): + assert process_native_stacktraces(symbolicator, data) is None + assert next(get_attachments_for_event(data)).load_data(symbolicator.project) == orjson.dumps( + payload + ) + + +@django_db_all +@pytest.mark.parametrize( + "raw", [b"invalid json", b"\xff", b"null", b'{"version":"2"}', b" " * (10 * 1024 * 1024 + 1)] +) +def test_flamegraph_invalid_attachment(cached_flamegraph, raw) -> None: + data, _, symbolicator = cached_flamegraph + store_attachments_for_event( + symbolicator.project, + data, + [ + CachedAttachment( + type="event.flamegraph", + data=raw, + ) + ], + timeout=60, + ) + with Feature({"organizations:flamegraph-attachments": True}): + assert process_native_stacktraces(symbolicator, data) is None + symbolicator.process_payload.assert_not_called() + assert next(get_attachments_for_event(data)).load_data(symbolicator.project) == raw + + +@django_db_all +@pytest.mark.parametrize( + "root", + [ + {"frame_id": 1, "sample_count": 10}, + {"frame_id": True, "sample_count": 10}, + {"frame_id": 0, "sample_count": 0}, + {"frame_id": 0, "sample_count": True}, + {"frame_id": 0, "sample_count": 9007199254740992}, + {"frame_id": 0, "sample_count": 10, "children": "invalid"}, + {"frame_id": 0, "sample_count": 10, "children": [{"frame_id": 0, "sample_count": 11}]}, + ], +) +def test_flamegraph_invalid_tree_is_not_symbolicated(cached_flamegraph, root) -> None: + data, payload, symbolicator = cached_flamegraph + payload["trees"][0]["roots"] = [root] + raw = orjson.dumps(payload) + store_attachments_for_event( + symbolicator.project, + data, + [CachedAttachment(type="event.flamegraph", data=raw)], + timeout=60, + ) + with Feature({"organizations:flamegraph-attachments": True}): + assert process_native_stacktraces(symbolicator, data) is None + symbolicator.process_payload.assert_not_called() + assert next(get_attachments_for_event(data)).load_data(symbolicator.project) == raw + + +@django_db_all +def test_flamegraph_large_payload_is_not_duplicated(cached_flamegraph) -> None: + data, payload, symbolicator = cached_flamegraph + payload["frames"][0]["custom"] = "x" * (6 * 1024 * 1024) + store_attachments_for_event( + symbolicator.project, + data, + [CachedAttachment(type="event.flamegraph", data=orjson.dumps(payload))], + timeout=60, + ) + symbolicator.process_payload.return_value = { + "status": "completed", + "stacktraces": [{"frames": [{"status": "symbolicated", "function": "main"}]}], + } + with Feature({"organizations:flamegraph-attachments": True}): + assert process_native_stacktraces(symbolicator, data) is data + attachment = next(get_attachments_for_event(data)) + processed = orjson.loads(attachment.load_data(symbolicator.project)) + assert processed["frames"][0]["function"] == "main" + assert processed["frames"][0]["custom"] == payload["frames"][0]["custom"] + assert processed["trees"] == payload["trees"] + assert attachment.size <= 10 * 1024 * 1024 + + +@django_db_all +@requires_objectstore +def test_flamegraph_objectstore_attachment(cached_flamegraph) -> None: + data, payload, symbolicator = cached_flamegraph + session = get_session(UsecaseId.ATTACHMENTS, symbolicator.project) + stored_id = session.put(orjson.dumps(payload)) + store_attachments_for_event( + symbolicator.project, + data, + [ + CachedAttachment( + type="event.flamegraph", + name="flamegraph.json", + content_type="application/json", + stored_id=stored_id, + size=len(orjson.dumps(payload)), + retention_days=7, + ) + ], + timeout=60, + ) + with Feature({"organizations:flamegraph-attachments": True}): + process_native_stacktraces(symbolicator, data) + attachment = next(get_attachments_for_event(data)) + assert attachment.stored_id == stored_id + assert attachment.retention_days == 7 + stored = session.get(stored_id) + assert stored is not None + processed = orjson.loads(stored.payload.read()) + assert processed["frames"][0]["function"] == "outer" + assert processed["frames"][1]["function"] == "inner" + assert attachment.size == len(orjson.dumps(processed)) + + +@pytest.fixture +def multiple_flamegraphs(cached_flamegraph): + data, payload, symbolicator = cached_flamegraph + second = deepcopy(payload) + second["debug_meta"]["images"][0]["debug_id"] = "502fc0a5-1ec1-3e47-9998-684fa139dca7" + store_attachments_for_event( + symbolicator.project, + data, + [ + CachedAttachment(type="event.flamegraph", data=orjson.dumps(payload)), + CachedAttachment(type="event.attachment", name="other.txt", data=b"unchanged"), + CachedAttachment(type="event.flamegraph", data=orjson.dumps(second)), + ], + timeout=60, + ) + return data, payload, second, symbolicator + + +@django_db_all +def test_flamegraph_multiple_attachments(multiple_flamegraphs) -> None: + data, first, second, symbolicator = multiple_flamegraphs + with Feature({"organizations:flamegraph-attachments": True}): + assert process_native_stacktraces(symbolicator, data) is data + assert symbolicator.process_payload.call_count == 2 + first_request, second_request = symbolicator.process_payload.call_args_list + assert first_request.kwargs["modules"] == first["debug_meta"]["images"] + assert second_request.kwargs["modules"] == second["debug_meta"]["images"] + first_attachment, other_attachment, second_attachment = get_attachments_for_event(data) + first_processed = orjson.loads(first_attachment.load_data(symbolicator.project)) + second_processed = orjson.loads(second_attachment.load_data(symbolicator.project)) + assert first_processed["frames"][0]["function"] == "outer" + assert second_processed["frames"][0]["function"] == "outer" + assert first_processed["_symbolication"]["status"] == "completed" + assert second_processed["_symbolication"]["status"] == "completed" + assert other_attachment.load_data(symbolicator.project) == b"unchanged" + + +@django_db_all +@pytest.mark.parametrize("missing_type", ["event.attachment", "event.flamegraph"]) +def test_flamegraph_checkpoint_does_not_read_missing_attachments( + multiple_flamegraphs, missing_type +) -> None: + data, _, _, symbolicator = multiple_flamegraphs + data["_attachments"][1]["type"] = missing_type + _, missing, _ = get_attachments_for_event(data) + missing.delete() + original_metadata = dict(data["_attachments"][1]) + with Feature({"organizations:flamegraph-attachments": True}): + assert process_native_stacktraces(symbolicator, data) is data + first, missing, second = get_attachments_for_event(data) + assert orjson.loads(first.load_data(symbolicator.project))["frames"][0]["function"] == "outer" + assert orjson.loads(second.load_data(symbolicator.project))["frames"][0]["function"] == "outer" + assert missing.meta() == original_metadata + with pytest.raises(MissingAttachmentChunks): + missing.load_data(symbolicator.project) + + +@django_db_all +@requires_objectstore +def test_flamegraph_checkpoints_do_not_rewrite_processed_objects(multiple_flamegraphs) -> None: + data, first, second, symbolicator = multiple_flamegraphs + session = get_session(UsecaseId.ATTACHMENTS, symbolicator.project) + first_id = session.put(orjson.dumps(first)) + second_id = session.put(orjson.dumps(second)) + store_attachments_for_event( + symbolicator.project, + data, + [ + CachedAttachment(type="event.flamegraph", stored_id=first_id), + CachedAttachment(type="event.flamegraph", stored_id=second_id), + ], + timeout=60, + ) + with ( + Feature({"organizations:flamegraph-attachments": True}), + mock.patch("sentry.attachments.base.get_session", return_value=session), + mock.patch.object(session, "put", wraps=session.put) as put, + ): + assert process_native_stacktraces(symbolicator, data) is data + assert put.call_count == 2 + assert [call.kwargs["key"] for call in put.call_args_list] == [first_id, second_id] + + +@django_db_all +def test_flamegraph_checkpoint_survives_later_timeout(multiple_flamegraphs) -> None: + data, _, second, symbolicator = multiple_flamegraphs + symbolicator.process_payload.side_effect = [ + symbolicator.process_payload.return_value, + TimeoutError(), + ] + with Feature({"organizations:flamegraph-attachments": True}): + assert process_native_stacktraces(symbolicator, data) is data + assert symbolicator.process_payload.call_count == 2 + first_attachment, other_attachment, second_attachment = get_attachments_for_event(data) + first_processed = orjson.loads(first_attachment.load_data(symbolicator.project)) + assert first_processed["frames"][0]["function"] == "outer" + assert first_processed["_symbolication"]["status"] == "completed" + assert other_attachment.load_data(symbolicator.project) == b"unchanged" + assert second_attachment.load_data(symbolicator.project) == orjson.dumps(second) + + def test_merge_symbolicator_image_empty() -> None: data: dict[str, Any] = {} _merge_image({}, {}, None, data)