From 82a93632d6efde1409478a925d2874f483c2450e Mon Sep 17 00:00:00 2001 From: Philip Niedertscheider Date: Mon, 5 Oct 2026 17:08:57 +0200 Subject: [PATCH 1/7] feat(native): Symbolicate flamegraph attachments Process version-1 flamegraphs through the existing native symbolication pipeline behind a feature flag. Preserve captured addresses and original tables, expand inline frames, and checkpoint attachment updates. Share native frame helpers and cover ingestion, storage backends, validation, reprocessing, and interrupted multi-attachment processing. --- src/sentry/features/temporary.py | 2 + src/sentry/ingest/consumer/__init__.py | 1 + src/sentry/ingest/consumer/processors.py | 3 +- src/sentry/lang/java/view_hierarchies.py | 2 +- src/sentry/lang/native/flamegraphs.py | 211 ++++++++ src/sentry/lang/native/frames.py | 136 ++++++ src/sentry/lang/native/processing.py | 167 ++----- src/sentry/reprocessing2.py | 2 +- .../test_ingest_consumer_processing.py | 107 +++- tests/sentry/lang/native/test_processing.py | 458 +++++++++++++++++- 10 files changed, 945 insertions(+), 144 deletions(-) create mode 100644 src/sentry/lang/native/flamegraphs.py create mode 100644 src/sentry/lang/native/frames.py diff --git a/src/sentry/features/temporary.py b/src/sentry/features/temporary.py index b5a892b78cba..bb8be41c2768 100644 --- a/src/sentry/features/temporary.py +++ b/src/sentry/features/temporary.py @@ -423,6 +423,8 @@ def register_temporary_features(manager: FeatureManager) -> None: manager.add("organizations:transaction-name-normalize", OrganizationFeature, FeatureHandlerStrategy.INTERNAL, default=True, api_expose=False) # Enables view hierarchy attachment scrubbing manager.add("organizations:view-hierarchy-scrubbing", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False) + # Enables symbolication for flamegraph attachments + manager.add("organizations:flamegraph-attachments", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False) # Enable AI-powered assertion suggestions for uptime monitors manager.add("organizations:uptime-ai-assertion-suggestions", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=True) # Enable auto spam classification at User Feedback ingest time diff --git a/src/sentry/ingest/consumer/__init__.py b/src/sentry/ingest/consumer/__init__.py index e69de29bb2d1..15881956a0e1 100644 --- a/src/sentry/ingest/consumer/__init__.py +++ b/src/sentry/ingest/consumer/__init__.py @@ -0,0 +1 @@ +CACHE_TIMEOUT = 3600 diff --git a/src/sentry/ingest/consumer/processors.py b/src/sentry/ingest/consumer/processors.py index 888aa0941941..c484bd718cce 100644 --- a/src/sentry/ingest/consumer/processors.py +++ b/src/sentry/ingest/consumer/processors.py @@ -16,6 +16,7 @@ from sentry.event_manager import save_attachment from sentry.feedback.lib.utils import FeedbackCreationSource, is_in_feedback_denylist from sentry.feedback.usecases.ingest.userreport import Conflict, save_userreport +from sentry.ingest.consumer import CACHE_TIMEOUT from sentry.ingest.types import ConsumerType from sentry.killswitches import killswitch_matches_context from sentry.models.organization import Organization @@ -39,8 +40,6 @@ logger = logging.getLogger(__name__) -CACHE_TIMEOUT = 3600 - IngestMessage = Mapping[str, Any] diff --git a/src/sentry/lang/java/view_hierarchies.py b/src/sentry/lang/java/view_hierarchies.py index 92d4158c4da8..6d23c7c27809 100644 --- a/src/sentry/lang/java/view_hierarchies.py +++ b/src/sentry/lang/java/view_hierarchies.py @@ -7,7 +7,7 @@ get_attachments_for_event, store_attachments_for_event, ) -from sentry.ingest.consumer.processors import CACHE_TIMEOUT +from sentry.ingest.consumer import CACHE_TIMEOUT from sentry.models.project import Project diff --git a/src/sentry/lang/native/flamegraphs.py b/src/sentry/lang/native/flamegraphs.py new file mode 100644 index 000000000000..421d2366fc32 --- /dev/null +++ b/src/sentry/lang/native/flamegraphs.py @@ -0,0 +1,211 @@ +from collections.abc import Generator +from copy import deepcopy +from typing import Any + +import orjson + +from sentry import features +from sentry.attachments import ( + CachedAttachment, + get_attachments_for_event, + store_attachments_for_event, +) +from sentry.ingest.consumer import CACHE_TIMEOUT +from sentry.lang.native.frames import get_frames_for_symbolication, merge_frame +from sentry.lang.native.utils import is_native_platform +from sentry.models.project import Project + +MAX_FLAMEGRAPH_SIZE = 10 * 1024 * 1024 +MAX_SYMBOLICATION_FRAMES = 2500 +MAX_SAMPLE_COUNT = 9007199254740991 + + +def _is_valid_flamegraph(payload: Any) -> bool: + if not isinstance(payload, dict) or payload.get("version") != "1": + return False + if not isinstance(payload.get("platform"), str): + return False + frames = payload.get("frames") + trees = payload.get("trees") + if not isinstance(frames, list) or not frames or not isinstance(trees, list) or not trees: + return False + for frame in frames: + if not isinstance(frame, dict) or not any( + isinstance(frame.get(field), str) and frame[field] + for field in ("filename", "function", "instruction_addr") + ): + return False + if "instruction_addr" in frame and not isinstance(frame["instruction_addr"], str): + return False + if "addr_mode" in frame and not isinstance(frame["addr_mode"], str): + return False + if "platform" in frame and not isinstance(frame["platform"], str): + return False + debug_meta = payload.get("debug_meta", {}) + if not isinstance(debug_meta, dict): + return False + images = debug_meta.get("images", []) + if not isinstance(images, list) or not all(isinstance(image, dict) for image in images): + return False + for tree in trees: + if ( + not isinstance(tree, dict) + or not isinstance(tree.get("roots"), list) + or not tree["roots"] + ): + return False + nodes = list(tree["roots"]) + while nodes: + node = nodes.pop() + if not isinstance(node, dict): + return False + frame_id = node.get("frame_id") + count = node.get("sample_count") + if type(frame_id) is not int or not 0 <= frame_id < len(frames): + return False + if type(count) is not int or not 0 < count <= MAX_SAMPLE_COUNT: + return False + children = node.get("children", []) + if not isinstance(children, list) or not all( + isinstance(child, dict) for child in children + ): + return False + child_counts = [child.get("sample_count") for child in children] + if any(type(value) is not int or value <= 0 for value in child_counts): + return False + if sum(child_counts) > count: + return False + nodes.extend(children) + return True + + +def _expand_inline_frames(trees: list[dict[str, Any]], chains: dict[int, list[int]]) -> None: + nodes = [root for tree in trees for root in tree["roots"]] + while nodes: + node = nodes.pop() + children = node.get("children") + nodes.extend(children or []) + chain = chains.get(node["frame_id"]) + if not chain or len(chain) == 1: + continue + node.pop("children", None) + current = node + for frame_id in chain[1:]: + child = {"frame_id": frame_id, "sample_count": node["sample_count"]} + current["children"] = [child] + current = child + if children is not None: + current["children"] = children + + +def get_flamegraph_stacktraces( + payload: dict[str, Any], modules: list[dict[str, Any]] +) -> tuple[list[int], list[dict[str, Any]]]: + frame_ids = [] + stacktraces = [] + for frame_id, frame in enumerate(payload["frames"][:MAX_SYMBOLICATION_FRAMES]): + if not is_native_platform(frame.get("platform", payload["platform"])): + continue + frames = get_frames_for_symbolication([frame], payload, modules, adjustment="none") + if frames: + frame_ids.append(frame_id) + stacktraces.append({"frames": frames}) + return frame_ids, stacktraces + + +def _merge_symbolicated_frames( + payload: dict[str, Any], frame_ids: list[int], stacktraces: list[dict[str, Any]] +) -> dict[str, Any]: + output = deepcopy(payload) + chains = {} + for frame_id, stacktrace in zip(frame_ids, stacktraces): + original = payload["frames"][frame_id] + resolved = stacktrace.get("frames", []) + if not resolved or any( + frame.get("status") not in (None, "symbolicated") for frame in resolved + ): + continue + chain = [] + for position, frame in enumerate(resolved): + merged = deepcopy(original) + merge_frame(merged, frame, original.get("platform", payload["platform"])) + # Symbolicator may adjust addresses or normalize relative addressing modes. + # The diagnostic must retain its captured symbolication inputs. + merged["instruction_addr"] = original["instruction_addr"] + if "addr_mode" in original: + merged["addr_mode"] = original["addr_mode"] + else: + merged.pop("addr_mode", None) + if position == 0: + output["frames"][frame_id] = merged + chain.append(frame_id) + else: + chain.append(len(output["frames"])) + output["frames"].append(merged) + chains[frame_id] = chain + _expand_inline_frames(output["trees"], chains) + output["_symbolication"] = { + "status": "completed", + "frames": payload["frames"], + "trees": payload["trees"], + } + return output + + +class Flamegraphs: + def __init__(self, project: Project, data: Any): + self._project = project + self._event = data + self._attachments = list(get_attachments_for_event(data)) + + def get_flamegraphs(self) -> Generator[tuple[int, dict[str, Any]]]: + if not any(attachment.type == "event.flamegraph" for attachment in self._attachments): + return + if not features.has("organizations:flamegraph-attachments", self._project.organization): + return + for index, attachment in enumerate(self._attachments): + if attachment.type != "event.flamegraph": + continue + raw = attachment.load_data(self._project) + if len(raw) > MAX_FLAMEGRAPH_SIZE: + continue + try: + payload = orjson.loads(raw) + except orjson.JSONDecodeError: + continue + if not isinstance(payload, dict): + continue + state = payload.get("_symbolication") + if isinstance(state, dict): + if state.get("status") == "completed": + continue + payload["frames"] = state.get("frames", payload.get("frames")) + payload["trees"] = state.get("trees", payload.get("trees")) + if _is_valid_flamegraph(payload): + yield index, payload + + def apply_response_and_save( + self, + attachment_index: int, + payload: dict[str, Any], + frame_ids: list[int], + response: Any, + ) -> bool: + if not response or response.get("status") != "completed": + return False + stacktraces = response.get("stacktraces", []) + if len(stacktraces) != len(frame_ids): + return False + output = _merge_symbolicated_frames(payload, frame_ids, stacktraces) + serialized = orjson.dumps(output) + attachment = self._attachments[attachment_index] + metadata = attachment.meta() + metadata.update(data=serialized, chunks=None, size=len(serialized)) + self._attachments[attachment_index] = CachedAttachment(**metadata) + # Checkpoint before the next request, which may exhaust the event's timeout. + store_attachments_for_event( + self._project, self._event, self._attachments, timeout=CACHE_TIMEOUT + ) + # Reload metadata so later checkpoints don't rewrite earlier processed payloads. + self._attachments = list(get_attachments_for_event(self._event)) + return True diff --git a/src/sentry/lang/native/frames.py b/src/sentry/lang/native/frames.py new file mode 100644 index 000000000000..0da7740049be --- /dev/null +++ b/src/sentry/lang/native/frames.py @@ -0,0 +1,136 @@ +import posixpath + +from symbolic.debuginfo import normalize_debug_id +from symbolic.exceptions import ParseDebugIdError + +from sentry.lang.native.utils import is_native_platform +from sentry.stacktraces.functions import trim_function_name +from sentry.utils.safe import get_path, trim + + +def merge_frame(new_frame, symbolicated, platform="native"): + # il2cpp events which have the "csharp" platform have good (C#) names + # coming from the SDK, we do not want to override those with bad (mangled) C++ names. + if platform != "csharp" and symbolicated.get("function"): + raw_func = trim(symbolicated["function"], 256) + func = trim(trim_function_name(symbolicated["function"], platform), 256) + + # if function and raw function match, we can get away without + # storing a raw function + if func == raw_func: + new_frame["function"] = raw_func + # otherwise we store both + else: + new_frame["raw_function"] = raw_func + new_frame["function"] = func + if symbolicated.get("instruction_addr"): + new_frame["instruction_addr"] = symbolicated["instruction_addr"] + if symbolicated.get("function_id"): + new_frame["function_id"] = symbolicated["function_id"] + if symbolicated.get("symbol"): + new_frame["symbol"] = symbolicated["symbol"] + if symbolicated.get("abs_path"): + new_frame["abs_path"] = symbolicated["abs_path"] + new_frame["filename"] = posixpath.basename(symbolicated["abs_path"]) + if symbolicated.get("filename"): + new_frame["filename"] = symbolicated["filename"] + if symbolicated.get("lineno"): + new_frame["lineno"] = symbolicated["lineno"] + if symbolicated.get("colno"): + new_frame["colno"] = symbolicated["colno"] + # similarly as with `function` above, we do want to retain the original "package". + if platform != "csharp" and symbolicated.get("package"): + new_frame["package"] = symbolicated["package"] + if symbolicated.get("trust"): + new_frame["trust"] = symbolicated["trust"] + if symbolicated.get("pre_context"): + new_frame["pre_context"] = symbolicated["pre_context"] + if symbolicated.get("context_line") is not None: + new_frame["context_line"] = symbolicated["context_line"] + if symbolicated.get("post_context"): + new_frame["post_context"] = symbolicated["post_context"] + if symbolicated.get("source_link"): + new_frame["source_link"] = symbolicated["source_link"] + if symbolicated.get("vars"): + new_frame["vars"] = symbolicated["vars"] + + addr_mode = symbolicated.get("addr_mode") + if addr_mode is None: + new_frame.pop("addr_mode", None) + else: + new_frame["addr_mode"] = addr_mode + + if symbolicated.get("status"): + frame_meta = new_frame.setdefault("data", {}) + frame_meta["symbolicator_status"] = symbolicated["status"] + + +def handles_frame(data, frame): + if not frame: + return False + + if get_path(frame, "data", "symbolicator_status") is not None: + return False + + # TODO: Consider ignoring platform + platform = frame.get("platform") or data.get("platform") + return is_native_platform(platform) and frame.get("instruction_addr") is not None + + +def get_frames_for_symbolication( + frames, + data, + modules, + adjustment=None, +): + modules_by_debug_id = None + rv = [] + adjustment = adjustment or "auto" + + for frame in reversed(frames): + if not handles_frame(data, frame): + continue + s_frame = dict(frame) + + if adjustment == "none": + s_frame["adjust_instruction_addr"] = False + + # validate and expand addressing modes. If we can't validate and + # expand it, we keep None which is absolute. That's not great but + # at least won't do damage. + addr_mode = s_frame.pop("addr_mode", None) + sanitized_addr_mode = None + + # None and abs mean absolute addressing explicitly. + if addr_mode in (None, "abs"): + pass + # this is relative addressing to module by index or debug id. + elif addr_mode.startswith("rel:"): + arg = addr_mode[4:] + idx = None + + if modules_by_debug_id is None: + modules_by_debug_id = {x.get("debug_id"): idx for idx, x in enumerate(modules)} + try: + idx = modules_by_debug_id.get(normalize_debug_id(arg)) + except ParseDebugIdError: + pass + + if idx is None and arg.isdigit(): + idx = int(arg) + + if idx is not None: + sanitized_addr_mode = "rel:%d" % idx + + if sanitized_addr_mode is not None: + s_frame["addr_mode"] = sanitized_addr_mode + rv.append(s_frame) + + if len(rv) > 0: + first_frame = rv[0] + if adjustment == "all": + first_frame["adjust_instruction_addr"] = True + elif adjustment == "all_but_first": + first_frame["adjust_instruction_addr"] = False + + return rv diff --git a/src/sentry/lang/native/processing.py b/src/sentry/lang/native/processing.py index fd3f6595c488..e096b8736487 100644 --- a/src/sentry/lang/native/processing.py +++ b/src/sentry/lang/native/processing.py @@ -1,16 +1,15 @@ from __future__ import annotations import logging -import posixpath from collections.abc import Mapping from typing import Any import sentry_sdk -from symbolic.debuginfo import normalize_debug_id -from symbolic.exceptions import ParseDebugIdError from sentry import options from sentry.lang.native.error import SymbolicationFailed, write_error +from sentry.lang.native.flamegraphs import Flamegraphs, get_flamegraph_stacktraces +from sentry.lang.native.frames import get_frames_for_symbolication, handles_frame, merge_frame from sentry.lang.native.symbolicator import FrameOrder, Symbolicator, SymbolicatorFunction from sentry.lang.native.utils import ( get_event_attachment, @@ -26,11 +25,10 @@ ) from sentry.models.eventerror import EventErrorType from sentry.options.rollout import in_random_rollout -from sentry.stacktraces.functions import trim_function_name from sentry.stacktraces.processing import StacktraceInfo, find_stacktraces_in_data from sentry.utils import metrics from sentry.utils.in_app import is_known_third_party, is_optional_package -from sentry.utils.safe import get_path, set_path, setdefault_path, trim +from sentry.utils.safe import get_path, set_path, setdefault_path logger = logging.getLogger(__name__) @@ -77,63 +75,6 @@ ] -def _merge_frame(new_frame, symbolicated, platform="native"): - # il2cpp events which have the "csharp" platform have good (C#) names - # coming from the SDK, we do not want to override those with bad (mangled) C++ names. - if platform != "csharp" and symbolicated.get("function"): - raw_func = trim(symbolicated["function"], 256) - func = trim(trim_function_name(symbolicated["function"], platform), 256) - - # if function and raw function match, we can get away without - # storing a raw function - if func == raw_func: - new_frame["function"] = raw_func - # otherwise we store both - else: - new_frame["raw_function"] = raw_func - new_frame["function"] = func - if symbolicated.get("instruction_addr"): - new_frame["instruction_addr"] = symbolicated["instruction_addr"] - if symbolicated.get("function_id"): - new_frame["function_id"] = symbolicated["function_id"] - if symbolicated.get("symbol"): - new_frame["symbol"] = symbolicated["symbol"] - if symbolicated.get("abs_path"): - new_frame["abs_path"] = symbolicated["abs_path"] - new_frame["filename"] = posixpath.basename(symbolicated["abs_path"]) - if symbolicated.get("filename"): - new_frame["filename"] = symbolicated["filename"] - if symbolicated.get("lineno"): - new_frame["lineno"] = symbolicated["lineno"] - if symbolicated.get("colno"): - new_frame["colno"] = symbolicated["colno"] - # similarly as with `function` above, we do want to retain the original "package". - if platform != "csharp" and symbolicated.get("package"): - new_frame["package"] = symbolicated["package"] - if symbolicated.get("trust"): - new_frame["trust"] = symbolicated["trust"] - if symbolicated.get("pre_context"): - new_frame["pre_context"] = symbolicated["pre_context"] - if symbolicated.get("context_line") is not None: - new_frame["context_line"] = symbolicated["context_line"] - if symbolicated.get("post_context"): - new_frame["post_context"] = symbolicated["post_context"] - if symbolicated.get("source_link"): - new_frame["source_link"] = symbolicated["source_link"] - if symbolicated.get("vars"): - new_frame["vars"] = symbolicated["vars"] - - addr_mode = symbolicated.get("addr_mode") - if addr_mode is None: - new_frame.pop("addr_mode", None) - else: - new_frame["addr_mode"] = addr_mode - - if symbolicated.get("status"): - frame_meta = new_frame.setdefault("data", {}) - frame_meta["symbolicator_status"] = symbolicated["status"] - - def _handle_image_status(status, image, os, data): if status in ("found", "unused"): return @@ -316,7 +257,7 @@ def _merge_full_response(data, response): for complete_frame in reversed(complete_stacktrace["frames"]): new_frame: dict[str, Any] = {} - _merge_frame(new_frame, complete_frame) + merge_frame(new_frame, complete_frame) data_stacktrace["frames"].append(new_frame) @@ -376,78 +317,26 @@ def process_applecrashreport(symbolicator: Symbolicator, data: Any) -> Any: return data -def _handles_frame(data, frame): - if not frame: - return False - - if get_path(frame, "data", "symbolicator_status") is not None: - return False - - # TODO: Consider ignoring platform - platform = frame.get("platform") or data.get("platform") - return is_native_platform(platform) and frame.get("instruction_addr") is not None - - -def get_frames_for_symbolication( - frames, - data, - modules, - adjustment=None, -): - modules_by_debug_id = None - rv = [] - adjustment = adjustment or "auto" - - for frame in reversed(frames): - if not _handles_frame(data, frame): - continue - s_frame = dict(frame) - - if adjustment == "none": - s_frame["adjust_instruction_addr"] = False - - # validate and expand addressing modes. If we can't validate and - # expand it, we keep None which is absolute. That's not great but - # at least won't do damage. - addr_mode = s_frame.pop("addr_mode", None) - sanitized_addr_mode = None - - # None and abs mean absolute addressing explicitly. - if addr_mode in (None, "abs"): - pass - # this is relative addressing to module by index or debug id. - elif addr_mode.startswith("rel:"): - arg = addr_mode[4:] - idx = None - - if modules_by_debug_id is None: - modules_by_debug_id = {x.get("debug_id"): idx for idx, x in enumerate(modules)} - try: - idx = modules_by_debug_id.get(normalize_debug_id(arg)) - except ParseDebugIdError: - pass - - if idx is None and arg.isdigit(): - idx = int(arg) - - if idx is not None: - sanitized_addr_mode = "rel:%d" % idx - - if sanitized_addr_mode is not None: - s_frame["addr_mode"] = sanitized_addr_mode - rv.append(s_frame) - - if len(rv) > 0: - first_frame = rv[0] - if adjustment == "all": - first_frame["adjust_instruction_addr"] = True - elif adjustment == "all_but_first": - first_frame["adjust_instruction_addr"] = False - - return rv - - def process_native_stacktraces(symbolicator: Symbolicator, data: Any) -> Any: + flamegraphs = Flamegraphs(symbolicator.project, data) + attachments_changed = False + for attachment_index, payload in flamegraphs.get_flamegraphs(): + modules = native_images_from_data(payload) + if not modules: + continue + frame_ids, stacktraces = get_flamegraph_stacktraces(payload, modules) + if not stacktraces: + continue + response = symbolicator.process_payload( + platform=payload["platform"], + stacktraces=stacktraces, + modules=modules, + frame_order=FrameOrder.caller_first, + apply_source_context=False, + ) + attachments_changed |= flamegraphs.apply_response_and_save( + attachment_index, payload, frame_ids, response + ) stacktrace_infos = [ stacktrace for stacktrace in find_stacktraces_in_data(data) @@ -470,7 +359,7 @@ def process_native_stacktraces(symbolicator: Symbolicator, data: Any) -> Any: ] if not any(stacktrace["frames"] for stacktrace in stacktraces): - return + return data if attachments_changed else None signal = signal_from_data(data) @@ -516,14 +405,14 @@ def process_native_stacktraces(symbolicator: Symbolicator, data: Any) -> Any: native_frames_idx = 0 for raw_frame in reversed(sinfo.stacktrace["frames"]): - if not _handles_frame(data, raw_frame): + if not handles_frame(data, raw_frame): new_frames.append(raw_frame) continue for complete_frame in complete_frames_by_idx.get(native_frames_idx) or (): merged_frame = dict(raw_frame) platform = merged_frame.get("platform") or data.get("platform") or "native" - _merge_frame(merged_frame, complete_frame, platform) + merge_frame(merged_frame, complete_frame, platform) if merged_frame.get("package"): raw_frame["package"] = merged_frame["package"] new_frames.append(merged_frame) @@ -639,7 +528,9 @@ def get_native_symbolication_functions( functions.append(SymbolicatorFunction.native) return functions # early return to prevent duplicates - if has_native_stacktraces(stacktraces): + if has_native_stacktraces(stacktraces) or any( + attachment.get("type") == "event.flamegraph" for attachment in data.get("_attachments", []) + ): # Run native symbolication first, to prevent sending minidump / ACR events to # symbolicator twice (after symbolication a minidump event will have native stacktraces). functions.insert(0, SymbolicatorFunction.native) diff --git a/src/sentry/reprocessing2.py b/src/sentry/reprocessing2.py index 24dcc1bda025..98398467f99e 100644 --- a/src/sentry/reprocessing2.py +++ b/src/sentry/reprocessing2.py @@ -229,7 +229,7 @@ def pull_event_data(project_id: int, event_id: str) -> ReprocessableEvent: def reprocess_event(project_id: int, event_id: str, start_time: float) -> None: - from sentry.ingest.consumer.processors import CACHE_TIMEOUT + from sentry.ingest.consumer import CACHE_TIMEOUT from sentry.tasks.store import preprocess_event_from_reprocessing reprocessable_event = pull_event_data(project_id, event_id) diff --git a/tests/sentry/ingest/ingest_consumer/test_ingest_consumer_processing.py b/tests/sentry/ingest/ingest_consumer/test_ingest_consumer_processing.py index 0d8277ab4081..a3e2ed79f487 100644 --- a/tests/sentry/ingest/ingest_consumer/test_ingest_consumer_processing.py +++ b/tests/sentry/ingest/ingest_consumer/test_ingest_consumer_processing.py @@ -41,7 +41,7 @@ from sentry.ingest.types import ConsumerType from sentry.lang.native.utils import STORE_CRASH_REPORTS_ALL from sentry.models.debugfile import create_files_from_dif_zip -from sentry.models.eventattachment import EventAttachment, PendingEventAttachment +from sentry.models.eventattachment import V2_PREFIX, EventAttachment, PendingEventAttachment from sentry.models.userreport import UserReport from sentry.objectstore import UsecaseId, get_session from sentry.services import eventstore @@ -611,6 +611,111 @@ def test_deobfuscate_view_hierarchy_with_objectstore_attachment( do_process_view_hierarchy(default_project, task_runner, use_objectstore=True) +@debug_files_test_both_backends +class TestSymbolicateFlamegraph: + @django_db_all + @requires_symbolicator + @thread_leak_allowlist(reason="django dev server", issue=97036) + def test_symbolicate_flamegraph_attachment( + self, default_project, task_runner, live_server + ) -> None: + do_process_flamegraph(default_project, task_runner, live_server) + + @django_db_all + @requires_objectstore + @requires_symbolicator + @thread_leak_allowlist(reason="django dev server", issue=97036) + def test_symbolicate_stored_flamegraph_attachment( + self, default_project, task_runner, live_server + ) -> None: + do_process_flamegraph(default_project, task_runner, live_server, use_objectstore=True) + + +def do_process_flamegraph(default_project, task_runner, live_server, use_objectstore=False): + default_project.update_option("sentry:builtin_symbol_sources", []) + event = get_normalized_event( + {"platform": "other", "message": "Flamegraph symbolication"}, default_project + ) + with zipfile.ZipFile(BytesIO(), "w") as archive: + archive.write(get_fixture_path("native", "hello.dsym"), "dSYM/hello") + create_files_from_dif_zip(archive, project=default_project) + diagnostic = { + "version": "1", + "platform": "cocoa", + "frames": [{"instruction_addr": "0x100000fa0"}], + "debug_meta": { + "images": [ + { + "type": "macho", + "debug_id": "502fc0a5-1ec1-3e47-9998-684fa139dca7", + "image_addr": "0x100000000", + "image_size": 4096, + "arch": "x86_64", + } + ] + }, + "trees": [{"roots": [{"frame_id": 0, "sample_count": 10}]}], + } + attachment_id = "ca90fb45-6dd9-40a0-a18f-8693aa621abb" + raw = orjson.dumps(diagnostic) + attachment_metadata = { + "id": attachment_id, + "name": "flamegraph.json", + "content_type": "application/json", + "attachment_type": "event.flamegraph", + "size": len(raw), + } + if use_objectstore: + session = get_session(UsecaseId.ATTACHMENTS, default_project) + attachment_metadata["stored_id"] = session.put(raw) + else: + process_attachment_chunk( + { + "payload": raw, + "event_id": event["event_id"], + "project_id": default_project.id, + "id": attachment_id, + "chunk_index": 0, + } + ) + attachment_metadata["chunks"] = 1 + with ( + Feature({"organizations:flamegraph-attachments": True}), + override_options({"system.url-prefix": live_server.url}), + patch("sentry.auth.system.is_internal_ip", return_value=True), + task_runner(), + ): + process_event( + ConsumerType.Events, + { + "payload": orjson.dumps(event).decode(), + "start_time": time.time(), + "event_id": event["event_id"], + "project_id": default_project.id, + "remote_addr": "127.0.0.1", + "attachments": [attachment_metadata], + }, + project=default_project, + ) + attachment = EventAttachment.objects.get( + project_id=default_project.id, event_id=event["event_id"] + ) + assert attachment.type == "event.flamegraph" + assert attachment.content_type == "application/json" + with attachment.getfile() as file: + processed = orjson.loads(file.read()) + assert processed["frames"][0]["function"] == "main" + assert processed["frames"][0]["instruction_addr"] == "0x100000fa0" + assert processed["trees"] == diagnostic["trees"] + assert processed["_symbolication"]["status"] == "completed" + assert attachment.size == len(orjson.dumps(processed)) + if use_objectstore: + assert attachment.blob_path == V2_PREFIX + attachment_metadata["stored_id"] + stored = session.get(attachment_metadata["stored_id"]) + assert stored is not None + assert orjson.loads(stored.payload.read()) == processed + + def do_process_view_hierarchy(project, task_runner, use_objectstore=False): payload = get_normalized_event( { diff --git a/tests/sentry/lang/native/test_processing.py b/tests/sentry/lang/native/test_processing.py index d6b002de0b58..477ab0b9feef 100644 --- a/tests/sentry/lang/native/test_processing.py +++ b/tests/sentry/lang/native/test_processing.py @@ -6,22 +6,32 @@ from __future__ import annotations import re +from copy import deepcopy from typing import Any from unittest import mock +import orjson import pytest +from sentry.attachments import ( + CachedAttachment, + get_attachments_for_event, + store_attachments_for_event, +) +from sentry.lang.native.frames import get_frames_for_symbolication from sentry.lang.native.processing import ( ELECTRON_FIRST_MODULE_REWRITE_RULES, _merge_image, - get_frames_for_symbolication, get_native_symbolication_functions, process_native_stacktraces, ) from sentry.lang.native.symbolicator import SymbolicatorFunction from sentry.models.eventerror import EventErrorType +from sentry.objectstore import UsecaseId, get_session from sentry.stacktraces.processing import find_stacktraces_in_data +from sentry.testutils.helpers.features import Feature from sentry.testutils.pytest.fixtures import django_db_all +from sentry.testutils.skips import requires_objectstore from sentry.utils.safe import get_path MINIDUMP_PLACEHOLDER = { @@ -94,6 +104,452 @@ def test_get_native_symbolication_functions( assert functions == expected_functions +@django_db_all +def test_process_native_flamegraph(cached_flamegraph) -> None: + data, _, symbolicator = cached_flamegraph + payload = { + "version": "1", + "platform": "cocoa", + "debug_meta": { + "images": [ + { + "type": "macho", + "debug_id": "84a04d24-0e60-3810-a8c0-90a65e2df61a", + "image_addr": "0x100000000", + "image_size": 8192, + "arch": "arm64", + } + ] + }, + "frames": [ + {"instruction_addr": "0x100001000", "addr_mode": "abs", "custom": "retained"}, + {"instruction_addr": "0x100001200", "function": "unresolved"}, + ], + "trees": [ + { + "thread_attributed": True, + "roots": [ + { + "frame_id": 0, + "sample_count": 10, + "children": [ + { + "frame_id": 0, + "sample_count": 6, + "children": [{"frame_id": 1, "sample_count": 4}], + } + ], + } + ], + } + ], + "custom": "retained", + } + original = deepcopy(payload) + store_attachments_for_event( + symbolicator.project, + data, + [ + CachedAttachment( + type="event.flamegraph", + name="flamegraph.json", + content_type="application/json", + data=orjson.dumps(payload), + ) + ], + timeout=60, + ) + data["debug_meta"] = {"images": [{"type": "elf", "debug_id": "event-image"}]} + symbolicator.process_payload.return_value = { + "status": "completed", + "stacktraces": [ + { + "frames": [ + { + "function": "outer", + "status": "symbolicated", + "instruction_addr": "0x100000fff", + }, + { + "function": "inlined", + "status": "symbolicated", + "filename": "Source.swift", + "lineno": 42, + }, + ] + }, + {"frames": [{"status": "missing", "instruction_addr": "0x1000011ff"}]}, + ], + } + with Feature({"organizations:flamegraph-attachments": True}): + result = process_native_stacktraces(symbolicator, data) + symbolicator.process_payload.assert_called_once() + request = symbolicator.process_payload.call_args.kwargs + assert request["platform"] == "cocoa" + assert request["modules"] == payload["debug_meta"]["images"] + assert request["apply_source_context"] is False + assert request["stacktraces"] == [ + { + "frames": [ + { + "instruction_addr": "0x100001000", + "custom": "retained", + "adjust_instruction_addr": False, + } + ] + }, + { + "frames": [ + { + "instruction_addr": "0x100001200", + "function": "unresolved", + "adjust_instruction_addr": False, + } + ] + }, + ] + assert result is data + processed = orjson.loads(next(get_attachments_for_event(data)).load_data(symbolicator.project)) + assert processed["frames"][0]["function"] == "outer" + assert processed["frames"][0]["instruction_addr"] == "0x100001000" + assert processed["frames"][0]["addr_mode"] == "abs" + assert processed["frames"][1] == original["frames"][1] + assert processed["frames"][2]["function"] == "inlined" + root = processed["trees"][0]["roots"][0] + assert root["sample_count"] == 10 + assert root["children"][0]["frame_id"] == 2 + assert root["children"][0]["sample_count"] == 10 + recursive = root["children"][0]["children"][0] + assert recursive["frame_id"] == 0 + assert recursive["children"][0]["sample_count"] == 6 + assert recursive["children"][0]["children"] == [{"frame_id": 1, "sample_count": 4}] + assert processed["debug_meta"] == original["debug_meta"] + assert processed["custom"] == "retained" + assert processed["_symbolication"]["frames"] == original["frames"] + assert processed["_symbolication"]["trees"] == original["trees"] + + +@django_db_all +def test_flamegraph_symbolication_is_idempotent(cached_flamegraph) -> None: + data, _, symbolicator = cached_flamegraph + with Feature({"organizations:flamegraph-attachments": True}): + process_native_stacktraces(symbolicator, data) + before = next(get_attachments_for_event(data)).load_data(symbolicator.project) + assert process_native_stacktraces(symbolicator, data) is None + symbolicator.process_payload.assert_called_once() + assert next(get_attachments_for_event(data)).load_data(symbolicator.project) == before + + +def test_route_flamegraph_to_native_symbolication() -> None: + data = {"platform": "other", "_attachments": [{"type": "event.flamegraph"}]} + assert get_native_symbolication_functions(data, []) == [SymbolicatorFunction.native] + + +@pytest.fixture +def cached_flamegraph(default_project): + payload = { + "version": "1", + "platform": "cocoa", + "frames": [{"instruction_addr": "0x100001000", "data": {"custom": "retained"}}], + "debug_meta": { + "images": [{"type": "macho", "debug_id": "84a04d24-0e60-3810-a8c0-90a65e2df61a"}] + }, + "trees": [{"roots": [{"frame_id": 0, "sample_count": 10}]}], + } + data = { + "platform": "other", + "project": default_project.id, + "event_id": "cc3e6c2bb6b6498097f336d1e6979f4b", + } + store_attachments_for_event( + default_project, + data, + [ + CachedAttachment( + type="event.flamegraph", + name="flamegraph.json", + content_type="application/json", + data=orjson.dumps(payload), + ) + ], + timeout=60, + ) + symbolicator = mock.Mock(project=default_project) + symbolicator.process_payload.return_value = { + "status": "completed", + "stacktraces": [ + { + "frames": [ + {"status": "symbolicated", "function": "outer"}, + {"status": "symbolicated", "function": "inner"}, + ] + } + ], + } + return data, payload, symbolicator + + +@django_db_all +def test_flamegraph_feature_disabled(cached_flamegraph) -> None: + data, payload, symbolicator = cached_flamegraph + with Feature({"organizations:flamegraph-attachments": False}): + assert process_native_stacktraces(symbolicator, data) is None + symbolicator.process_payload.assert_not_called() + assert next(get_attachments_for_event(data)).load_data(symbolicator.project) == orjson.dumps( + payload + ) + + +@django_db_all +def test_flamegraph_frame_limit(cached_flamegraph) -> None: + data, payload, symbolicator = cached_flamegraph + payload["frames"] *= 2501 + store_attachments_for_event( + symbolicator.project, + data, + [ + CachedAttachment( + type="event.flamegraph", + data=orjson.dumps(payload), + ) + ], + timeout=60, + ) + symbolicator.process_payload.return_value = { + "status": "completed", + "stacktraces": [{"frames": []}] * 2500, + } + with Feature({"organizations:flamegraph-attachments": True}): + process_native_stacktraces(symbolicator, data) + assert len(symbolicator.process_payload.call_args.kwargs["stacktraces"]) == 2500 + processed = orjson.loads(next(get_attachments_for_event(data)).load_data(symbolicator.project)) + assert processed["frames"] == payload["frames"] + assert processed["trees"] == payload["trees"] + + +@django_db_all +@pytest.mark.parametrize( + "response", + [{"status": "failed"}, {"status": "pending"}, {"status": "completed", "stacktraces": []}], +) +def test_flamegraph_failure_preserves_attachment(cached_flamegraph, response) -> None: + data, payload, symbolicator = cached_flamegraph + symbolicator.process_payload.return_value = response + with Feature({"organizations:flamegraph-attachments": True}): + assert process_native_stacktraces(symbolicator, data) is None + assert next(get_attachments_for_event(data)).load_data(symbolicator.project) == orjson.dumps( + payload + ) + + +@django_db_all +@pytest.mark.parametrize( + "raw", [b"invalid json", b"null", b'{"version":"2"}', b" " * (10 * 1024 * 1024 + 1)] +) +def test_flamegraph_invalid_attachment(cached_flamegraph, raw) -> None: + data, _, symbolicator = cached_flamegraph + store_attachments_for_event( + symbolicator.project, + data, + [ + CachedAttachment( + type="event.flamegraph", + data=raw, + ) + ], + timeout=60, + ) + with Feature({"organizations:flamegraph-attachments": True}): + assert process_native_stacktraces(symbolicator, data) is None + symbolicator.process_payload.assert_not_called() + assert next(get_attachments_for_event(data)).load_data(symbolicator.project) == raw + + +@django_db_all +@pytest.mark.parametrize( + "root", + [ + {"frame_id": 1, "sample_count": 10}, + {"frame_id": True, "sample_count": 10}, + {"frame_id": 0, "sample_count": 0}, + {"frame_id": 0, "sample_count": True}, + {"frame_id": 0, "sample_count": 9007199254740992}, + {"frame_id": 0, "sample_count": 10, "children": "invalid"}, + {"frame_id": 0, "sample_count": 10, "children": [{"frame_id": 0, "sample_count": 11}]}, + ], +) +def test_flamegraph_invalid_tree_is_not_symbolicated(cached_flamegraph, root) -> None: + data, payload, symbolicator = cached_flamegraph + payload["trees"][0]["roots"] = [root] + raw = orjson.dumps(payload) + store_attachments_for_event( + symbolicator.project, + data, + [CachedAttachment(type="event.flamegraph", data=raw)], + timeout=60, + ) + with Feature({"organizations:flamegraph-attachments": True}): + assert process_native_stacktraces(symbolicator, data) is None + symbolicator.process_payload.assert_not_called() + assert next(get_attachments_for_event(data)).load_data(symbolicator.project) == raw + + +@django_db_all +def test_flamegraph_reprocessing_uses_original_tables(cached_flamegraph) -> None: + data, payload, symbolicator = cached_flamegraph + with Feature({"organizations:flamegraph-attachments": True}): + process_native_stacktraces(symbolicator, data) + processed = orjson.loads(next(get_attachments_for_event(data)).load_data(symbolicator.project)) + processed["_symbolication"]["status"] = "pending" + store_attachments_for_event( + symbolicator.project, + data, + [ + CachedAttachment( + type="event.flamegraph", + data=orjson.dumps(processed), + ) + ], + timeout=60, + ) + with Feature({"organizations:flamegraph-attachments": True}): + process_native_stacktraces(symbolicator, data) + processed = orjson.loads(next(get_attachments_for_event(data)).load_data(symbolicator.project)) + assert len(processed["frames"]) == 2 + assert processed["trees"] == [ + { + "roots": [ + { + "frame_id": 0, + "sample_count": 10, + "children": [{"frame_id": 1, "sample_count": 10}], + } + ] + } + ] + assert processed["_symbolication"]["frames"] == payload["frames"] + assert processed["_symbolication"]["trees"] == payload["trees"] + assert symbolicator.process_payload.call_count == 2 + + +@django_db_all +@requires_objectstore +def test_flamegraph_objectstore_attachment(cached_flamegraph) -> None: + data, payload, symbolicator = cached_flamegraph + session = get_session(UsecaseId.ATTACHMENTS, symbolicator.project) + stored_id = session.put(orjson.dumps(payload)) + store_attachments_for_event( + symbolicator.project, + data, + [ + CachedAttachment( + type="event.flamegraph", + name="flamegraph.json", + content_type="application/json", + stored_id=stored_id, + size=len(orjson.dumps(payload)), + retention_days=7, + ) + ], + timeout=60, + ) + with Feature({"organizations:flamegraph-attachments": True}): + process_native_stacktraces(symbolicator, data) + attachment = next(get_attachments_for_event(data)) + assert attachment.stored_id == stored_id + assert attachment.retention_days == 7 + stored = session.get(stored_id) + assert stored is not None + processed = orjson.loads(stored.payload.read()) + assert processed["frames"][0]["function"] == "outer" + assert processed["frames"][1]["function"] == "inner" + assert attachment.size == len(orjson.dumps(processed)) + + +@pytest.fixture +def multiple_flamegraphs(cached_flamegraph): + data, payload, symbolicator = cached_flamegraph + second = deepcopy(payload) + second["debug_meta"]["images"][0]["debug_id"] = "502fc0a5-1ec1-3e47-9998-684fa139dca7" + store_attachments_for_event( + symbolicator.project, + data, + [ + CachedAttachment(type="event.flamegraph", data=orjson.dumps(payload)), + CachedAttachment(type="event.attachment", name="other.txt", data=b"unchanged"), + CachedAttachment(type="event.flamegraph", data=orjson.dumps(second)), + ], + timeout=60, + ) + return data, payload, second, symbolicator + + +@django_db_all +def test_flamegraph_multiple_attachments(multiple_flamegraphs) -> None: + data, first, second, symbolicator = multiple_flamegraphs + with Feature({"organizations:flamegraph-attachments": True}): + assert process_native_stacktraces(symbolicator, data) is data + assert symbolicator.process_payload.call_count == 2 + first_request, second_request = symbolicator.process_payload.call_args_list + assert first_request.kwargs["modules"] == first["debug_meta"]["images"] + assert second_request.kwargs["modules"] == second["debug_meta"]["images"] + first_attachment, other_attachment, second_attachment = get_attachments_for_event(data) + first_processed = orjson.loads(first_attachment.load_data(symbolicator.project)) + second_processed = orjson.loads(second_attachment.load_data(symbolicator.project)) + assert first_processed["frames"][0]["function"] == "outer" + assert second_processed["frames"][0]["function"] == "outer" + assert first_processed["_symbolication"]["status"] == "completed" + assert second_processed["_symbolication"]["status"] == "completed" + assert other_attachment.load_data(symbolicator.project) == b"unchanged" + + +@django_db_all +@requires_objectstore +def test_flamegraph_checkpoints_do_not_rewrite_processed_objects(multiple_flamegraphs) -> None: + data, first, second, symbolicator = multiple_flamegraphs + session = get_session(UsecaseId.ATTACHMENTS, symbolicator.project) + first_id = session.put(orjson.dumps(first)) + second_id = session.put(orjson.dumps(second)) + store_attachments_for_event( + symbolicator.project, + data, + [ + CachedAttachment(type="event.flamegraph", stored_id=first_id), + CachedAttachment(type="event.flamegraph", stored_id=second_id), + ], + timeout=60, + ) + with ( + Feature({"organizations:flamegraph-attachments": True}), + mock.patch("sentry.attachments.base.get_session", return_value=session), + mock.patch.object(session, "put", wraps=session.put) as put, + ): + assert process_native_stacktraces(symbolicator, data) is data + assert put.call_count == 2 + assert [call.kwargs["key"] for call in put.call_args_list] == [first_id, second_id] + + +@django_db_all +def test_flamegraph_checkpoint_survives_later_timeout(multiple_flamegraphs) -> None: + from sentry.tasks.symbolication import SymbolicationTimeout + + data, _, second, symbolicator = multiple_flamegraphs + symbolicator.process_payload.side_effect = [ + symbolicator.process_payload.return_value, + SymbolicationTimeout(), + ] + with Feature({"organizations:flamegraph-attachments": True}): + with pytest.raises(SymbolicationTimeout): + process_native_stacktraces(symbolicator, data) + assert symbolicator.process_payload.call_count == 2 + first_attachment, other_attachment, second_attachment = get_attachments_for_event(data) + first_processed = orjson.loads(first_attachment.load_data(symbolicator.project)) + assert first_processed["frames"][0]["function"] == "outer" + assert first_processed["_symbolication"]["status"] == "completed" + assert other_attachment.load_data(symbolicator.project) == b"unchanged" + assert second_attachment.load_data(symbolicator.project) == orjson.dumps(second) + + def test_merge_symbolicator_image_empty() -> None: data: dict[str, Any] = {} _merge_image({}, {}, None, data) From b10c4b715d32ab7de319b189c9e41bf476e9befc Mon Sep 17 00:00:00 2001 From: Philip Niedertscheider Date: Mon, 5 Oct 2026 17:21:34 +0200 Subject: [PATCH 2/7] feat(native): Expose the flamegraph attachment feature flag --- src/sentry/features/temporary.py | 2 +- tests/sentry/api/serializers/test_organization.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/sentry/features/temporary.py b/src/sentry/features/temporary.py index bb8be41c2768..ad78255b4896 100644 --- a/src/sentry/features/temporary.py +++ b/src/sentry/features/temporary.py @@ -424,7 +424,7 @@ def register_temporary_features(manager: FeatureManager) -> None: # Enables view hierarchy attachment scrubbing manager.add("organizations:view-hierarchy-scrubbing", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False) # Enables symbolication for flamegraph attachments - manager.add("organizations:flamegraph-attachments", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False) + manager.add("organizations:flamegraph-attachments", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=True) # Enable AI-powered assertion suggestions for uptime monitors manager.add("organizations:uptime-ai-assertion-suggestions", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=True) # Enable auto spam classification at User Feedback ingest time diff --git a/tests/sentry/api/serializers/test_organization.py b/tests/sentry/api/serializers/test_organization.py index 041f680add5b..7b2af591e81f 100644 --- a/tests/sentry/api/serializers/test_organization.py +++ b/tests/sentry/api/serializers/test_organization.py @@ -88,6 +88,11 @@ def test_simple(self) -> None: "uptime", ] + @with_feature("organizations:flamegraph-attachments") + def test_flamegraph_feature_is_exposed(self) -> None: + result = serialize(self.organization, self.user) + assert "flamegraph-attachments" in result["features"] + @mock.patch("sentry.features.batch_has") def test_organization_batch_has(self, mock_batch: mock.MagicMock) -> None: user = self.create_user() From dcdf5a7a593c34b07ac930b36ca7dac867494b4d Mon Sep 17 00:00:00 2001 From: Philip Niedertscheider Date: Thu, 15 Oct 2026 10:21:36 +0200 Subject: [PATCH 3/7] fix(native): Harden flamegraph attachment symbolication Preserve event processing when attachments are invalid or unavailable. Handle deep trees iteratively, enforce the rewritten payload size limit, and preserve relative image references. Reuse existing JSON and attachment helpers and add regression coverage. --- fixtures/stubs-for-mypy/rapidjson.pyi | 4 +- src/sentry/lang/native/flamegraphs.py | 99 ++++++---- src/sentry/lang/native/processing.py | 2 +- tests/sentry/lang/native/test_processing.py | 205 +++++++++++++++++++- 4 files changed, 264 insertions(+), 46 deletions(-) diff --git a/fixtures/stubs-for-mypy/rapidjson.pyi b/fixtures/stubs-for-mypy/rapidjson.pyi index f40ce19c07c3..453507e36ba2 100644 --- a/fixtures/stubs-for-mypy/rapidjson.pyi +++ b/fixtures/stubs-for-mypy/rapidjson.pyi @@ -2,5 +2,5 @@ from typing import Any class JSONDecodeError(ValueError): ... -def dumps(o: object) -> str: ... -def loads(s: str | bytes) -> Any: ... +def dumps(o: object, *, ensure_ascii: bool = True, allow_nan: bool = True) -> str: ... +def loads(s: str | bytes, *, allow_nan: bool = True) -> Any: ... diff --git a/src/sentry/lang/native/flamegraphs.py b/src/sentry/lang/native/flamegraphs.py index 421d2366fc32..16768e284a25 100644 --- a/src/sentry/lang/native/flamegraphs.py +++ b/src/sentry/lang/native/flamegraphs.py @@ -1,18 +1,18 @@ from collections.abc import Generator -from copy import deepcopy from typing import Any -import orjson +import rapidjson from sentry import features from sentry.attachments import ( CachedAttachment, + MissingAttachmentChunks, get_attachments_for_event, store_attachments_for_event, ) from sentry.ingest.consumer import CACHE_TIMEOUT from sentry.lang.native.frames import get_frames_for_symbolication, merge_frame -from sentry.lang.native.utils import is_native_platform +from sentry.lang.native.utils import is_native_image from sentry.models.project import Project MAX_FLAMEGRAPH_SIZE = 10 * 1024 * 1024 @@ -41,12 +41,20 @@ def _is_valid_flamegraph(payload: Any) -> bool: return False if "platform" in frame and not isinstance(frame["platform"], str): return False + if "data" in frame and not isinstance(frame["data"], dict): + return False debug_meta = payload.get("debug_meta", {}) if not isinstance(debug_meta, dict): return False images = debug_meta.get("images", []) if not isinstance(images, list) or not all(isinstance(image, dict) for image in images): return False + for image in images: + if any( + image.get(field) is not None and not isinstance(image[field], str) + for field in ("debug_id", "id", "uuid") + ): + return False for tree in trees: if ( not isinstance(tree, dict) @@ -79,44 +87,61 @@ def _is_valid_flamegraph(payload: Any) -> bool: return True -def _expand_inline_frames(trees: list[dict[str, Any]], chains: dict[int, list[int]]) -> None: - nodes = [root for tree in trees for root in tree["roots"]] +def _expand_inline_frames( + trees: list[dict[str, Any]], chains: dict[int, list[int]] +) -> list[dict[str, Any]]: + output = [{**tree, "roots": []} for tree in trees] + nodes = [ + (root, new_tree["roots"]) + for tree, new_tree in zip(trees, output) + for root in reversed(tree["roots"]) + ] while nodes: - node = nodes.pop() - children = node.get("children") - nodes.extend(children or []) - chain = chains.get(node["frame_id"]) - if not chain or len(chain) == 1: - continue - node.pop("children", None) + original, siblings = nodes.pop() + node = dict(original) + siblings.append(node) + children = node.pop("children", None) current = node - for frame_id in chain[1:]: + for frame_id in chains.get(node["frame_id"], [])[1:]: child = {"frame_id": frame_id, "sample_count": node["sample_count"]} current["children"] = [child] current = child if children is not None: - current["children"] = children + current["children"] = [] + nodes.extend((child, current["children"]) for child in reversed(children)) + return output -def get_flamegraph_stacktraces( - payload: dict[str, Any], modules: list[dict[str, Any]] -) -> tuple[list[int], list[dict[str, Any]]]: +def get_flamegraph_stacktraces(payload: dict[str, Any]) -> tuple[list[int], list[dict[str, Any]]]: frame_ids = [] stacktraces = [] + original_modules = payload.get("debug_meta", {}).get("images", []) + module_indices: dict[int, int] = {} + for index, image in enumerate(original_modules): + if is_native_image(image): + module_indices[index] = len(module_indices) for frame_id, frame in enumerate(payload["frames"][:MAX_SYMBOLICATION_FRAMES]): - if not is_native_platform(frame.get("platform", payload["platform"])): + frames = get_frames_for_symbolication([frame], payload, original_modules, adjustment="none") + if not frames: continue - frames = get_frames_for_symbolication([frame], payload, modules, adjustment="none") - if frames: - frame_ids.append(frame_id) - stacktraces.append({"frames": frames}) + # Resolve against the captured image table before remapping to the filtered request. + addr_mode = frames[0].get("addr_mode") + if frame.get("addr_mode", "").startswith("rel:") and addr_mode is None: + continue + if addr_mode is not None: + module_index = module_indices.get(int(addr_mode[4:])) + if module_index is None: + continue + frames[0]["addr_mode"] = f"rel:{module_index}" + frame_ids.append(frame_id) + stacktraces.append({"frames": frames}) return frame_ids, stacktraces def _merge_symbolicated_frames( payload: dict[str, Any], frame_ids: list[int], stacktraces: list[dict[str, Any]] ) -> dict[str, Any]: - output = deepcopy(payload) + output = {**payload, "frames": list(payload["frames"])} chains = {} for frame_id, stacktrace in zip(frame_ids, stacktraces): original = payload["frames"][frame_id] @@ -127,7 +152,9 @@ def _merge_symbolicated_frames( continue chain = [] for position, frame in enumerate(resolved): - merged = deepcopy(original) + merged = dict(original) + if "data" in merged: + merged["data"] = dict(merged["data"]) merge_frame(merged, frame, original.get("platform", payload["platform"])) # Symbolicator may adjust addresses or normalize relative addressing modes. # The diagnostic must retain its captured symbolication inputs. @@ -143,7 +170,7 @@ def _merge_symbolicated_frames( chain.append(len(output["frames"])) output["frames"].append(merged) chains[frame_id] = chain - _expand_inline_frames(output["trees"], chains) + output["trees"] = _expand_inline_frames(payload["trees"], chains) output["_symbolication"] = { "status": "completed", "frames": payload["frames"], @@ -166,12 +193,15 @@ def get_flamegraphs(self) -> Generator[tuple[int, dict[str, Any]]]: for index, attachment in enumerate(self._attachments): if attachment.type != "event.flamegraph": continue - raw = attachment.load_data(self._project) + try: + raw = attachment.load_data(self._project) + except (MissingAttachmentChunks, FileNotFoundError): + continue if len(raw) > MAX_FLAMEGRAPH_SIZE: continue try: - payload = orjson.loads(raw) - except orjson.JSONDecodeError: + payload = rapidjson.loads(raw, allow_nan=False) + except (rapidjson.JSONDecodeError, UnicodeDecodeError): continue if not isinstance(payload, dict): continue @@ -197,15 +227,16 @@ def apply_response_and_save( if len(stacktraces) != len(frame_ids): return False output = _merge_symbolicated_frames(payload, frame_ids, stacktraces) - serialized = orjson.dumps(output) + serialized = rapidjson.dumps(output, ensure_ascii=False, allow_nan=False).encode() + if len(serialized) > MAX_FLAMEGRAPH_SIZE: + return False attachment = self._attachments[attachment_index] metadata = attachment.meta() metadata.update(data=serialized, chunks=None, size=len(serialized)) - self._attachments[attachment_index] = CachedAttachment(**metadata) - # Checkpoint before the next request, which may exhaust the event's timeout. + # Checkpoint only this attachment, without reading or rewriting unrelated payloads. + checkpoint = dict(self._event) store_attachments_for_event( - self._project, self._event, self._attachments, timeout=CACHE_TIMEOUT + self._project, checkpoint, [CachedAttachment(**metadata)], timeout=CACHE_TIMEOUT ) - # Reload metadata so later checkpoints don't rewrite earlier processed payloads. - self._attachments = list(get_attachments_for_event(self._event)) + self._event["_attachments"][attachment_index] = checkpoint["_attachments"][0] return True diff --git a/src/sentry/lang/native/processing.py b/src/sentry/lang/native/processing.py index e096b8736487..2bc02767493d 100644 --- a/src/sentry/lang/native/processing.py +++ b/src/sentry/lang/native/processing.py @@ -324,7 +324,7 @@ def process_native_stacktraces(symbolicator: Symbolicator, data: Any) -> Any: modules = native_images_from_data(payload) if not modules: continue - frame_ids, stacktraces = get_flamegraph_stacktraces(payload, modules) + frame_ids, stacktraces = get_flamegraph_stacktraces(payload) if not stacktraces: continue response = symbolicator.process_payload( diff --git a/tests/sentry/lang/native/test_processing.py b/tests/sentry/lang/native/test_processing.py index 477ab0b9feef..45a351b34c41 100644 --- a/tests/sentry/lang/native/test_processing.py +++ b/tests/sentry/lang/native/test_processing.py @@ -12,12 +12,19 @@ import orjson import pytest +import rapidjson from sentry.attachments import ( CachedAttachment, + MissingAttachmentChunks, get_attachments_for_event, store_attachments_for_event, ) +from sentry.lang.native.flamegraphs import ( + _is_valid_flamegraph, + _merge_symbolicated_frames, + get_flamegraph_stacktraces, +) from sentry.lang.native.frames import get_frames_for_symbolication from sentry.lang.native.processing import ( ELECTRON_FIRST_MODULE_REWRITE_RULES, @@ -107,7 +114,7 @@ def test_get_native_symbolication_functions( @django_db_all def test_process_native_flamegraph(cached_flamegraph) -> None: data, _, symbolicator = cached_flamegraph - payload = { + payload: dict[str, Any] = { "version": "1", "platform": "cocoa", "debug_meta": { @@ -246,8 +253,8 @@ def test_route_flamegraph_to_native_symbolication() -> None: @pytest.fixture -def cached_flamegraph(default_project): - payload = { +def flamegraph_payload(): + return { "version": "1", "platform": "cocoa", "frames": [{"instruction_addr": "0x100001000", "data": {"custom": "retained"}}], @@ -256,6 +263,11 @@ def cached_flamegraph(default_project): }, "trees": [{"roots": [{"frame_id": 0, "sample_count": 10}]}], } + + +@pytest.fixture +def cached_flamegraph(default_project, flamegraph_payload): + payload = flamegraph_payload data = { "platform": "other", "project": default_project.id, @@ -289,6 +301,70 @@ def cached_flamegraph(default_project): return data, payload, symbolicator +@pytest.mark.parametrize( + ("field", "value"), + [("data", "invalid"), ("data", None), ("addr_mode", 1), ("platform", [])], +) +def test_flamegraph_rejects_invalid_frame_fields(flamegraph_payload, field, value) -> None: + flamegraph_payload["frames"][0][field] = value + assert not _is_valid_flamegraph(flamegraph_payload) + + +@pytest.mark.parametrize("field", ["debug_id", "id", "uuid"]) +def test_flamegraph_rejects_invalid_image_identifiers(flamegraph_payload, field) -> None: + flamegraph_payload["debug_meta"]["images"][0][field] = [] + assert not _is_valid_flamegraph(flamegraph_payload) + + +@django_db_all +@pytest.mark.parametrize("error", [MissingAttachmentChunks(), FileNotFoundError()]) +def test_flamegraph_unavailable_data_does_not_block_event_symbolication( + cached_flamegraph, error +) -> None: + data, _, symbolicator = cached_flamegraph + data["platform"] = "native" + data["exception"] = {"values": [deepcopy(NATIVE_EXCEPTION)]} + symbolicator.process_payload.return_value = { + "status": "completed", + "modules": [], + "stacktraces": [ + {"frames": [{"original_index": 0, "function": "main", "status": "symbolicated"}]} + ], + } + with ( + Feature({"organizations:flamegraph-attachments": True}), + mock.patch.object(CachedAttachment, "load_data", side_effect=error), + ): + assert process_native_stacktraces(symbolicator, data) is data + assert data["exception"]["values"][0]["stacktrace"]["frames"][0]["function"] == "main" + + +@django_db_all +def test_flamegraph_invalid_frame_does_not_block_event_symbolication(cached_flamegraph) -> None: + data, payload, symbolicator = cached_flamegraph + payload["frames"][0]["data"] = "invalid" + raw = orjson.dumps(payload) + store_attachments_for_event( + symbolicator.project, + data, + [CachedAttachment(type="event.flamegraph", data=raw)], + timeout=60, + ) + data["platform"] = "native" + data["exception"] = {"values": [deepcopy(NATIVE_EXCEPTION)]} + symbolicator.process_payload.return_value = { + "status": "completed", + "modules": [], + "stacktraces": [ + {"frames": [{"original_index": 0, "function": "main", "status": "symbolicated"}]} + ], + } + with Feature({"organizations:flamegraph-attachments": True}): + assert process_native_stacktraces(symbolicator, data) is data + assert data["exception"]["values"][0]["stacktrace"]["frames"][0]["function"] == "main" + assert next(get_attachments_for_event(data)).load_data(symbolicator.project) == raw + + @django_db_all def test_flamegraph_feature_disabled(cached_flamegraph) -> None: data, payload, symbolicator = cached_flamegraph @@ -300,6 +376,99 @@ def test_flamegraph_feature_disabled(cached_flamegraph) -> None: ) +@django_db_all +@pytest.mark.parametrize("depth", [100, 600]) +def test_flamegraph_deep_inline_expansion(cached_flamegraph, depth) -> None: + data, payload, symbolicator = cached_flamegraph + node = {"frame_id": 0, "sample_count": 10, "custom": "retained"} + for _ in range(depth - 1): + node = {"frame_id": 0, "sample_count": 10, "children": [node], "custom": "retained"} + payload["trees"][0]["roots"] = [node] + raw = rapidjson.dumps(payload).encode() + store_attachments_for_event( + symbolicator.project, + data, + [CachedAttachment(type="event.flamegraph", data=raw)], + timeout=60, + ) + with Feature({"organizations:flamegraph-attachments": True}): + assert process_native_stacktraces(symbolicator, data) is data + processed = rapidjson.loads( + next(get_attachments_for_event(data)).load_data(symbolicator.project) + ) + assert processed["frames"][0]["function"] == "outer" + assert processed["frames"][1]["function"] == "inner" + node = processed["trees"][0]["roots"][0] + for _ in range(depth - 1): + assert node["frame_id"] == 0 + assert node["sample_count"] == 10 + assert node["custom"] == "retained" + inline = node["children"][0] + assert inline["frame_id"] == 1 + assert inline["sample_count"] == 10 + node = inline["children"][0] + assert node == { + "frame_id": 0, + "sample_count": 10, + "custom": "retained", + "children": [{"frame_id": 1, "sample_count": 10}], + } + assert rapidjson.dumps(processed["_symbolication"]["trees"]) == rapidjson.dumps( + payload["trees"] + ) + + +@django_db_all +def test_flamegraph_oversized_rewrite_preserves_original(cached_flamegraph) -> None: + data, payload, symbolicator = cached_flamegraph + payload["frames"][0]["custom"] = "x" * (6 * 1024 * 1024) + raw = orjson.dumps(payload) + store_attachments_for_event( + symbolicator.project, + data, + [CachedAttachment(type="event.flamegraph", data=raw)], + timeout=60, + ) + with Feature({"organizations:flamegraph-attachments": True}): + assert process_native_stacktraces(symbolicator, data) is None + assert next(get_attachments_for_event(data)).load_data(symbolicator.project) == raw + + +@pytest.mark.parametrize("addr_mode", ["rel:1", "rel:84a04d24-0e60-3810-a8c0-90a65e2df61a"]) +def test_flamegraph_relative_address_uses_filtered_module_index( + flamegraph_payload, addr_mode +) -> None: + flamegraph_payload["debug_meta"]["images"].insert(0, {"type": "macho", "image_addr": "0x2000"}) + flamegraph_payload["frames"][0]["addr_mode"] = addr_mode + original = deepcopy(flamegraph_payload) + frame_ids, stacktraces = get_flamegraph_stacktraces(flamegraph_payload) + assert frame_ids == [0] + assert stacktraces[0]["frames"][0]["addr_mode"] == "rel:0" + processed = _merge_symbolicated_frames( + flamegraph_payload, + frame_ids, + [{"frames": [{"status": "symbolicated", "function": "main"}]}], + ) + assert processed["frames"][0]["function"] == "main" + assert processed["frames"][0]["addr_mode"] == addr_mode + assert processed["frames"][0]["instruction_addr"] == "0x100001000" + assert processed["debug_meta"] == original["debug_meta"] + assert flamegraph_payload == original + + +@pytest.mark.parametrize( + "addr_mode", ["rel:0", "rel:2", "rel:00000000-0000-0000-0000-000000000001"] +) +def test_flamegraph_relative_address_without_matching_image_is_preserved( + flamegraph_payload, addr_mode +) -> None: + flamegraph_payload["debug_meta"]["images"].insert(0, {"type": "macho", "image_addr": "0x2000"}) + flamegraph_payload["frames"][0]["addr_mode"] = addr_mode + original = deepcopy(flamegraph_payload) + assert get_flamegraph_stacktraces(flamegraph_payload) == ([], []) + assert flamegraph_payload == original + + @django_db_all def test_flamegraph_frame_limit(cached_flamegraph) -> None: data, payload, symbolicator = cached_flamegraph @@ -344,7 +513,7 @@ def test_flamegraph_failure_preserves_attachment(cached_flamegraph, response) -> @django_db_all @pytest.mark.parametrize( - "raw", [b"invalid json", b"null", b'{"version":"2"}', b" " * (10 * 1024 * 1024 + 1)] + "raw", [b"invalid json", b"\xff", b"null", b'{"version":"2"}', b" " * (10 * 1024 * 1024 + 1)] ) def test_flamegraph_invalid_attachment(cached_flamegraph, raw) -> None: data, _, symbolicator = cached_flamegraph @@ -395,7 +564,7 @@ def test_flamegraph_invalid_tree_is_not_symbolicated(cached_flamegraph, root) -> @django_db_all -def test_flamegraph_reprocessing_uses_original_tables(cached_flamegraph) -> None: +def test_flamegraph_retry_uses_original_tables(cached_flamegraph) -> None: data, payload, symbolicator = cached_flamegraph with Feature({"organizations:flamegraph-attachments": True}): process_native_stacktraces(symbolicator, data) @@ -503,6 +672,26 @@ def test_flamegraph_multiple_attachments(multiple_flamegraphs) -> None: assert other_attachment.load_data(symbolicator.project) == b"unchanged" +@django_db_all +@pytest.mark.parametrize("missing_type", ["event.attachment", "event.flamegraph"]) +def test_flamegraph_checkpoint_does_not_read_missing_attachments( + multiple_flamegraphs, missing_type +) -> None: + data, _, _, symbolicator = multiple_flamegraphs + data["_attachments"][1]["type"] = missing_type + _, missing, _ = get_attachments_for_event(data) + missing.delete() + original_metadata = dict(data["_attachments"][1]) + with Feature({"organizations:flamegraph-attachments": True}): + assert process_native_stacktraces(symbolicator, data) is data + first, missing, second = get_attachments_for_event(data) + assert orjson.loads(first.load_data(symbolicator.project))["frames"][0]["function"] == "outer" + assert orjson.loads(second.load_data(symbolicator.project))["frames"][0]["function"] == "outer" + assert missing.meta() == original_metadata + with pytest.raises(MissingAttachmentChunks): + missing.load_data(symbolicator.project) + + @django_db_all @requires_objectstore def test_flamegraph_checkpoints_do_not_rewrite_processed_objects(multiple_flamegraphs) -> None: @@ -531,15 +720,13 @@ def test_flamegraph_checkpoints_do_not_rewrite_processed_objects(multiple_flameg @django_db_all def test_flamegraph_checkpoint_survives_later_timeout(multiple_flamegraphs) -> None: - from sentry.tasks.symbolication import SymbolicationTimeout - data, _, second, symbolicator = multiple_flamegraphs symbolicator.process_payload.side_effect = [ symbolicator.process_payload.return_value, - SymbolicationTimeout(), + TimeoutError(), ] with Feature({"organizations:flamegraph-attachments": True}): - with pytest.raises(SymbolicationTimeout): + with pytest.raises(TimeoutError): process_native_stacktraces(symbolicator, data) assert symbolicator.process_payload.call_count == 2 first_attachment, other_attachment, second_attachment = get_attachments_for_event(data) From 16ffcaf2c4d9454318ffb9c6a1db43e5e66d88ee Mon Sep 17 00:00:00 2001 From: Philip Niedertscheider Date: Thu, 15 Oct 2026 10:53:33 +0200 Subject: [PATCH 4/7] ref(native): Simplify flamegraph symbolication Keep the existing frame helpers in processing.py and build the image mapping in one place. Remove unused rerun snapshots while retaining completion tracking and attachment size checks. Cover large payloads without retaining duplicate original tables. --- src/sentry/lang/native/flamegraphs.py | 94 +------- src/sentry/lang/native/frames.py | 136 ------------ src/sentry/lang/native/processing.py | 232 ++++++++++++++++++-- tests/sentry/lang/native/test_processing.py | 67 +++--- 4 files changed, 249 insertions(+), 280 deletions(-) delete mode 100644 src/sentry/lang/native/frames.py diff --git a/src/sentry/lang/native/flamegraphs.py b/src/sentry/lang/native/flamegraphs.py index 16768e284a25..456b082c934f 100644 --- a/src/sentry/lang/native/flamegraphs.py +++ b/src/sentry/lang/native/flamegraphs.py @@ -11,8 +11,6 @@ store_attachments_for_event, ) from sentry.ingest.consumer import CACHE_TIMEOUT -from sentry.lang.native.frames import get_frames_for_symbolication, merge_frame -from sentry.lang.native.utils import is_native_image from sentry.models.project import Project MAX_FLAMEGRAPH_SIZE = 10 * 1024 * 1024 @@ -87,7 +85,7 @@ def _is_valid_flamegraph(payload: Any) -> bool: return True -def _expand_inline_frames( +def expand_inline_frames( trees: list[dict[str, Any]], chains: dict[int, list[int]] ) -> list[dict[str, Any]]: output = [{**tree, "roots": []} for tree in trees] @@ -112,73 +110,6 @@ def _expand_inline_frames( return output -def get_flamegraph_stacktraces(payload: dict[str, Any]) -> tuple[list[int], list[dict[str, Any]]]: - frame_ids = [] - stacktraces = [] - original_modules = payload.get("debug_meta", {}).get("images", []) - module_indices: dict[int, int] = {} - for index, image in enumerate(original_modules): - if is_native_image(image): - module_indices[index] = len(module_indices) - for frame_id, frame in enumerate(payload["frames"][:MAX_SYMBOLICATION_FRAMES]): - frames = get_frames_for_symbolication([frame], payload, original_modules, adjustment="none") - if not frames: - continue - # Resolve against the captured image table before remapping to the filtered request. - addr_mode = frames[0].get("addr_mode") - if frame.get("addr_mode", "").startswith("rel:") and addr_mode is None: - continue - if addr_mode is not None: - module_index = module_indices.get(int(addr_mode[4:])) - if module_index is None: - continue - frames[0]["addr_mode"] = f"rel:{module_index}" - frame_ids.append(frame_id) - stacktraces.append({"frames": frames}) - return frame_ids, stacktraces - - -def _merge_symbolicated_frames( - payload: dict[str, Any], frame_ids: list[int], stacktraces: list[dict[str, Any]] -) -> dict[str, Any]: - output = {**payload, "frames": list(payload["frames"])} - chains = {} - for frame_id, stacktrace in zip(frame_ids, stacktraces): - original = payload["frames"][frame_id] - resolved = stacktrace.get("frames", []) - if not resolved or any( - frame.get("status") not in (None, "symbolicated") for frame in resolved - ): - continue - chain = [] - for position, frame in enumerate(resolved): - merged = dict(original) - if "data" in merged: - merged["data"] = dict(merged["data"]) - merge_frame(merged, frame, original.get("platform", payload["platform"])) - # Symbolicator may adjust addresses or normalize relative addressing modes. - # The diagnostic must retain its captured symbolication inputs. - merged["instruction_addr"] = original["instruction_addr"] - if "addr_mode" in original: - merged["addr_mode"] = original["addr_mode"] - else: - merged.pop("addr_mode", None) - if position == 0: - output["frames"][frame_id] = merged - chain.append(frame_id) - else: - chain.append(len(output["frames"])) - output["frames"].append(merged) - chains[frame_id] = chain - output["trees"] = _expand_inline_frames(payload["trees"], chains) - output["_symbolication"] = { - "status": "completed", - "frames": payload["frames"], - "trees": payload["trees"], - } - return output - - class Flamegraphs: def __init__(self, project: Project, data: Any): self._project = project @@ -206,28 +137,13 @@ def get_flamegraphs(self) -> Generator[tuple[int, dict[str, Any]]]: if not isinstance(payload, dict): continue state = payload.get("_symbolication") - if isinstance(state, dict): - if state.get("status") == "completed": - continue - payload["frames"] = state.get("frames", payload.get("frames")) - payload["trees"] = state.get("trees", payload.get("trees")) + if isinstance(state, dict) and state.get("status") == "completed": + continue if _is_valid_flamegraph(payload): yield index, payload - def apply_response_and_save( - self, - attachment_index: int, - payload: dict[str, Any], - frame_ids: list[int], - response: Any, - ) -> bool: - if not response or response.get("status") != "completed": - return False - stacktraces = response.get("stacktraces", []) - if len(stacktraces) != len(frame_ids): - return False - output = _merge_symbolicated_frames(payload, frame_ids, stacktraces) - serialized = rapidjson.dumps(output, ensure_ascii=False, allow_nan=False).encode() + def save(self, attachment_index: int, payload: dict[str, Any]) -> bool: + serialized = rapidjson.dumps(payload, ensure_ascii=False, allow_nan=False).encode() if len(serialized) > MAX_FLAMEGRAPH_SIZE: return False attachment = self._attachments[attachment_index] diff --git a/src/sentry/lang/native/frames.py b/src/sentry/lang/native/frames.py deleted file mode 100644 index 0da7740049be..000000000000 --- a/src/sentry/lang/native/frames.py +++ /dev/null @@ -1,136 +0,0 @@ -import posixpath - -from symbolic.debuginfo import normalize_debug_id -from symbolic.exceptions import ParseDebugIdError - -from sentry.lang.native.utils import is_native_platform -from sentry.stacktraces.functions import trim_function_name -from sentry.utils.safe import get_path, trim - - -def merge_frame(new_frame, symbolicated, platform="native"): - # il2cpp events which have the "csharp" platform have good (C#) names - # coming from the SDK, we do not want to override those with bad (mangled) C++ names. - if platform != "csharp" and symbolicated.get("function"): - raw_func = trim(symbolicated["function"], 256) - func = trim(trim_function_name(symbolicated["function"], platform), 256) - - # if function and raw function match, we can get away without - # storing a raw function - if func == raw_func: - new_frame["function"] = raw_func - # otherwise we store both - else: - new_frame["raw_function"] = raw_func - new_frame["function"] = func - if symbolicated.get("instruction_addr"): - new_frame["instruction_addr"] = symbolicated["instruction_addr"] - if symbolicated.get("function_id"): - new_frame["function_id"] = symbolicated["function_id"] - if symbolicated.get("symbol"): - new_frame["symbol"] = symbolicated["symbol"] - if symbolicated.get("abs_path"): - new_frame["abs_path"] = symbolicated["abs_path"] - new_frame["filename"] = posixpath.basename(symbolicated["abs_path"]) - if symbolicated.get("filename"): - new_frame["filename"] = symbolicated["filename"] - if symbolicated.get("lineno"): - new_frame["lineno"] = symbolicated["lineno"] - if symbolicated.get("colno"): - new_frame["colno"] = symbolicated["colno"] - # similarly as with `function` above, we do want to retain the original "package". - if platform != "csharp" and symbolicated.get("package"): - new_frame["package"] = symbolicated["package"] - if symbolicated.get("trust"): - new_frame["trust"] = symbolicated["trust"] - if symbolicated.get("pre_context"): - new_frame["pre_context"] = symbolicated["pre_context"] - if symbolicated.get("context_line") is not None: - new_frame["context_line"] = symbolicated["context_line"] - if symbolicated.get("post_context"): - new_frame["post_context"] = symbolicated["post_context"] - if symbolicated.get("source_link"): - new_frame["source_link"] = symbolicated["source_link"] - if symbolicated.get("vars"): - new_frame["vars"] = symbolicated["vars"] - - addr_mode = symbolicated.get("addr_mode") - if addr_mode is None: - new_frame.pop("addr_mode", None) - else: - new_frame["addr_mode"] = addr_mode - - if symbolicated.get("status"): - frame_meta = new_frame.setdefault("data", {}) - frame_meta["symbolicator_status"] = symbolicated["status"] - - -def handles_frame(data, frame): - if not frame: - return False - - if get_path(frame, "data", "symbolicator_status") is not None: - return False - - # TODO: Consider ignoring platform - platform = frame.get("platform") or data.get("platform") - return is_native_platform(platform) and frame.get("instruction_addr") is not None - - -def get_frames_for_symbolication( - frames, - data, - modules, - adjustment=None, -): - modules_by_debug_id = None - rv = [] - adjustment = adjustment or "auto" - - for frame in reversed(frames): - if not handles_frame(data, frame): - continue - s_frame = dict(frame) - - if adjustment == "none": - s_frame["adjust_instruction_addr"] = False - - # validate and expand addressing modes. If we can't validate and - # expand it, we keep None which is absolute. That's not great but - # at least won't do damage. - addr_mode = s_frame.pop("addr_mode", None) - sanitized_addr_mode = None - - # None and abs mean absolute addressing explicitly. - if addr_mode in (None, "abs"): - pass - # this is relative addressing to module by index or debug id. - elif addr_mode.startswith("rel:"): - arg = addr_mode[4:] - idx = None - - if modules_by_debug_id is None: - modules_by_debug_id = {x.get("debug_id"): idx for idx, x in enumerate(modules)} - try: - idx = modules_by_debug_id.get(normalize_debug_id(arg)) - except ParseDebugIdError: - pass - - if idx is None and arg.isdigit(): - idx = int(arg) - - if idx is not None: - sanitized_addr_mode = "rel:%d" % idx - - if sanitized_addr_mode is not None: - s_frame["addr_mode"] = sanitized_addr_mode - rv.append(s_frame) - - if len(rv) > 0: - first_frame = rv[0] - if adjustment == "all": - first_frame["adjust_instruction_addr"] = True - elif adjustment == "all_but_first": - first_frame["adjust_instruction_addr"] = False - - return rv diff --git a/src/sentry/lang/native/processing.py b/src/sentry/lang/native/processing.py index 2bc02767493d..23976246fcbb 100644 --- a/src/sentry/lang/native/processing.py +++ b/src/sentry/lang/native/processing.py @@ -1,15 +1,21 @@ from __future__ import annotations import logging +import posixpath from collections.abc import Mapping from typing import Any import sentry_sdk +from symbolic.debuginfo import normalize_debug_id +from symbolic.exceptions import ParseDebugIdError from sentry import options from sentry.lang.native.error import SymbolicationFailed, write_error -from sentry.lang.native.flamegraphs import Flamegraphs, get_flamegraph_stacktraces -from sentry.lang.native.frames import get_frames_for_symbolication, handles_frame, merge_frame +from sentry.lang.native.flamegraphs import ( + MAX_SYMBOLICATION_FRAMES, + Flamegraphs, + expand_inline_frames, +) from sentry.lang.native.symbolicator import FrameOrder, Symbolicator, SymbolicatorFunction from sentry.lang.native.utils import ( get_event_attachment, @@ -19,16 +25,18 @@ is_applecrashreport_event, is_minidump_event, is_native_event, + is_native_image, is_native_platform, native_images_from_data, signal_from_data, ) from sentry.models.eventerror import EventErrorType from sentry.options.rollout import in_random_rollout +from sentry.stacktraces.functions import trim_function_name from sentry.stacktraces.processing import StacktraceInfo, find_stacktraces_in_data from sentry.utils import metrics from sentry.utils.in_app import is_known_third_party, is_optional_package -from sentry.utils.safe import get_path, set_path, setdefault_path +from sentry.utils.safe import get_path, set_path, setdefault_path, trim logger = logging.getLogger(__name__) @@ -75,6 +83,63 @@ ] +def _merge_frame(new_frame, symbolicated, platform="native"): + # il2cpp events which have the "csharp" platform have good (C#) names + # coming from the SDK, we do not want to override those with bad (mangled) C++ names. + if platform != "csharp" and symbolicated.get("function"): + raw_func = trim(symbolicated["function"], 256) + func = trim(trim_function_name(symbolicated["function"], platform), 256) + + # if function and raw function match, we can get away without + # storing a raw function + if func == raw_func: + new_frame["function"] = raw_func + # otherwise we store both + else: + new_frame["raw_function"] = raw_func + new_frame["function"] = func + if symbolicated.get("instruction_addr"): + new_frame["instruction_addr"] = symbolicated["instruction_addr"] + if symbolicated.get("function_id"): + new_frame["function_id"] = symbolicated["function_id"] + if symbolicated.get("symbol"): + new_frame["symbol"] = symbolicated["symbol"] + if symbolicated.get("abs_path"): + new_frame["abs_path"] = symbolicated["abs_path"] + new_frame["filename"] = posixpath.basename(symbolicated["abs_path"]) + if symbolicated.get("filename"): + new_frame["filename"] = symbolicated["filename"] + if symbolicated.get("lineno"): + new_frame["lineno"] = symbolicated["lineno"] + if symbolicated.get("colno"): + new_frame["colno"] = symbolicated["colno"] + # similarly as with `function` above, we do want to retain the original "package". + if platform != "csharp" and symbolicated.get("package"): + new_frame["package"] = symbolicated["package"] + if symbolicated.get("trust"): + new_frame["trust"] = symbolicated["trust"] + if symbolicated.get("pre_context"): + new_frame["pre_context"] = symbolicated["pre_context"] + if symbolicated.get("context_line") is not None: + new_frame["context_line"] = symbolicated["context_line"] + if symbolicated.get("post_context"): + new_frame["post_context"] = symbolicated["post_context"] + if symbolicated.get("source_link"): + new_frame["source_link"] = symbolicated["source_link"] + if symbolicated.get("vars"): + new_frame["vars"] = symbolicated["vars"] + + addr_mode = symbolicated.get("addr_mode") + if addr_mode is None: + new_frame.pop("addr_mode", None) + else: + new_frame["addr_mode"] = addr_mode + + if symbolicated.get("status"): + frame_meta = new_frame.setdefault("data", {}) + frame_meta["symbolicator_status"] = symbolicated["status"] + + def _handle_image_status(status, image, os, data): if status in ("found", "unused"): return @@ -257,7 +322,7 @@ def _merge_full_response(data, response): for complete_frame in reversed(complete_stacktrace["frames"]): new_frame: dict[str, Any] = {} - merge_frame(new_frame, complete_frame) + _merge_frame(new_frame, complete_frame) data_stacktrace["frames"].append(new_frame) @@ -317,14 +382,151 @@ def process_applecrashreport(symbolicator: Symbolicator, data: Any) -> Any: return data +def _handles_frame(data, frame): + if not frame: + return False + + if get_path(frame, "data", "symbolicator_status") is not None: + return False + + # TODO: Consider ignoring platform + platform = frame.get("platform") or data.get("platform") + return is_native_platform(platform) and frame.get("instruction_addr") is not None + + +def get_frames_for_symbolication( + frames, + data, + modules, + adjustment=None, +): + modules_by_debug_id = None + rv = [] + adjustment = adjustment or "auto" + + for frame in reversed(frames): + if not _handles_frame(data, frame): + continue + s_frame = dict(frame) + + if adjustment == "none": + s_frame["adjust_instruction_addr"] = False + + # validate and expand addressing modes. If we can't validate and + # expand it, we keep None which is absolute. That's not great but + # at least won't do damage. + addr_mode = s_frame.pop("addr_mode", None) + sanitized_addr_mode = None + + # None and abs mean absolute addressing explicitly. + if addr_mode in (None, "abs"): + pass + # this is relative addressing to module by index or debug id. + elif addr_mode.startswith("rel:"): + arg = addr_mode[4:] + idx = None + + if modules_by_debug_id is None: + modules_by_debug_id = {x.get("debug_id"): idx for idx, x in enumerate(modules)} + try: + idx = modules_by_debug_id.get(normalize_debug_id(arg)) + except ParseDebugIdError: + pass + + if idx is None and arg.isdigit(): + idx = int(arg) + + if idx is not None: + sanitized_addr_mode = "rel:%d" % idx + + if sanitized_addr_mode is not None: + s_frame["addr_mode"] = sanitized_addr_mode + rv.append(s_frame) + + if len(rv) > 0: + first_frame = rv[0] + if adjustment == "all": + first_frame["adjust_instruction_addr"] = True + elif adjustment == "all_but_first": + first_frame["adjust_instruction_addr"] = False + + return rv + + +def _get_flamegraph_stacktraces( + payload: dict[str, Any], +) -> tuple[list[dict[str, Any]], list[int], list[dict[str, Any]]]: + frame_ids = [] + stacktraces = [] + original_modules = payload.get("debug_meta", {}).get("images", []) + modules: list[dict[str, Any]] = [] + module_indices: dict[int, int] = {} + for index, image in enumerate(original_modules): + if is_native_image(image): + module_indices[index] = len(modules) + modules.append(image) + if not modules: + return [], [], [] + for frame_id, frame in enumerate(payload["frames"][:MAX_SYMBOLICATION_FRAMES]): + frames = get_frames_for_symbolication([frame], payload, original_modules, adjustment="none") + if not frames: + continue + # Resolve against the captured image table before remapping to the filtered request. + addr_mode = frames[0].get("addr_mode") + if frame.get("addr_mode", "").startswith("rel:") and addr_mode is None: + continue + if addr_mode is not None: + module_index = module_indices.get(int(addr_mode[4:])) + if module_index is None: + continue + frames[0]["addr_mode"] = f"rel:{module_index}" + frame_ids.append(frame_id) + stacktraces.append({"frames": frames}) + return modules, frame_ids, stacktraces + + +def _merge_flamegraph_frames( + payload: dict[str, Any], frame_ids: list[int], stacktraces: list[dict[str, Any]] +) -> dict[str, Any]: + output = {**payload, "frames": list(payload["frames"])} + chains = {} + for frame_id, stacktrace in zip(frame_ids, stacktraces): + original = payload["frames"][frame_id] + resolved = stacktrace.get("frames", []) + if not resolved or any( + frame.get("status") not in (None, "symbolicated") for frame in resolved + ): + continue + chain = [] + for position, frame in enumerate(resolved): + merged = dict(original) + if "data" in merged: + merged["data"] = dict(merged["data"]) + _merge_frame(merged, frame, original.get("platform", payload["platform"])) + # Symbolicator may adjust addresses or normalize relative addressing modes. + # The diagnostic must retain its captured symbolication inputs. + merged["instruction_addr"] = original["instruction_addr"] + if "addr_mode" in original: + merged["addr_mode"] = original["addr_mode"] + else: + merged.pop("addr_mode", None) + if position == 0: + output["frames"][frame_id] = merged + chain.append(frame_id) + else: + chain.append(len(output["frames"])) + output["frames"].append(merged) + chains[frame_id] = chain + output["trees"] = expand_inline_frames(payload["trees"], chains) + output["_symbolication"] = {"status": "completed"} + return output + + def process_native_stacktraces(symbolicator: Symbolicator, data: Any) -> Any: flamegraphs = Flamegraphs(symbolicator.project, data) attachments_changed = False for attachment_index, payload in flamegraphs.get_flamegraphs(): - modules = native_images_from_data(payload) - if not modules: - continue - frame_ids, stacktraces = get_flamegraph_stacktraces(payload) + modules, frame_ids, stacktraces = _get_flamegraph_stacktraces(payload) if not stacktraces: continue response = symbolicator.process_payload( @@ -334,9 +536,13 @@ def process_native_stacktraces(symbolicator: Symbolicator, data: Any) -> Any: frame_order=FrameOrder.caller_first, apply_source_context=False, ) - attachments_changed |= flamegraphs.apply_response_and_save( - attachment_index, payload, frame_ids, response - ) + if not response or response.get("status") != "completed": + continue + resolved_stacktraces = response.get("stacktraces", []) + if len(resolved_stacktraces) != len(frame_ids): + continue + output = _merge_flamegraph_frames(payload, frame_ids, resolved_stacktraces) + attachments_changed |= flamegraphs.save(attachment_index, output) stacktrace_infos = [ stacktrace for stacktrace in find_stacktraces_in_data(data) @@ -405,14 +611,14 @@ def process_native_stacktraces(symbolicator: Symbolicator, data: Any) -> Any: native_frames_idx = 0 for raw_frame in reversed(sinfo.stacktrace["frames"]): - if not handles_frame(data, raw_frame): + if not _handles_frame(data, raw_frame): new_frames.append(raw_frame) continue for complete_frame in complete_frames_by_idx.get(native_frames_idx) or (): merged_frame = dict(raw_frame) platform = merged_frame.get("platform") or data.get("platform") or "native" - merge_frame(merged_frame, complete_frame, platform) + _merge_frame(merged_frame, complete_frame, platform) if merged_frame.get("package"): raw_frame["package"] = merged_frame["package"] new_frames.append(merged_frame) diff --git a/tests/sentry/lang/native/test_processing.py b/tests/sentry/lang/native/test_processing.py index 45a351b34c41..1aa3ce31e1dc 100644 --- a/tests/sentry/lang/native/test_processing.py +++ b/tests/sentry/lang/native/test_processing.py @@ -20,15 +20,13 @@ get_attachments_for_event, store_attachments_for_event, ) -from sentry.lang.native.flamegraphs import ( - _is_valid_flamegraph, - _merge_symbolicated_frames, - get_flamegraph_stacktraces, -) -from sentry.lang.native.frames import get_frames_for_symbolication +from sentry.lang.native.flamegraphs import _is_valid_flamegraph from sentry.lang.native.processing import ( ELECTRON_FIRST_MODULE_REWRITE_RULES, + _get_flamegraph_stacktraces, + _merge_flamegraph_frames, _merge_image, + get_frames_for_symbolication, get_native_symbolication_functions, process_native_stacktraces, ) @@ -232,8 +230,6 @@ def test_process_native_flamegraph(cached_flamegraph) -> None: assert recursive["children"][0]["children"] == [{"frame_id": 1, "sample_count": 4}] assert processed["debug_meta"] == original["debug_meta"] assert processed["custom"] == "retained" - assert processed["_symbolication"]["frames"] == original["frames"] - assert processed["_symbolication"]["trees"] == original["trees"] @django_db_all @@ -413,9 +409,6 @@ def test_flamegraph_deep_inline_expansion(cached_flamegraph, depth) -> None: "custom": "retained", "children": [{"frame_id": 1, "sample_count": 10}], } - assert rapidjson.dumps(processed["_symbolication"]["trees"]) == rapidjson.dumps( - payload["trees"] - ) @django_db_all @@ -441,10 +434,11 @@ def test_flamegraph_relative_address_uses_filtered_module_index( flamegraph_payload["debug_meta"]["images"].insert(0, {"type": "macho", "image_addr": "0x2000"}) flamegraph_payload["frames"][0]["addr_mode"] = addr_mode original = deepcopy(flamegraph_payload) - frame_ids, stacktraces = get_flamegraph_stacktraces(flamegraph_payload) + modules, frame_ids, stacktraces = _get_flamegraph_stacktraces(flamegraph_payload) + assert modules == original["debug_meta"]["images"][1:] assert frame_ids == [0] assert stacktraces[0]["frames"][0]["addr_mode"] == "rel:0" - processed = _merge_symbolicated_frames( + processed = _merge_flamegraph_frames( flamegraph_payload, frame_ids, [{"frames": [{"status": "symbolicated", "function": "main"}]}], @@ -465,7 +459,10 @@ def test_flamegraph_relative_address_without_matching_image_is_preserved( flamegraph_payload["debug_meta"]["images"].insert(0, {"type": "macho", "image_addr": "0x2000"}) flamegraph_payload["frames"][0]["addr_mode"] = addr_mode original = deepcopy(flamegraph_payload) - assert get_flamegraph_stacktraces(flamegraph_payload) == ([], []) + modules, frame_ids, stacktraces = _get_flamegraph_stacktraces(flamegraph_payload) + assert modules == original["debug_meta"]["images"][1:] + assert frame_ids == [] + assert stacktraces == [] assert flamegraph_payload == original @@ -564,41 +561,27 @@ def test_flamegraph_invalid_tree_is_not_symbolicated(cached_flamegraph, root) -> @django_db_all -def test_flamegraph_retry_uses_original_tables(cached_flamegraph) -> None: +def test_flamegraph_large_payload_is_not_duplicated(cached_flamegraph) -> None: data, payload, symbolicator = cached_flamegraph - with Feature({"organizations:flamegraph-attachments": True}): - process_native_stacktraces(symbolicator, data) - processed = orjson.loads(next(get_attachments_for_event(data)).load_data(symbolicator.project)) - processed["_symbolication"]["status"] = "pending" + payload["frames"][0]["custom"] = "x" * (6 * 1024 * 1024) store_attachments_for_event( symbolicator.project, data, - [ - CachedAttachment( - type="event.flamegraph", - data=orjson.dumps(processed), - ) - ], + [CachedAttachment(type="event.flamegraph", data=orjson.dumps(payload))], timeout=60, ) + symbolicator.process_payload.return_value = { + "status": "completed", + "stacktraces": [{"frames": [{"status": "symbolicated", "function": "main"}]}], + } with Feature({"organizations:flamegraph-attachments": True}): - process_native_stacktraces(symbolicator, data) - processed = orjson.loads(next(get_attachments_for_event(data)).load_data(symbolicator.project)) - assert len(processed["frames"]) == 2 - assert processed["trees"] == [ - { - "roots": [ - { - "frame_id": 0, - "sample_count": 10, - "children": [{"frame_id": 1, "sample_count": 10}], - } - ] - } - ] - assert processed["_symbolication"]["frames"] == payload["frames"] - assert processed["_symbolication"]["trees"] == payload["trees"] - assert symbolicator.process_payload.call_count == 2 + assert process_native_stacktraces(symbolicator, data) is data + attachment = next(get_attachments_for_event(data)) + processed = orjson.loads(attachment.load_data(symbolicator.project)) + assert processed["frames"][0]["function"] == "main" + assert processed["frames"][0]["custom"] == payload["frames"][0]["custom"] + assert processed["trees"] == payload["trees"] + assert attachment.size <= 10 * 1024 * 1024 @django_db_all From e2b6666127b08b11bc600320e87c110e0b201378 Mon Sep 17 00:00:00 2001 From: Philip Niedertscheider Date: Thu, 15 Oct 2026 11:01:53 +0200 Subject: [PATCH 5/7] Update src/sentry/lang/native/flamegraphs.py --- src/sentry/lang/native/flamegraphs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sentry/lang/native/flamegraphs.py b/src/sentry/lang/native/flamegraphs.py index 456b082c934f..73dd3a5d1afb 100644 --- a/src/sentry/lang/native/flamegraphs.py +++ b/src/sentry/lang/native/flamegraphs.py @@ -13,6 +13,7 @@ from sentry.ingest.consumer import CACHE_TIMEOUT from sentry.models.project import Project +# See: https://develop.sentry.dev/sdk/telemetry/attachments/#flame-graph-validation-and-limits MAX_FLAMEGRAPH_SIZE = 10 * 1024 * 1024 MAX_SYMBOLICATION_FRAMES = 2500 MAX_SAMPLE_COUNT = 9007199254740991 From 9249338b72468e1d4f32638fd4a47673a47ec4b5 Mon Sep 17 00:00:00 2001 From: Philip Niedertscheider Date: Thu, 15 Oct 2026 11:31:04 +0200 Subject: [PATCH 6/7] test(native): Preserve the stored attachment ID type Use the typed objectstore ID directly in flamegraph assertions. --- .../ingest_consumer/test_ingest_consumer_processing.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/sentry/ingest/ingest_consumer/test_ingest_consumer_processing.py b/tests/sentry/ingest/ingest_consumer/test_ingest_consumer_processing.py index da7a2520b447..66bc236deeff 100644 --- a/tests/sentry/ingest/ingest_consumer/test_ingest_consumer_processing.py +++ b/tests/sentry/ingest/ingest_consumer/test_ingest_consumer_processing.py @@ -789,7 +789,8 @@ def do_process_flamegraph(default_project, task_runner, live_server, use_objects } if use_objectstore: session = get_session(UsecaseId.ATTACHMENTS, default_project) - attachment_metadata["stored_id"] = session.put(raw) + stored_id = session.put(raw) + attachment_metadata["stored_id"] = stored_id else: process_attachment_chunk( { @@ -832,8 +833,8 @@ def do_process_flamegraph(default_project, task_runner, live_server, use_objects assert processed["_symbolication"]["status"] == "completed" assert attachment.size == len(orjson.dumps(processed)) if use_objectstore: - assert attachment.blob_path == V2_PREFIX + attachment_metadata["stored_id"] - stored = session.get(attachment_metadata["stored_id"]) + assert attachment.blob_path == V2_PREFIX + stored_id + stored = session.get(stored_id) assert stored is not None assert orjson.loads(stored.payload.read()) == processed From 6fd4842f89ae841d171fb62ce83b9cb03391ddf2 Mon Sep 17 00:00:00 2001 From: Philip Niedertscheider Date: Thu, 15 Oct 2026 16:11:47 +0200 Subject: [PATCH 7/7] fix(native): Isolate flamegraph symbolication failures and metadata --- src/sentry/lang/native/processing.py | 23 +++--- tests/sentry/lang/native/test_processing.py | 77 ++++++++++++++++++++- 2 files changed, 89 insertions(+), 11 deletions(-) diff --git a/src/sentry/lang/native/processing.py b/src/sentry/lang/native/processing.py index 23976246fcbb..1db6a97aaf32 100644 --- a/src/sentry/lang/native/processing.py +++ b/src/sentry/lang/native/processing.py @@ -6,6 +6,7 @@ from typing import Any import sentry_sdk +from requests.exceptions import RequestException from symbolic.debuginfo import normalize_debug_id from symbolic.exceptions import ParseDebugIdError @@ -499,7 +500,9 @@ def _merge_flamegraph_frames( continue chain = [] for position, frame in enumerate(resolved): - merged = dict(original) + merged = dict(original) if position == 0 else {} + if "platform" in original: + merged["platform"] = original["platform"] if "data" in merged: merged["data"] = dict(merged["data"]) _merge_frame(merged, frame, original.get("platform", payload["platform"])) @@ -529,13 +532,17 @@ def process_native_stacktraces(symbolicator: Symbolicator, data: Any) -> Any: modules, frame_ids, stacktraces = _get_flamegraph_stacktraces(payload) if not stacktraces: continue - response = symbolicator.process_payload( - platform=payload["platform"], - stacktraces=stacktraces, - modules=modules, - frame_order=FrameOrder.caller_first, - apply_source_context=False, - ) + try: + response = symbolicator.process_payload( + platform=payload["platform"], + stacktraces=stacktraces, + modules=modules, + frame_order=FrameOrder.caller_first, + apply_source_context=False, + ) + except (OSError, RequestException) as error: + sentry_sdk.capture_exception(error) + continue if not response or response.get("status") != "completed": continue resolved_stacktraces = response.get("stacktraces", []) diff --git a/tests/sentry/lang/native/test_processing.py b/tests/sentry/lang/native/test_processing.py index 1aa3ce31e1dc..453261d788a6 100644 --- a/tests/sentry/lang/native/test_processing.py +++ b/tests/sentry/lang/native/test_processing.py @@ -13,6 +13,7 @@ import orjson import pytest import rapidjson +from requests.exceptions import ConnectionError, Timeout from sentry.attachments import ( CachedAttachment, @@ -312,6 +313,61 @@ def test_flamegraph_rejects_invalid_image_identifiers(flamegraph_payload, field) assert not _is_valid_flamegraph(flamegraph_payload) +def test_flamegraph_inline_frames_do_not_inherit_captured_metadata(flamegraph_payload) -> None: + original = flamegraph_payload["frames"][0] + original.update( + filename="captured.cpp", + lineno=42, + function="captured", + addr_mode="rel:0", + platform="native", + ) + before = deepcopy(flamegraph_payload) + output = _merge_flamegraph_frames( + flamegraph_payload, + [0], + [{"frames": [{"status": "symbolicated"}, {"status": "symbolicated"}]}], + ) + assert output["frames"][0]["filename"] == "captured.cpp" + assert output["frames"][0]["data"]["custom"] == "retained" + inline = output["frames"][1] + assert inline["instruction_addr"] == original["instruction_addr"] + assert inline["addr_mode"] == "rel:0" + assert inline["platform"] == "native" + assert "filename" not in inline + assert "lineno" not in inline + assert "function" not in inline + assert "custom" not in inline["data"] + assert flamegraph_payload == before + + +@django_db_all +@pytest.mark.parametrize("error", [TimeoutError(), Timeout(), ConnectionError()]) +def test_flamegraph_transport_error_does_not_block_event_symbolication( + cached_flamegraph, error +) -> None: + data, payload, symbolicator = cached_flamegraph + data["platform"] = "native" + data["exception"] = {"values": [deepcopy(NATIVE_EXCEPTION)]} + symbolicator.process_payload.side_effect = [ + error, + { + "status": "completed", + "modules": [], + "stacktraces": [ + {"frames": [{"original_index": 0, "function": "main", "status": "symbolicated"}]} + ], + }, + ] + with Feature({"organizations:flamegraph-attachments": True}): + assert process_native_stacktraces(symbolicator, data) is data + assert symbolicator.process_payload.call_count == 2 + assert data["exception"]["values"][0]["stacktrace"]["frames"][0]["function"] == "main" + assert next(get_attachments_for_event(data)).load_data(symbolicator.project) == orjson.dumps( + payload + ) + + @django_db_all @pytest.mark.parametrize("error", [MissingAttachmentChunks(), FileNotFoundError()]) def test_flamegraph_unavailable_data_does_not_block_event_symbolication( @@ -412,7 +468,7 @@ def test_flamegraph_deep_inline_expansion(cached_flamegraph, depth) -> None: @django_db_all -def test_flamegraph_oversized_rewrite_preserves_original(cached_flamegraph) -> None: +def test_flamegraph_large_frame_metadata_is_not_duplicated(cached_flamegraph) -> None: data, payload, symbolicator = cached_flamegraph payload["frames"][0]["custom"] = "x" * (6 * 1024 * 1024) raw = orjson.dumps(payload) @@ -423,7 +479,23 @@ def test_flamegraph_oversized_rewrite_preserves_original(cached_flamegraph) -> N timeout=60, ) with Feature({"organizations:flamegraph-attachments": True}): + assert process_native_stacktraces(symbolicator, data) is data + processed = orjson.loads(next(get_attachments_for_event(data)).load_data(symbolicator.project)) + assert processed["frames"][0]["custom"] == payload["frames"][0]["custom"] + assert "custom" not in processed["frames"][1] + assert processed["_symbolication"]["status"] == "completed" + + +@django_db_all +def test_flamegraph_oversized_rewrite_preserves_original(cached_flamegraph) -> None: + data, payload, symbolicator = cached_flamegraph + raw = orjson.dumps(payload) + with ( + Feature({"organizations:flamegraph-attachments": True}), + mock.patch("sentry.lang.native.flamegraphs.MAX_FLAMEGRAPH_SIZE", len(raw)), + ): assert process_native_stacktraces(symbolicator, data) is None + symbolicator.process_payload.assert_called_once() assert next(get_attachments_for_event(data)).load_data(symbolicator.project) == raw @@ -709,8 +781,7 @@ def test_flamegraph_checkpoint_survives_later_timeout(multiple_flamegraphs) -> N TimeoutError(), ] with Feature({"organizations:flamegraph-attachments": True}): - with pytest.raises(TimeoutError): - process_native_stacktraces(symbolicator, data) + assert process_native_stacktraces(symbolicator, data) is data assert symbolicator.process_payload.call_count == 2 first_attachment, other_attachment, second_attachment = get_attachments_for_event(data) first_processed = orjson.loads(first_attachment.load_data(symbolicator.project))