From 30a1d08e0fd68034192e555c2f20b38daf6ef4c1 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 19 Nov 2026 19:16:51 +0100 Subject: [PATCH] feat: Setup checklist for agents --- MANIFEST.in | 1 + sentry_sdk/SETUP_CHECKLIST.md | 47 ++++++++ sentry_sdk/_init_implementation.py | 134 ++++++++++++++++++++++- sentry_sdk/utils.py | 33 +++--- setup.py | 2 +- tests/conftest.py | 3 + tests/test_client.py | 166 +++++++++++++++++++++++++++++ 7 files changed, 370 insertions(+), 16 deletions(-) create mode 100644 sentry_sdk/SETUP_CHECKLIST.md diff --git a/MANIFEST.in b/MANIFEST.in index 58948bc839..0c88186053 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ include LICENSE include sentry_sdk/py.typed +include sentry_sdk/SETUP_CHECKLIST.md graft tests diff --git a/sentry_sdk/SETUP_CHECKLIST.md b/sentry_sdk/SETUP_CHECKLIST.md new file mode 100644 index 0000000000..51ac498bbd --- /dev/null +++ b/sentry_sdk/SETUP_CHECKLIST.md @@ -0,0 +1,47 @@ +# Sentry Python SDK: setup checklist + +This is Sentry's official setup guidance, shipped in the `sentry-sdk` package and maintained by Sentry with the SDK. It matches the agent setup guide at https://docs.sentry.io/ai/agent-setup/get-started.md. + +## Required + +- **Errors.** This is Sentry's core product. Set it up unless the user asked you not to. It needs a DSN: if the user hasn't given you one and you can't get one yourself, ask for it. Read it from an environment variable (`SENTRY_DSN`) and don't commit it. Before you finish, send a test error and check that it arrives. + +## Recommended + +Sentry links errors, traces and logs, so an error shows the request that caused it and the logs around it. Tracing and logs send more data and count toward the user's Sentry quota, so explain what each one adds and ask the user before you turn it on. + +- **Tracing.** Shows which request caused an error and where the time went. Set `traces_sample_rate` by environment: `1.0` locally, so you can see a trace when you verify, and a low rate in production (`0.1` is a common start) unless the project already has one. Don't default it to `0`: a rate of `0` sends no traces at all. +- **Logs.** Set `enable_logs=True`. The app's logs show up next to the errors they belong to. +- **Environment and release.** Set `environment` so production issues are separate from local ones. The SDK reads the release from `SENTRY_RELEASE` or the git commit. If the app runs without `.git` (for example in a Docker image), set `SENTRY_RELEASE` at build time or pass the app's version as `release`, so you can see which deploy caused an issue. + +To tell production from local runs, use a check the app already has, such as its debug flag or its environment setting. Don't add a new variable that nothing sets. + +```python +import os +import sentry_sdk + +from myapp import settings + +is_production = not settings.DEBUG + +sentry_sdk.init( + dsn=os.environ.get("SENTRY_DSN"), + environment="production" if is_production else "development", + traces_sample_rate=0.1 if is_production else 1.0, + enable_logs=True, +) +``` + +## Situational + +Do the Required and Recommended items first. Then look at the ones below. Set them up only when the project matches. + +- **Crons**, if the app schedules jobs with Celery beat, RQ (rq-scheduler), arq or Huey. Sentry alerts when a job fails or doesn't run, and every scheduled job gets a monitor, including ones added later. For Celery beat, pass `CeleryIntegration(monitor_beat_tasks=True)`. For RQ or arq, pass `monitor_cron_jobs=True` to `RqIntegration` or `ArqIntegration`. For Huey, pass `HueyIntegration(monitor_periodic_tasks=True)`. +- **AI monitoring**, if the app calls an LLM. Most AI libraries are traced automatically. For LiteLLM or Mistral, add `LiteLLMIntegration()` or `MistralIntegration()`. Prompts and responses are only recorded with `send_default_pii=True`, so ask the user before you set it. +- **Caches**, if the app uses the Django cache or Redis as a cache. For Django, pass `cache_spans=True` to `DjangoIntegration`. For Redis, pass the key prefixes of the cache to `RedisIntegration`, as in `cache_prefixes=["mycache:"]`. +- **Profiling**, if the user cares about the performance of specific code paths. Set `profile_session_sample_rate` and `profile_lifecycle="trace"`. +- **Metrics**, if the user wants to track counts or values over time (signups, queue depth). Call `sentry_sdk.metrics.count(...)`, `gauge(...)` or `distribution(...)` where they happen. + +## Before you finish + +Tell the user what you set up, the trace sample rate you chose for each environment, whether logs are on, and which situational items you included and why. diff --git a/sentry_sdk/_init_implementation.py b/sentry_sdk/_init_implementation.py index a4de802ae4..aa483adaec 100644 --- a/sentry_sdk/_init_implementation.py +++ b/sentry_sdk/_init_implementation.py @@ -1,10 +1,22 @@ +import atexit +import inspect +import os +import sys +import time import warnings from typing import TYPE_CHECKING import sentry_sdk +from sentry_sdk.utils import ( + _git, + _is_external_source, + capture_internal_exceptions, + env_to_bool, +) if TYPE_CHECKING: - from typing import Any, ContextManager, Optional + from types import FrameType + from typing import Any, ContextManager, Dict, Optional import sentry_sdk.consts @@ -48,6 +60,121 @@ def _check_python_deprecations() -> None: pass +_coding_agent_hint_shown = False + + +def _running_under_coding_agent() -> bool: + if os.environ.get("CI"): + return False + if env_to_bool(os.environ.get("SENTRY_AGENT_HINTS"), strict=True) is False: + return False + + ai_agent = os.environ.get("AI_AGENT") + if ai_agent: + return env_to_bool(ai_agent, strict=True) is not False + + # AI_AGENT is the cross-tool convention. These cover agents that don't set it. + return any( + os.environ.get(name) + for name in ( + "CLAUDECODE", + "CODEX_THREAD_ID", + "CODEX_SANDBOX", + "CURSOR_AGENT", + "GEMINI_CLI", + ) + ) + + +def _changed_by_this_user(init_file: str) -> bool: + """Whether the recent change to the file that calls init() was made here. + + A checkout or pull also gives the file a new mtime. + """ + + def git(*args: str) -> "Optional[str]": + # Read only: don't take the index lock (an agent may be running git at the + # same time) and don't run a configured fsmonitor command. + return _git( + "--no-optional-locks", + "-c", + "core.fsmonitor=false", + *args, + cwd=os.path.dirname(init_file), + timeout=2, + ) + + status = git("status", "--porcelain", "--", init_file) + if status is None: + cutoff = time.time() - 10 * 60 + try: + return os.stat(init_file).st_mtime >= cutoff + except OSError: + return False + if status: + return True + + # The file is clean, so only count a commit that was made here. + since = int(time.time() - 60 * 60) + committer = git("log", "-1", f"--since={since}", "--format=%ce", "--", init_file) + return bool(committer) and committer == git("config", "user.email") + + +def _write_coding_agent_hint(init_file: str, off: str) -> None: + with capture_internal_exceptions(): + if not _changed_by_this_user(init_file): + return + + checklist = os.path.join(os.path.dirname(__file__), "SETUP_CHECKLIST.md") + sys.stdout.flush() + sys.stderr.write( + f"[sentry] {off} If you're setting up Sentry, the optional settings are " + f"listed in {checklist}. Check with the user before enabling them. " + "(hide with SENTRY_AGENT_HINTS=0)\n" + ) + sys.stderr.flush() + + +def show_coding_agent_hint(options: "Dict[str, Any]", caller: "FrameType") -> None: + """Tell a coding agent which Sentry features are off while Sentry is being set up.""" + global _coding_agent_hint_shown + + rate = options["traces_sample_rate"] + tracing_on = options["traces_sampler"] is not None or rate not in (0, None) + logs_on = bool(options["enable_logs"]) + if ( + _coding_agent_hint_shown + or sys.platform not in ("darwin", "linux") + or not options["dsn"] + or (tracing_on and logs_on) + or not _running_under_coding_agent() + ): + return + + # A script's own path is relative on Python 3.8 and older. + init_file = os.path.abspath(caller.f_code.co_filename) + # An installed tool that uses Sentry itself isn't the user's app. + if _is_external_source(init_file): + return + try: + if os.stat(init_file).st_mtime < time.time() - 60 * 60: + return + except OSError: + return + + if not tracing_on and not logs_on: + off = "Tracing and logs are off, so Sentry only gets errors." + elif not tracing_on: + off = "Tracing is off, so Sentry gets errors and logs but no traces." + else: + off = "Logs are off, so Sentry gets errors and traces but no logs." + + _coding_agent_hint_shown = True + # Agents often cut a command's output with `| tail`, so the hint goes last. + # Git runs at exit, not here, so it doesn't slow down startup. + atexit.register(_write_coding_agent_hint, init_file, off) + + def _init(*args: "Optional[str]", **kwargs: "Any") -> "ContextManager[Any]": """Initializes the SDK and optionally integrations. @@ -55,6 +182,11 @@ def _init(*args: "Optional[str]", **kwargs: "Any") -> "ContextManager[Any]": """ client = sentry_sdk.Client(*args, **kwargs) sentry_sdk.get_global_scope().set_client(client) + with capture_internal_exceptions(): + # The frame that called sentry_sdk.init(). None on Pythons without frames. + frame = inspect.currentframe() + if frame is not None and frame.f_back is not None: + show_coding_agent_hint(client.options, frame.f_back) _check_python_deprecations() rv = _InitGuard(client) return rv diff --git a/sentry_sdk/utils.py b/sentry_sdk/utils.py index b1acf95bc5..ddf0bc5493 100644 --- a/sentry_sdk/utils.py +++ b/sentry_sdk/utils.py @@ -128,7 +128,10 @@ def json_dumps(data: "Any") -> bytes: return json.dumps(data, allow_nan=False, separators=(",", ":")).encode("utf-8") -def get_git_revision() -> "Optional[str]": +def _git( + *args: str, cwd: "Optional[str]" = None, timeout: "Optional[float]" = None +) -> "Optional[str]": + """A git command's output, or None when git has no answer.""" try: with open(os.path.devnull, "w+") as null: # prevent command prompt windows from popping up on windows @@ -137,22 +140,24 @@ def get_git_revision() -> "Optional[str]": startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW - revision = ( - subprocess.Popen( - ["git", "rev-parse", "HEAD"], - startupinfo=startupinfo, - stdout=subprocess.PIPE, - stderr=null, - stdin=null, - ) - .communicate()[0] - .strip() - .decode("utf-8") + result = subprocess.run( + ["git", *args], + cwd=cwd, + startupinfo=startupinfo, + stdout=subprocess.PIPE, + stderr=null, + stdin=null, + universal_newlines=True, + timeout=timeout, ) - except (OSError, IOError, FileNotFoundError): + except (OSError, subprocess.SubprocessError): return None - return revision + return result.stdout.strip() if result.returncode == 0 else None + + +def get_git_revision() -> "Optional[str]": + return _git("rev-parse", "HEAD") def get_default_release() -> "Optional[str]": diff --git a/setup.py b/setup.py index 407c4ed5a3..39572cfcb8 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ def get_file_text(file_name): long_description_content_type="text/markdown", packages=find_packages(exclude=("tests", "tests.*")), # PEP 561 - package_data={"sentry_sdk": ["py.typed"]}, + package_data={"sentry_sdk": ["py.typed", "SETUP_CHECKLIST.md"]}, zip_safe=False, license_expression="MIT", python_requires=">=3.6", diff --git a/tests/conftest.py b/tests/conftest.py index b75c2868f3..5b2e151e10 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -146,6 +146,9 @@ SENTRY_EVENT_SCHEMA = "./checkouts/data-schemas/relay/event.schema.json" +# The suite may itself run under a coding agent. The hint tests turn hints back on. +os.environ["SENTRY_AGENT_HINTS"] = "0" + if not os.path.isfile(SENTRY_EVENT_SCHEMA): SENTRY_EVENT_SCHEMA = None else: diff --git a/tests/test_client.py b/tests/test_client.py index 207d6b7590..04107c4354 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,6 +1,7 @@ import contextlib import json import os +import shutil import subprocess import sys import time @@ -1841,3 +1842,168 @@ async def test_async_socks_proxy(testcase): ) await client.close_async() + + +@pytest.fixture +def run_app(tmp_path, monkeypatch, capsys): + """Run a script that calls sentry_sdk.init() and return what the SDK prints at exit.""" + for name in ( + "CI", + "SENTRY_AGENT_HINTS", + "AI_AGENT", + "CLAUDECODE", + "CODEX_THREAD_ID", + "CODEX_SANDBOX", + "CURSOR_AGENT", + "GEMINI_CLI", + ): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr( + sentry_sdk._init_implementation, "_coding_agent_hint_shown", False + ) + at_exit = [] + monkeypatch.setattr( + sentry_sdk._init_implementation.atexit, + "register", + lambda hook, *args: at_exit.append((hook, args)), + ) + app = tmp_path / "app.py" + + def run(env, init_args="", before_run=None): + for name, value in env.items(): + monkeypatch.setenv(name, value) + app.write_text( + f"import sentry_sdk\nsentry_sdk.init(dsn='https://public@sentry.example.com/1', {init_args})\n" + ) + if before_run: + before_run(app) + + try: + exec(compile(app.read_text(), str(app), "exec"), {}) + for hook, args in at_exit: + if hook is sentry_sdk._init_implementation._write_coding_agent_hint: + hook(*args) + finally: + sentry_sdk.get_client().close() + return capsys.readouterr().err + + return run + + +@pytest.mark.parametrize( + "env, init_args", + [ + ({}, "traces_sample_rate=0"), + ({"AI_AGENT": "claude-code"}, "traces_sample_rate=0.1, enable_logs=True"), + ({"AI_AGENT": "claude-code"}, "traces_sampler=lambda _: 1.0, enable_logs=True"), + ({"AI_AGENT": "0"}, "traces_sample_rate=0"), + ({"AI_AGENT": "claude-code", "CI": "true"}, "traces_sample_rate=0"), + ({"CLAUDECODE": "1", "SENTRY_AGENT_HINTS": "0"}, "traces_sample_rate=0"), + ], +) +def test_coding_agent_hint_not_shown(run_app, env, init_args): + assert "[sentry]" not in run_app(env, init_args) + + +@pytest.mark.parametrize( + "env, init_args, off", + [ + ( + {"CLAUDECODE": "1"}, + "", + "Tracing and logs are off, so Sentry only gets errors.", + ), + ( + {"CODEX_THREAD_ID": "1"}, + "enable_logs=True", + "Tracing is off, so Sentry gets errors and logs but no traces.", + ), + ( + {"AI_AGENT": "codex"}, + "traces_sample_rate=0.1", + "Logs are off, so Sentry gets errors and traces but no logs.", + ), + ], +) +def test_coding_agent_hint_says_what_is_off(run_app, env, init_args, off): + assert f"[sentry] {off} " in run_app(env, init_args) + + +def _agent_hint_git(app, *args, committer="dev@example.com", date=None): + env = { + **os.environ, + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_SYSTEM": os.devnull, + "GIT_AUTHOR_NAME": "Dev", + "GIT_AUTHOR_EMAIL": committer, + "GIT_COMMITTER_NAME": "Dev", + "GIT_COMMITTER_EMAIL": committer, + } + if date: + env["GIT_AUTHOR_DATE"] = env["GIT_COMMITTER_DATE"] = date + subprocess.run( + ["git", *args], + cwd=str(app.parent), + env=env, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + +def _commit(app, **kwargs): + _agent_hint_git(app, "init", "-q") + _agent_hint_git(app, "config", "user.email", "dev@example.com") + _agent_hint_git(app, "add", app.name) + _agent_hint_git(app, "commit", "-q", "-m", "Set up Sentry", **kwargs) + + +def _changed_minutes_ago(app, minutes): + changed = time.time() - minutes * 60 + os.utime(app, (changed, changed)) + + +def _committed_and_edited(app, minutes_ago=0): + _commit(app, date="2020-01-01T00:00:00") + app.write_text(app.read_text() + "\n") + _changed_minutes_ago(app, minutes_ago) + + +@pytest.mark.skipif(shutil.which("git") is None, reason="needs git") +@pytest.mark.parametrize( + "before_run, shown", + [ + pytest.param(None, True, id="new file, no git repo"), + pytest.param( + lambda app: _changed_minutes_ago(app, 20), + False, + id="no git repo, changed 20 minutes ago", + ), + pytest.param(_committed_and_edited, True, id="uncommitted change"), + pytest.param( + lambda app: _committed_and_edited(app, minutes_ago=20), + True, + id="uncommitted change, 20 minutes ago", + ), + pytest.param( + lambda app: _committed_and_edited(app, minutes_ago=120), + False, + id="uncommitted change, two hours ago", + ), + pytest.param(_commit, True, id="committed here a moment ago"), + pytest.param( + lambda app: _commit(app, committer="teammate@example.com"), + False, + id="a teammate's new commit", + ), + pytest.param( + lambda app: _commit(app, date="2020-01-01T00:00:00"), + False, + id="fresh checkout of an old commit", + ), + ], +) +def test_coding_agent_hint_only_during_setup(run_app, before_run, shown): + printed = run_app({"AI_AGENT": "claude-code"}, before_run=before_run) + + assert ("SETUP_CHECKLIST.md" in printed) is shown