diff --git a/datadog_sync/utils/configuration.py b/datadog_sync/utils/configuration.py index e274a46f..31c92bf8 100644 --- a/datadog_sync/utils/configuration.py +++ b/datadog_sync/utils/configuration.py @@ -189,15 +189,65 @@ async def exit_async(self): await self.destination_client._end_session() +# Regex metacharacters that, when unescaped in an ExactMatch body, mean the +# value is a genuine regex rather than a regex-escaped literal id. Mirrors the +# metacharacter set escaped by Go's regexp.QuoteMeta (and a subset of the +# non-alphanumerics escaped by Python's re.escape). +_REGEX_METACHARS = frozenset(r".^$*+?()[]{}|") + + +def _regex_literal_from_exact_match_body(body: str) -> str: + """Recover the literal string an ExactMatch pattern body matches exactly. + + ExactMatch wraps the filter Value in ``^...$`` and compiles it as a regex, + so producers must regex-escape metacharacters in the Value for the match to + be literal (e.g. Go ``regexp.QuoteMeta`` / Python ``re.escape`` turn the + metric id ``svc.request.count`` into ``svc\\.request\\.count``). The + ID-targeted state load reuses the Value as a literal storage key, so the + escaping must be reversed here — otherwise the escaped body is looked up as + a key that never matches the real (unescaped) blob, silently dropping the + resource from state. + + Raises ValueError when the body is not a pure literal — an unescaped + metacharacter, a semantic or ambiguous alphanumeric escape, or a dangling + backslash. Callers treat that as "not ID-targetable" and fall back to + type-scoped loading, which matches via the compiled regex and stays correct. + """ + out = [] + i = 0 + n = len(body) + while i < n: + c = body[i] + if c == "\\": + if i + 1 >= n: + raise ValueError(f"dangling escape in ExactMatch pattern body: {body!r}") + escaped = body[i + 1] + # Alphanumeric escapes can change regex semantics (for example + # \d, \w, \x41, \n, and backreferences). Without the original + # unescaped value, they cannot safely be converted to a storage ID. + if escaped.isalnum(): + raise ValueError(f"semantic or ambiguous escape \\{escaped} in ExactMatch pattern body: {body!r}") + out.append(escaped) + i += 2 + continue + if c in _REGEX_METACHARS: + raise ValueError(f"unescaped metacharacter {c!r} in ExactMatch pattern body: {body!r}") + out.append(c) + i += 1 + return "".join(out) + + def _unwrap_exact_match_pattern(pattern: str) -> str: - """Extract the raw ID value from an ExactMatch ^...$-wrapped regex pattern. + """Extract the literal ID value from an ExactMatch ^...$-wrapped regex pattern. - Defensive check: ExactMatch always produces ^...$, so ValueError should not - fire in practice. Raises ValueError so callers can detect unexpected patterns. + The value between the anchors is regex-escaped by the producer, so it is + un-escaped back to the literal id (see _regex_literal_from_exact_match_body). + Raises ValueError when the pattern is not ^...$-anchored or its body is not a + pure literal, so callers can fall back to type-scoped loading. """ if not (pattern.startswith("^") and pattern.endswith("$")): raise ValueError(f"Expected ExactMatch regex ^...$, got: {pattern!r}") - return pattern[1:-1] + return _regex_literal_from_exact_match_body(pattern[1:-1]) _ID_FILE_IMPORT_SUPPORTED_TYPES = frozenset({"monitors", "authn_mappings", "team_memberships"}) diff --git a/tests/unit/test_minimize_reads_id_targeted.py b/tests/unit/test_minimize_reads_id_targeted.py index 9c2cf48e..feedc7a0 100644 --- a/tests/unit/test_minimize_reads_id_targeted.py +++ b/tests/unit/test_minimize_reads_id_targeted.py @@ -9,6 +9,7 @@ from pathlib import Path from unittest.mock import MagicMock, patch +import pytest from botocore.exceptions import ClientError from datadog_sync.constants import Origin @@ -110,6 +111,117 @@ def test_end_to_end_through_process_filters(self): assert result == {"dashboards": ["dash-1", "dash-2"]} +# ─── Regex-escaped ExactMatch values (regexp.QuoteMeta / re.escape) ────────── + + +class TestExactMatchEscapedValues: + """Producers that regex-escape the filter Value before ExactMatch wrapping. + + ExactMatch wraps the Value in ``^...$`` and compiles it as a regex, so a + caller must escape regex metacharacters (e.g. Go ``regexp.QuoteMeta`` or + Python ``re.escape``) for the match to be literal — otherwise a metric id + like ``svc.request.count`` would also match ``svcXrequestYcount``. The + ID-targeted state load then reuses the Value as a literal storage key, so it + must recover the unescaped literal id, not the escaped pattern body. + """ + + def test_unwrap_recovers_literal_from_escaped_dots(self): + from datadog_sync.utils.configuration import _unwrap_exact_match_pattern + + # What process_filters stores when the producer escaped dots. + pattern = r"^svc\.request\.count$" + assert _unwrap_exact_match_pattern(pattern) == "svc.request.count" + + def test_unwrap_plain_literal_unchanged(self): + from datadog_sync.utils.configuration import _unwrap_exact_match_pattern + + # UUID-style id with no metacharacters: escaping is a no-op upstream. + assert _unwrap_exact_match_pattern("^dash-1$") == "dash-1" + + def test_unwrap_recovers_literal_backslash(self): + from datadog_sync.utils.configuration import _unwrap_exact_match_pattern + + assert _unwrap_exact_match_pattern(r"^metric\\dimension$") == r"metric\dimension" + + def test_unwrap_real_regex_raises(self): + """An unescaped metacharacter means a genuine regex, not an escaped id.""" + from datadog_sync.utils.configuration import _unwrap_exact_match_pattern + + with pytest.raises(ValueError): + _unwrap_exact_match_pattern("^svc.*count$") + + def test_semantic_regex_escape_falls_back(self): + """A regex escape such as ``\\d`` must not become the literal id ``d``.""" + from datadog_sync.utils.configuration import extract_exact_id_filters + from datadog_sync.utils.filter import process_filters + + filters = process_filters([r"Type=logs_metrics;Name=id;Value=\d;Operator=ExactMatch"]) + + assert filters["logs_metrics"][0].attr_re.fullmatch("7") + assert not filters["logs_metrics"][0].attr_re.fullmatch("d") + assert extract_exact_id_filters(filters, "or", ["logs_metrics"]) is None + + def test_extract_exact_id_filters_escaped_metric_ids(self): + """Regression: a producer regex-escapes dotted logs_metrics ids. + + Reproduces the drop where dotted ids vanished at ID-targeted state load + because the escaped pattern body (``svc\\.request\\.count``) was used as + a literal storage key and never matched the real blob key + (``svc.request.count``). Only the dot-free id survived unfixed. + """ + import re + from datadog_sync.utils.configuration import extract_exact_id_filters + from datadog_sync.utils.filter import Filter + + raw_ids = [ + "metric-no-dots", # no metacharacters — survives even unfixed + "svc.request.count", + "app.errors.total.count", + ] + # Mirror the producer: regex-escape each id, then ExactMatch-wrap it. + rt_filters = [ + Filter( + resource_type="logs_metrics", + attr_name="id", + attr_re=re.compile(f"^{re.escape(_id)}$"), + operator="exactmatch", + ) + for _id in raw_ids + ] + result = extract_exact_id_filters({"logs_metrics": rt_filters}, "or", ["logs_metrics"]) + assert result == {"logs_metrics": raw_ids} + + def test_escaped_id_load_end_to_end(self, tmp_path): + """State.get_by_ids resolves an escaped-value id against the real blob key.""" + import re + from datadog_sync.utils.configuration import extract_exact_id_filters + from datadog_sync.utils.filter import Filter + from datadog_sync.utils.state import State + + _id = "svc.request.count" + src = tmp_path / "source" + src.mkdir() + # Import writes one blob per resource, keyed by the literal (unescaped) id. + (src / f"logs_metrics.{_id}.json").write_text(json.dumps({_id: {"id": _id, "attributes": {}}})) + + rt_filter = Filter( + resource_type="logs_metrics", + attr_name="id", + attr_re=re.compile(f"^{re.escape(_id)}$"), + operator="exactmatch", + ) + exact_ids = extract_exact_id_filters({"logs_metrics": [rt_filter]}, "or", ["logs_metrics"]) + + state = State( + StorageType.LOCAL_FILE, + source_resources_path=str(src), + destination_resources_path=str(tmp_path / "destination"), + resource_per_file=True, + exact_ids=exact_ids, + ) + assert _id in state.source["logs_metrics"] + + # ─── State with exact_ids ───────────────────────────────────────────────────