Fix ID-targeted state load dropping resources with regex-escaped filter values

datadog_sync/utils/configuration.py

@@ -189,15 +189,65 @@ async def exit_async(self):
         await self.destination_client._end_session()
 
 
+# Regex metacharacters that, when unescaped in an ExactMatch body, mean the
+# value is a genuine regex rather than a regex-escaped literal id. Mirrors the
+# metacharacter set escaped by Go's regexp.QuoteMeta (and a subset of the
+# non-alphanumerics escaped by Python's re.escape).
+_REGEX_METACHARS = frozenset(r".^$*+?()[]{}|")
+
+
+def _regex_literal_from_exact_match_body(body: str) -> str:
+    """Recover the literal string an ExactMatch pattern body matches exactly.
+
+    ExactMatch wraps the filter Value in ``^...$`` and compiles it as a regex,
+    so producers must regex-escape metacharacters in the Value for the match to
+    be literal (e.g. Go ``regexp.QuoteMeta`` / Python ``re.escape`` turn the
+    metric id ``svc.request.count`` into ``svc\\.request\\.count``). The
+    ID-targeted state load reuses the Value as a literal storage key, so the
+    escaping must be reversed here — otherwise the escaped body is looked up as
+    a key that never matches the real (unescaped) blob, silently dropping the
+    resource from state.
+
+    Raises ValueError when the body is not a pure literal — an unescaped
+    metacharacter, a semantic or ambiguous alphanumeric escape, or a dangling
+    backslash. Callers treat that as "not ID-targetable" and fall back to
+    type-scoped loading, which matches via the compiled regex and stays correct.
+    """
+    out = []
+    i = 0
+    n = len(body)
+    while i < n:
+        c = body[i]
+        if c == "\\":
+            if i + 1 >= n:
+                raise ValueError(f"dangling escape in ExactMatch pattern body: {body!r}")
+            escaped = body[i + 1]
+            # Alphanumeric escapes can change regex semantics (for example
+            # \d, \w, \x41, \n, and backreferences). Without the original
+            # unescaped value, they cannot safely be converted to a storage ID.
+            if escaped.isalnum():
+                raise ValueError(f"semantic or ambiguous escape \\{escaped} in ExactMatch pattern body: {body!r}")
+            out.append(escaped)
+            i += 2
+            continue
+        if c in _REGEX_METACHARS:
+            raise ValueError(f"unescaped metacharacter {c!r} in ExactMatch pattern body: {body!r}")
+        out.append(c)
+        i += 1
+    return "".join(out)
+
+
 def _unwrap_exact_match_pattern(pattern: str) -> str:
-    """Extract the raw ID value from an ExactMatch ^...$-wrapped regex pattern.
+    """Extract the literal ID value from an ExactMatch ^...$-wrapped regex pattern.
 
-    Defensive check: ExactMatch always produces ^...$, so ValueError should not
-    fire in practice. Raises ValueError so callers can detect unexpected patterns.
+    The value between the anchors is regex-escaped by the producer, so it is
+    un-escaped back to the literal id (see _regex_literal_from_exact_match_body).
+    Raises ValueError when the pattern is not ^...$-anchored or its body is not a
+    pure literal, so callers can fall back to type-scoped loading.
     """
     if not (pattern.startswith("^") and pattern.endswith("$")):
         raise ValueError(f"Expected ExactMatch regex ^...$, got: {pattern!r}")
-    return pattern[1:-1]
+    return _regex_literal_from_exact_match_body(pattern[1:-1])
 
 
 _ID_FILE_IMPORT_SUPPORTED_TYPES = frozenset({"monitors", "authn_mappings", "team_memberships"})

tests/unit/test_minimize_reads_id_targeted.py

@@ -9,6 +9,7 @@
 from pathlib import Path
 from unittest.mock import MagicMock, patch
 
+import pytest
 from botocore.exceptions import ClientError
 
 from datadog_sync.constants import Origin
@@ -110,6 +111,117 @@ def test_end_to_end_through_process_filters(self):
         assert result == {"dashboards": ["dash-1", "dash-2"]}
 
 
+# ─── Regex-escaped ExactMatch values (regexp.QuoteMeta / re.escape) ──────────
+
+
+class TestExactMatchEscapedValues:
+    """Producers that regex-escape the filter Value before ExactMatch wrapping.
+
+    ExactMatch wraps the Value in ``^...$`` and compiles it as a regex, so a
+    caller must escape regex metacharacters (e.g. Go ``regexp.QuoteMeta`` or
+    Python ``re.escape``) for the match to be literal — otherwise a metric id
+    like ``svc.request.count`` would also match ``svcXrequestYcount``. The
+    ID-targeted state load then reuses the Value as a literal storage key, so it
+    must recover the unescaped literal id, not the escaped pattern body.
+    """
+
+    def test_unwrap_recovers_literal_from_escaped_dots(self):
+        from datadog_sync.utils.configuration import _unwrap_exact_match_pattern
+
+        # What process_filters stores when the producer escaped dots.
+        pattern = r"^svc\.request\.count$"
+        assert _unwrap_exact_match_pattern(pattern) == "svc.request.count"
+
+    def test_unwrap_plain_literal_unchanged(self):
+        from datadog_sync.utils.configuration import _unwrap_exact_match_pattern
+
+        # UUID-style id with no metacharacters: escaping is a no-op upstream.
+        assert _unwrap_exact_match_pattern("^dash-1$") == "dash-1"
+
+    def test_unwrap_recovers_literal_backslash(self):
+        from datadog_sync.utils.configuration import _unwrap_exact_match_pattern
+
+        assert _unwrap_exact_match_pattern(r"^metric\\dimension$") == r"metric\dimension"
+
+    def test_unwrap_real_regex_raises(self):
+        """An unescaped metacharacter means a genuine regex, not an escaped id."""
+        from datadog_sync.utils.configuration import _unwrap_exact_match_pattern
+
+        with pytest.raises(ValueError):
+            _unwrap_exact_match_pattern("^svc.*count$")
+
+    def test_semantic_regex_escape_falls_back(self):
+        """A regex escape such as ``\\d`` must not become the literal id ``d``."""
+        from datadog_sync.utils.configuration import extract_exact_id_filters
+        from datadog_sync.utils.filter import process_filters
+
+        filters = process_filters([r"Type=logs_metrics;Name=id;Value=\d;Operator=ExactMatch"])
+
+        assert filters["logs_metrics"][0].attr_re.fullmatch("7")
+        assert not filters["logs_metrics"][0].attr_re.fullmatch("d")
+        assert extract_exact_id_filters(filters, "or", ["logs_metrics"]) is None
+
+    def test_extract_exact_id_filters_escaped_metric_ids(self):
+        """Regression: a producer regex-escapes dotted logs_metrics ids.
+
+        Reproduces the drop where dotted ids vanished at ID-targeted state load
+        because the escaped pattern body (``svc\\.request\\.count``) was used as
+        a literal storage key and never matched the real blob key
+        (``svc.request.count``). Only the dot-free id survived unfixed.
+        """
+        import re
+        from datadog_sync.utils.configuration import extract_exact_id_filters
+        from datadog_sync.utils.filter import Filter
+
+        raw_ids = [
+            "metric-no-dots",  # no metacharacters — survives even unfixed
+            "svc.request.count",
+            "app.errors.total.count",
+        ]
+        # Mirror the producer: regex-escape each id, then ExactMatch-wrap it.
+        rt_filters = [
+            Filter(
+                resource_type="logs_metrics",
+                attr_name="id",
+                attr_re=re.compile(f"^{re.escape(_id)}$"),
+                operator="exactmatch",
+            )
+            for _id in raw_ids
+        ]
+        result = extract_exact_id_filters({"logs_metrics": rt_filters}, "or", ["logs_metrics"])
+        assert result == {"logs_metrics": raw_ids}
+
+    def test_escaped_id_load_end_to_end(self, tmp_path):
+        """State.get_by_ids resolves an escaped-value id against the real blob key."""
+        import re
+        from datadog_sync.utils.configuration import extract_exact_id_filters
+        from datadog_sync.utils.filter import Filter
+        from datadog_sync.utils.state import State
+
+        _id = "svc.request.count"
+        src = tmp_path / "source"
+        src.mkdir()
+        # Import writes one blob per resource, keyed by the literal (unescaped) id.
+        (src / f"logs_metrics.{_id}.json").write_text(json.dumps({_id: {"id": _id, "attributes": {}}}))
+
+        rt_filter = Filter(
+            resource_type="logs_metrics",
+            attr_name="id",
+            attr_re=re.compile(f"^{re.escape(_id)}$"),
+            operator="exactmatch",
+        )
+        exact_ids = extract_exact_id_filters({"logs_metrics": [rt_filter]}, "or", ["logs_metrics"])
+
+        state = State(
+            StorageType.LOCAL_FILE,
+            source_resources_path=str(src),
+            destination_resources_path=str(tmp_path / "destination"),
+            resource_per_file=True,
+            exact_ids=exact_ids,
+        )
+        assert _id in state.source["logs_metrics"]
+
+
 # ─── State with exact_ids ───────────────────────────────────────────────────