From c4dc978b9fe2920e3ad34cc1672f7d47a074404a Mon Sep 17 00:00:00 2001 From: michael-richey Date: Wed, 5 Aug 2026 15:06:41 -0400 Subject: [PATCH 1/4] Fix ID-targeted state load dropping resources with escaped filter values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ExactMatch filter values are regex-escaped by callers (e.g. via regexp.QuoteMeta or re.escape) so that metacharacters match literally once the value is wrapped in ^...$ and compiled. The --minimize-reads ID-targeted state load then reused the pattern body as a literal storage key without reversing that escaping, so an id such as "svc.request.count" was looked up as "svc\.request\.count" and never matched its stored blob. Those resources were silently dropped from state before diff/apply — no create, no error, no filtered count. Un-escape the ExactMatch body back to the literal id in _unwrap_exact_match_pattern. When the body is not a pure literal (an unescaped metacharacter, i.e. a genuine regex, or a dangling backslash) raise ValueError so extract_exact_id_filters falls back to type-scoped loading, which matches via the compiled regex and stays correct. Add red/green regression tests covering the unwrap helper and an end-to-end escaped-value load through State.get_by_ids. Co-Authored-By: Claude Opus 4.8 (1M context) --- datadog_sync/utils/configuration.py | 51 +++++++++- tests/unit/test_minimize_reads_id_targeted.py | 96 +++++++++++++++++++ 2 files changed, 143 insertions(+), 4 deletions(-) diff --git a/datadog_sync/utils/configuration.py b/datadog_sync/utils/configuration.py index e274a46f..44478760 100644 --- a/datadog_sync/utils/configuration.py +++ b/datadog_sync/utils/configuration.py @@ -189,15 +189,58 @@ async def exit_async(self): await self.destination_client._end_session() +# Regex metacharacters that, when unescaped in an ExactMatch body, mean the +# value is a genuine regex rather than a regex-escaped literal id. Mirrors the +# set escaped by Go's regexp.QuoteMeta and Python's re.escape. +_REGEX_METACHARS = frozenset(r".^$*+?()[]{}|") + + +def _regex_literal_from_exact_match_body(body: str) -> str: + """Recover the literal string an ExactMatch pattern body matches exactly. + + ExactMatch wraps the filter Value in ``^...$`` and compiles it as a regex, + so producers must regex-escape metacharacters in the Value for the match to + be literal (e.g. Go ``regexp.QuoteMeta`` / Python ``re.escape`` turn the + metric id ``svc.request.count`` into ``svc\\.request\\.count``). The + ID-targeted state load reuses the Value as a literal storage key, so the + escaping must be reversed here — otherwise the escaped body is looked up as + a key that never matches the real (unescaped) blob, silently dropping the + resource from state. + + Raises ValueError when the body is not a pure literal — an unescaped + metacharacter (a real regex) or a dangling backslash. Callers treat that as + "not ID-targetable" and fall back to type-scoped loading, which matches via + the compiled regex and stays correct. + """ + out = [] + i = 0 + n = len(body) + while i < n: + c = body[i] + if c == "\\": + if i + 1 >= n: + raise ValueError(f"dangling escape in ExactMatch pattern body: {body!r}") + out.append(body[i + 1]) + i += 2 + continue + if c in _REGEX_METACHARS: + raise ValueError(f"unescaped metacharacter {c!r} in ExactMatch pattern body: {body!r}") + out.append(c) + i += 1 + return "".join(out) + + def _unwrap_exact_match_pattern(pattern: str) -> str: - """Extract the raw ID value from an ExactMatch ^...$-wrapped regex pattern. + """Extract the literal ID value from an ExactMatch ^...$-wrapped regex pattern. - Defensive check: ExactMatch always produces ^...$, so ValueError should not - fire in practice. Raises ValueError so callers can detect unexpected patterns. + The value between the anchors is regex-escaped by the producer, so it is + un-escaped back to the literal id (see _regex_literal_from_exact_match_body). + Raises ValueError when the pattern is not ^...$-anchored or its body is not a + pure literal, so callers can fall back to type-scoped loading. """ if not (pattern.startswith("^") and pattern.endswith("$")): raise ValueError(f"Expected ExactMatch regex ^...$, got: {pattern!r}") - return pattern[1:-1] + return _regex_literal_from_exact_match_body(pattern[1:-1]) _ID_FILE_IMPORT_SUPPORTED_TYPES = frozenset({"monitors", "authn_mappings", "team_memberships"}) diff --git a/tests/unit/test_minimize_reads_id_targeted.py b/tests/unit/test_minimize_reads_id_targeted.py index 9c2cf48e..4f933f43 100644 --- a/tests/unit/test_minimize_reads_id_targeted.py +++ b/tests/unit/test_minimize_reads_id_targeted.py @@ -9,6 +9,7 @@ from pathlib import Path from unittest.mock import MagicMock, patch +import pytest from botocore.exceptions import ClientError from datadog_sync.constants import Origin @@ -110,6 +111,101 @@ def test_end_to_end_through_process_filters(self): assert result == {"dashboards": ["dash-1", "dash-2"]} +# ─── Regex-escaped ExactMatch values (regexp.QuoteMeta / re.escape) ────────── + + +class TestExactMatchEscapedValues: + """Producers that regex-escape the filter Value before ExactMatch wrapping. + + ExactMatch wraps the Value in ``^...$`` and compiles it as a regex, so a + caller must escape regex metacharacters (e.g. Go ``regexp.QuoteMeta`` or + Python ``re.escape``) for the match to be literal — otherwise a metric id + like ``svc.request.count`` would also match ``svcXrequestYcount``. The + ID-targeted state load then reuses the Value as a literal storage key, so it + must recover the unescaped literal id, not the escaped pattern body. + """ + + def test_unwrap_recovers_literal_from_escaped_dots(self): + from datadog_sync.utils.configuration import _unwrap_exact_match_pattern + + # What process_filters stores when the producer escaped dots. + pattern = r"^svc\.request\.count$" + assert _unwrap_exact_match_pattern(pattern) == "svc.request.count" + + def test_unwrap_plain_literal_unchanged(self): + from datadog_sync.utils.configuration import _unwrap_exact_match_pattern + + # UUID-style id with no metacharacters: escaping is a no-op upstream. + assert _unwrap_exact_match_pattern("^dash-1$") == "dash-1" + + def test_unwrap_real_regex_raises(self): + """An unescaped metacharacter means a genuine regex, not an escaped id.""" + from datadog_sync.utils.configuration import _unwrap_exact_match_pattern + + with pytest.raises(ValueError): + _unwrap_exact_match_pattern("^svc.*count$") + + def test_extract_exact_id_filters_escaped_metric_ids(self): + """Regression: a producer regex-escapes dotted logs_metrics ids. + + Reproduces the drop where dotted ids vanished at ID-targeted state load + because the escaped pattern body (``svc\\.request\\.count``) was used as + a literal storage key and never matched the real blob key + (``svc.request.count``). Only the dot-free id survived unfixed. + """ + import re + from datadog_sync.utils.configuration import extract_exact_id_filters + from datadog_sync.utils.filter import Filter + + raw_ids = [ + "metric-no-dots", # no metacharacters — survives even unfixed + "svc.request.count", + "app.errors.total.count", + ] + # Mirror the producer: regex-escape each id, then ExactMatch-wrap it. + rt_filters = [ + Filter( + resource_type="logs_metrics", + attr_name="id", + attr_re=re.compile(f"^{re.escape(_id)}$"), + operator="exactmatch", + ) + for _id in raw_ids + ] + result = extract_exact_id_filters({"logs_metrics": rt_filters}, "or", ["logs_metrics"]) + assert result == {"logs_metrics": raw_ids} + + def test_escaped_id_load_end_to_end(self, tmp_path): + """State.get_by_ids resolves an escaped-value id against the real blob key.""" + import re + from datadog_sync.utils.configuration import extract_exact_id_filters + from datadog_sync.utils.filter import Filter + from datadog_sync.utils.state import State + + _id = "svc.request.count" + src = tmp_path / "source" + src.mkdir() + # Import writes one blob per resource, keyed by the literal (unescaped) id. + (src / f"logs_metrics.{_id}.json").write_text(json.dumps({_id: {"id": _id, "attributes": {}}})) + + rt_filter = Filter( + resource_type="logs_metrics", + attr_name="id", + attr_re=re.compile(f"^{re.escape(_id)}$"), + operator="exactmatch", + ) + exact_ids = extract_exact_id_filters({"logs_metrics": [rt_filter]}, "or", ["logs_metrics"]) + + state = State( + StorageType.LOCAL_FILE, + source_resources_path=str(src), + destination_resources_path=str(tmp_path / "destination"), + resource_per_file=True, + exact_ids=exact_ids, + ) + assert _id in state.source["logs_metrics"] + + # ─── State with exact_ids ─────────────────────────────────────────────────── From 5c2895564962b7dba2b1266ad33aa5f14c9be7b4 Mon Sep 17 00:00:00 2001 From: michael-richey Date: Wed, 5 Aug 2026 15:23:11 -0400 Subject: [PATCH 2/4] Handle semantic regex escapes in ID targeting --- datadog_sync/utils/configuration.py | 14 ++++++++++---- tests/unit/test_minimize_reads_id_targeted.py | 16 ++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/datadog_sync/utils/configuration.py b/datadog_sync/utils/configuration.py index 44478760..037c8bfd 100644 --- a/datadog_sync/utils/configuration.py +++ b/datadog_sync/utils/configuration.py @@ -208,9 +208,9 @@ def _regex_literal_from_exact_match_body(body: str) -> str: resource from state. Raises ValueError when the body is not a pure literal — an unescaped - metacharacter (a real regex) or a dangling backslash. Callers treat that as - "not ID-targetable" and fall back to type-scoped loading, which matches via - the compiled regex and stays correct. + metacharacter, a semantic or ambiguous alphanumeric escape, or a dangling + backslash. Callers treat that as "not ID-targetable" and fall back to + type-scoped loading, which matches via the compiled regex and stays correct. """ out = [] i = 0 @@ -220,7 +220,13 @@ def _regex_literal_from_exact_match_body(body: str) -> str: if c == "\\": if i + 1 >= n: raise ValueError(f"dangling escape in ExactMatch pattern body: {body!r}") - out.append(body[i + 1]) + escaped = body[i + 1] + # Alphanumeric escapes can change regex semantics (for example + # \d, \w, \x41, \n, and backreferences). Without the original + # unescaped value, they cannot safely be converted to a storage ID. + if escaped.isalnum(): + raise ValueError(f"semantic or ambiguous escape \\{escaped} in ExactMatch pattern body: {body!r}") + out.append(escaped) i += 2 continue if c in _REGEX_METACHARS: diff --git a/tests/unit/test_minimize_reads_id_targeted.py b/tests/unit/test_minimize_reads_id_targeted.py index 4f933f43..feedc7a0 100644 --- a/tests/unit/test_minimize_reads_id_targeted.py +++ b/tests/unit/test_minimize_reads_id_targeted.py @@ -138,6 +138,11 @@ def test_unwrap_plain_literal_unchanged(self): # UUID-style id with no metacharacters: escaping is a no-op upstream. assert _unwrap_exact_match_pattern("^dash-1$") == "dash-1" + def test_unwrap_recovers_literal_backslash(self): + from datadog_sync.utils.configuration import _unwrap_exact_match_pattern + + assert _unwrap_exact_match_pattern(r"^metric\\dimension$") == r"metric\dimension" + def test_unwrap_real_regex_raises(self): """An unescaped metacharacter means a genuine regex, not an escaped id.""" from datadog_sync.utils.configuration import _unwrap_exact_match_pattern @@ -145,6 +150,17 @@ def test_unwrap_real_regex_raises(self): with pytest.raises(ValueError): _unwrap_exact_match_pattern("^svc.*count$") + def test_semantic_regex_escape_falls_back(self): + """A regex escape such as ``\\d`` must not become the literal id ``d``.""" + from datadog_sync.utils.configuration import extract_exact_id_filters + from datadog_sync.utils.filter import process_filters + + filters = process_filters([r"Type=logs_metrics;Name=id;Value=\d;Operator=ExactMatch"]) + + assert filters["logs_metrics"][0].attr_re.fullmatch("7") + assert not filters["logs_metrics"][0].attr_re.fullmatch("d") + assert extract_exact_id_filters(filters, "or", ["logs_metrics"]) is None + def test_extract_exact_id_filters_escaped_metric_ids(self): """Regression: a producer regex-escapes dotted logs_metrics ids. From 575254c883cfd86a98d59b215ae85539ea5a8c88 Mon Sep 17 00:00:00 2001 From: Michael Richey <41595765+michael-richey@users.noreply.github.com> Date: Wed, 5 Aug 2026 15:33:58 -0400 Subject: [PATCH 3/4] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- datadog_sync/utils/configuration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datadog_sync/utils/configuration.py b/datadog_sync/utils/configuration.py index 037c8bfd..d46387bd 100644 --- a/datadog_sync/utils/configuration.py +++ b/datadog_sync/utils/configuration.py @@ -191,8 +191,8 @@ async def exit_async(self): # Regex metacharacters that, when unescaped in an ExactMatch body, mean the # value is a genuine regex rather than a regex-escaped literal id. Mirrors the -# set escaped by Go's regexp.QuoteMeta and Python's re.escape. -_REGEX_METACHARS = frozenset(r".^$*+?()[]{}|") +# metacharacter set escaped by Go's regexp.QuoteMeta (and a subset of the +# non-alphanumerics escaped by Python's re.escape). def _regex_literal_from_exact_match_body(body: str) -> str: From b53183413306a0d8066f248f65d9abe119f56d3e Mon Sep 17 00:00:00 2001 From: michael-richey Date: Wed, 5 Aug 2026 15:49:52 -0400 Subject: [PATCH 4/4] Fix undefined _REGEX_METACHARS constant A prior autofix commit rewrote the comment above this constant but dropped the actual assignment, leaving the name referenced but undefined and breaking ruff (F821). --- datadog_sync/utils/configuration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/datadog_sync/utils/configuration.py b/datadog_sync/utils/configuration.py index d46387bd..31c92bf8 100644 --- a/datadog_sync/utils/configuration.py +++ b/datadog_sync/utils/configuration.py @@ -193,6 +193,7 @@ async def exit_async(self): # value is a genuine regex rather than a regex-escaped literal id. Mirrors the # metacharacter set escaped by Go's regexp.QuoteMeta (and a subset of the # non-alphanumerics escaped by Python's re.escape). +_REGEX_METACHARS = frozenset(r".^$*+?()[]{}|") def _regex_literal_from_exact_match_body(body: str) -> str: