diff --git a/opensyndrome/filter.py b/opensyndrome/filter.py index cbad64e..6d6100a 100644 --- a/opensyndrome/filter.py +++ b/opensyndrome/filter.py @@ -1,11 +1,13 @@ from __future__ import annotations + +import re from dataclasses import dataclass, field from typing import Any + import polars as pl from opensyndrome import schema - _VALID_CONCEPTS = [ format_type.value for format_type in schema.Type if format_type.value != "criterion" ] @@ -166,16 +168,28 @@ def _cast( def _code_to_regex(code: str) -> str: - """Convert an OSD code to a regex pattern. + """Convert an OSD code to a regex pattern, ignoring dot placement. + + ICD-10 codes are conventionally written with a dot after the 3-character + category (``A69.2``), but plenty of real definitions and datasets omit it + (``A692``) — SIA's own billing codes among them, and NSSP/CCDD category + lists the other way around. *code* and the value it is matched against + are compared with dots optional at that one position, so it makes no + difference which side, if either, carries one: ``A692``, ``A69.2`` and + ``A69`` all match ``A69.2`` and ``A692`` alike. - - ``%`` wildcard → ``.*`` (e.g. ``J1%`` → ``^J1.*$``) - - Exact code → anchored to avoid prefix collisions - (e.g. ``A90`` matches ``A90`` and ``A90.1`` but not ``A900``) + ``%`` is still a wildcard (``J1%`` → ``^J1.*$``). """ - if "%" in code: - return f"^{code.replace('%', '.*')}$" - # Allow optional sub-code suffix (dot-separated), e.g. A90.1 - return f"^{code}(\\..+)?$" + stripped = code.replace(".", "") + if "%" in stripped: + return f"^{re.escape(stripped).replace('%', '.*')}$" + + category, sub_code = stripped[:3], stripped[3:] + pattern = re.escape(category) + if sub_code: + pattern += r"\.?" + re.escape(sub_code) + # A value with more digits than the code is still one of its sub-codes. + return f"^{pattern}(\\.?\\w+)?$" def _apply_flags(pattern: str, flags: str) -> str: diff --git a/tests/test_filter.py b/tests/test_filter.py index c5c83ad..d5e1ffc 100644 --- a/tests/test_filter.py +++ b/tests/test_filter.py @@ -1,23 +1,24 @@ import re -import pytest + import polars as pl +import pytest from opensyndrome.filter import ( ColumnSpec, - load_profile, - ProfileData, InvalidOperator, - UnresolvableCriterion, OSDEngine, - _code_to_regex, + ProfileData, + UnresolvableCriterion, _apply_flags, - _cast, - _combine, - _build_code_expr, - _build_text_expr, _build_attr_expr, + _build_code_expr, _build_ontology_id_expr, + _build_text_expr, + _cast, + _code_to_regex, + _combine, _parse_criterion, + load_profile, ) SEX_ENCODINGS = {"sex": {"male": "M", "female": "F"}} @@ -155,7 +156,7 @@ class TestCodeToRegex: @pytest.mark.parametrize( "code, matching, non_matching", [ - ("A90", ["A90", "A90.1", "A90.12"], ["A900", "A901", "B90"]), + ("A90", ["A90", "A90.1", "A90.12", "A900", "A901"], ["B90", "A9"]), ("J1%", ["J10", "J11", "J18", "J1X"], ["J20", "K10"]), ("A9%", ["A90", "A91", "A99"], ["B90", "A100"]), ], @@ -171,15 +172,34 @@ def test_pattern_matches_and_rejects(self, code, matching, non_matching): pattern, value ), f"Expected '{value}' NOT to match pattern for '{code}'" - def test_exact_code_does_not_match_extension(self): - pattern = _code_to_regex("A90") - assert not re.match(pattern, "A900") - assert not re.match(pattern, "A901") - - def test_exact_code_matches_sub_code(self): + def test_exact_code_matches_sub_code_with_or_without_dot(self): + # ICD-10's own convention dots the sub-code (A90.1), but plenty of real + # datasets and definitions don't (SIA's billing codes, NSSP/CCDD). pattern = _code_to_regex("A90") assert re.match(pattern, "A90.1") assert re.match(pattern, "A90.12") + assert re.match(pattern, "A901") + assert re.match(pattern, "A9012") + + @pytest.mark.parametrize( + "code, value", + [ + ("A692", "A69.2"), # definition undotted, value dotted (NSSP/CCDD style) + ("A69.2", "A692"), # definition dotted, value undotted (SIA style) + ("A69.2", "A69.2"), # both dotted + ("A692", "A692"), # both undotted + ], + ) + def test_matches_regardless_of_which_side_carries_the_dot(self, code, value): + pattern = _code_to_regex(code) + assert re.match( + pattern, value + ), f"Expected '{value}' to match pattern for '{code}'" + + def test_unrelated_category_still_rejected(self): + pattern = _code_to_regex("A692") + assert not re.match(pattern, "A693") + assert not re.match(pattern, "B692") class TestApplyFlags: