fix(scanner): make esprima2 string/template scanning linear

pyjsclear/esprima_patch.py

@@ -0,0 +1,60 @@
+"""Make esprima2's string/template scanning linear instead of quadratic.
+
+esprima2 builds string and template literal values with ``value += ch`` in a
+``while`` loop (``Scanner.scanStringLiteral`` / ``scanTemplate``). In CPython
+that misses the in-place string-concat optimization, so scanning a single large
+literal is O(n^2) — one multi-megabyte string literal takes tens of seconds.
+
+The fix (proposed upstream, s0md3v/esprima2#10) is to accumulate into a list and
+join once at the end. Rather than copy the method bodies, we take the installed
+method source and apply that two-line transform: ``value += ch`` becomes a list
+extend (``list += "chars"`` extends by character), and the accumulator is joined
+at the return. Deriving from the live source keeps the patch faithful to
+whatever esprima2 version is installed. It is gated to versions verified to
+carry the expected source; any other version, or an unreadable source, leaves
+esprima untouched.
+"""
+
+import inspect
+import textwrap
+
+import esprima
+from esprima import scanner
+
+
+_SUPPORTED_ESPRIMA_VERSIONS = frozenset({(5, 0, 1), (5, 0, 2), (6, 0, 0)})
+
+# method -> (old, new) source edits turning the char-by-char accumulator into a
+# list joined once at the end.
+_SUBSTITUTIONS = {
+    'scanStringLiteral': [("str = ''", 'str = []'), ('value=str,', "value=''.join(str),")],
+    'scanTemplate': [("cooked = ''", 'cooked = []'), ('cooked=cooked,', "cooked=''.join(cooked),")],
+}
+
+
+def apply_patch() -> bool:
+    """Rebind esprima's scanner methods to linear versions. Returns True if applied."""
+    if getattr(esprima, '__version__', None) not in _SUPPORTED_ESPRIMA_VERSIONS:
+        return False
+
+    for method, edits in _SUBSTITUTIONS.items():
+        func = getattr(scanner.Scanner, method)
+        if getattr(func, '_linearized', False):
+            continue
+        try:
+            source = textwrap.dedent(inspect.getsource(func))
+        except OSError:
+            return False
+        for old, new in edits:
+            if source.count(old) != 1:
+                return False  # source drifted from what we verified; leave esprima untouched
+            source = source.replace(old, new)
+        namespace: dict = {}
+        exec(source, vars(scanner), namespace)  # noqa: S102 - transform of the installed esprima source
+        patched = namespace[method]
+        patched._linearized = True
+        setattr(scanner.Scanner, method, patched)
+    return True
+
+
+apply_patch()

pyjsclear/parser.py

@@ -2,7 +2,10 @@
 
 import re
 
-import esprima
+
+# Apply the esprima quadratic-scan patch before esprima is used to parse
+import pyjsclear.esprima_patch  # noqa: F401 isort:skip
+import esprima  # isort: skip
 
 
 _ASYNC_KEY_MAP: dict[str, str] = {'isAsync': 'async', 'allowAwait': 'await'}

tests/unit/esprima_patch_test.py

@@ -0,0 +1,94 @@
+"""Unit tests for pyjsclear.esprima_patch (linear string/template scanning)."""
+
+import time
+
+import esprima
+import pytest
+
+import pyjsclear.esprima_patch as esprima_patch
+
+
+def _literal_value(code):
+    return esprima.parseScript(code).body[0].declarations[0].init.value
+
+
+def _template_cooked(code):
+    quasi = esprima.parseScript(code).body[0].declarations[0].init.quasis[0]
+    return quasi.value.cooked
+
+
+class TestPatchApplied:
+    def test_patch_targets_supported_version(self):
+        assert esprima.__version__ in esprima_patch._SUPPORTED_ESPRIMA_VERSIONS
+
+    def test_scanner_methods_are_replaced(self):
+        assert getattr(esprima.scanner.Scanner.scanStringLiteral, '_linearized', False)
+        assert getattr(esprima.scanner.Scanner.scanTemplate, '_linearized', False)
+
+
+class TestStringLiteralEquivalence:
+    @pytest.mark.parametrize(
+        'code, expected',
+        [
+            (r"var a='';", ''),
+            (r"var a='plain';", 'plain'),
+            (r'var a="double";', 'double'),
+            (r"var a='tab\ttab\nnl';", 'tab\ttab\nnl'),
+            (r"var a='\r\b\f\v';", '\r\b\f\x0b'),
+            (r"var a='\x41\x42';", 'AB'),
+            (r"var a='\u{1F600}';", '\U0001f600'),
+            (r"var a='\101\102';", 'AB'),  # octal
+            (r"var a='\0';", '\0'),
+            (r"var a='\z\q';", 'zq'),  # unknown escape -> literal char
+            (r"var a='\\';", '\\'),
+            (r"var a='quote\'in';", "quote'in"),
+            (r"var a='émoji😀';", 'émoji😀'),
+            ("var a='line\\\ncont';", 'linecont'),  # line continuation
+            ("var a='line\\\r\ncont';", 'linecont'),  # CRLF continuation
+        ],
+    )
+    def test_value(self, code, expected):
+        assert _literal_value(code) == expected
+
+
+class TestTemplateEquivalence:
+    @pytest.mark.parametrize(
+        'code, expected',
+        [
+            ('var a=`plain`;', 'plain'),
+            (r'var a=`tab\tnl\n`;', 'tab\tnl\n'),
+            (r'var a=`\x41B`;', 'AB'),
+            ('var a=`raw\nline`;', 'raw\nline'),
+        ],
+    )
+    def test_cooked(self, code, expected):
+        assert _template_cooked(code) == expected
+
+
+class TestInvalidLiteralsStillRaise:
+    @pytest.mark.parametrize(
+        'code',
+        [
+            r"var a='unterminated;",
+            "var a='raw\nnl';",
+            r"var a='\x4';",
+            r"var a='\8';",
+        ],
+    )
+    def test_raises(self, code):
+        with pytest.raises(esprima.Error):
+            esprima.parseScript(code)
+
+
+class TestLinearScaling:
+    def test_large_single_string_literal_parses_in_linear_time(self):
+        # Unpatched this is O(n^2): ~7s+ at 1MB. Patched it is well under a second.
+        body = ('ABCDefgh0123+/' * (1024 * 1024 // 14 + 1))[: 1024 * 1024]
+        code = "var s='" + body + "';"
+
+        start = time.perf_counter()
+        value = _literal_value(code)
+        elapsed = time.perf_counter() - start
+
+        assert value == body
+        assert elapsed < 3.0, f'parse took {elapsed:.2f}s; quadratic scan likely regressed'