From eeab68413184c7d91635d27d6d2deb23acf0c62a Mon Sep 17 00:00:00 2001 From: Itamar Gafni Date: Wed, 26 Aug 2026 16:24:04 +0300 Subject: [PATCH] fix(scanner): make esprima2 string/template scanning linear esprima2 builds string and template literal values with `value += ch` in a while loop (Scanner.scanStringLiteral / scanTemplate). In CPython that misses the in-place string-concat optimization, so each append copies the whole buffer and scanning a single literal is O(n^2). Source that is one large string literal (e.g. a multi-megabyte base64 blob) takes ~70s to parse. Add pyjsclear/esprima_patch.py, applied at import of pyjsclear.parser. It takes the installed scanner method source and applies the two-line fix proposed upstream (s0md3v/esprima2#10): accumulate into a list and join once at the end. Deriving from the live source keeps it faithful to whatever esprima2 is installed; gated to verified versions (5.0.1, 5.0.2, 6.0.0), and it bails (leaving esprima untouched) if the source or version is unexpected. Parse of a 3.13MB single-literal file drops from ~70s to ~0.4s. Co-Authored-By: Claude Opus 4.8 --- pyjsclear/esprima_patch.py | 60 ++++++++++++++++++++ pyjsclear/parser.py | 5 +- tests/unit/esprima_patch_test.py | 94 ++++++++++++++++++++++++++++++++ 3 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 pyjsclear/esprima_patch.py create mode 100644 tests/unit/esprima_patch_test.py diff --git a/pyjsclear/esprima_patch.py b/pyjsclear/esprima_patch.py new file mode 100644 index 0000000..be65e8f --- /dev/null +++ b/pyjsclear/esprima_patch.py @@ -0,0 +1,60 @@ +"""Make esprima2's string/template scanning linear instead of quadratic. + +esprima2 builds string and template literal values with ``value += ch`` in a +``while`` loop (``Scanner.scanStringLiteral`` / ``scanTemplate``). In CPython +that misses the in-place string-concat optimization, so scanning a single large +literal is O(n^2) — one multi-megabyte string literal takes tens of seconds. + +The fix (proposed upstream, s0md3v/esprima2#10) is to accumulate into a list and +join once at the end. Rather than copy the method bodies, we take the installed +method source and apply that two-line transform: ``value += ch`` becomes a list +extend (``list += "chars"`` extends by character), and the accumulator is joined +at the return. Deriving from the live source keeps the patch faithful to +whatever esprima2 version is installed. It is gated to versions verified to +carry the expected source; any other version, or an unreadable source, leaves +esprima untouched. +""" + +import inspect +import textwrap + +import esprima +from esprima import scanner + + +_SUPPORTED_ESPRIMA_VERSIONS = frozenset({(5, 0, 1), (5, 0, 2), (6, 0, 0)}) + +# method -> (old, new) source edits turning the char-by-char accumulator into a +# list joined once at the end. +_SUBSTITUTIONS = { + 'scanStringLiteral': [("str = ''", 'str = []'), ('value=str,', "value=''.join(str),")], + 'scanTemplate': [("cooked = ''", 'cooked = []'), ('cooked=cooked,', "cooked=''.join(cooked),")], +} + + +def apply_patch() -> bool: + """Rebind esprima's scanner methods to linear versions. Returns True if applied.""" + if getattr(esprima, '__version__', None) not in _SUPPORTED_ESPRIMA_VERSIONS: + return False + + for method, edits in _SUBSTITUTIONS.items(): + func = getattr(scanner.Scanner, method) + if getattr(func, '_linearized', False): + continue + try: + source = textwrap.dedent(inspect.getsource(func)) + except OSError: + return False + for old, new in edits: + if source.count(old) != 1: + return False # source drifted from what we verified; leave esprima untouched + source = source.replace(old, new) + namespace: dict = {} + exec(source, vars(scanner), namespace) # noqa: S102 - transform of the installed esprima source + patched = namespace[method] + patched._linearized = True + setattr(scanner.Scanner, method, patched) + return True + + +apply_patch() diff --git a/pyjsclear/parser.py b/pyjsclear/parser.py index 828e18c..42da8a4 100644 --- a/pyjsclear/parser.py +++ b/pyjsclear/parser.py @@ -2,7 +2,10 @@ import re -import esprima + +# Apply the esprima quadratic-scan patch before esprima is used to parse +import pyjsclear.esprima_patch # noqa: F401 isort:skip +import esprima # isort: skip _ASYNC_KEY_MAP: dict[str, str] = {'isAsync': 'async', 'allowAwait': 'await'} diff --git a/tests/unit/esprima_patch_test.py b/tests/unit/esprima_patch_test.py new file mode 100644 index 0000000..94e3526 --- /dev/null +++ b/tests/unit/esprima_patch_test.py @@ -0,0 +1,94 @@ +"""Unit tests for pyjsclear.esprima_patch (linear string/template scanning).""" + +import time + +import esprima +import pytest + +import pyjsclear.esprima_patch as esprima_patch + + +def _literal_value(code): + return esprima.parseScript(code).body[0].declarations[0].init.value + + +def _template_cooked(code): + quasi = esprima.parseScript(code).body[0].declarations[0].init.quasis[0] + return quasi.value.cooked + + +class TestPatchApplied: + def test_patch_targets_supported_version(self): + assert esprima.__version__ in esprima_patch._SUPPORTED_ESPRIMA_VERSIONS + + def test_scanner_methods_are_replaced(self): + assert getattr(esprima.scanner.Scanner.scanStringLiteral, '_linearized', False) + assert getattr(esprima.scanner.Scanner.scanTemplate, '_linearized', False) + + +class TestStringLiteralEquivalence: + @pytest.mark.parametrize( + 'code, expected', + [ + (r"var a='';", ''), + (r"var a='plain';", 'plain'), + (r'var a="double";', 'double'), + (r"var a='tab\ttab\nnl';", 'tab\ttab\nnl'), + (r"var a='\r\b\f\v';", '\r\b\f\x0b'), + (r"var a='\x41\x42';", 'AB'), + (r"var a='\u{1F600}';", '\U0001f600'), + (r"var a='\101\102';", 'AB'), # octal + (r"var a='\0';", '\0'), + (r"var a='\z\q';", 'zq'), # unknown escape -> literal char + (r"var a='\\';", '\\'), + (r"var a='quote\'in';", "quote'in"), + (r"var a='émoji😀';", 'émoji😀'), + ("var a='line\\\ncont';", 'linecont'), # line continuation + ("var a='line\\\r\ncont';", 'linecont'), # CRLF continuation + ], + ) + def test_value(self, code, expected): + assert _literal_value(code) == expected + + +class TestTemplateEquivalence: + @pytest.mark.parametrize( + 'code, expected', + [ + ('var a=`plain`;', 'plain'), + (r'var a=`tab\tnl\n`;', 'tab\tnl\n'), + (r'var a=`\x41B`;', 'AB'), + ('var a=`raw\nline`;', 'raw\nline'), + ], + ) + def test_cooked(self, code, expected): + assert _template_cooked(code) == expected + + +class TestInvalidLiteralsStillRaise: + @pytest.mark.parametrize( + 'code', + [ + r"var a='unterminated;", + "var a='raw\nnl';", + r"var a='\x4';", + r"var a='\8';", + ], + ) + def test_raises(self, code): + with pytest.raises(esprima.Error): + esprima.parseScript(code) + + +class TestLinearScaling: + def test_large_single_string_literal_parses_in_linear_time(self): + # Unpatched this is O(n^2): ~7s+ at 1MB. Patched it is well under a second. + body = ('ABCDefgh0123+/' * (1024 * 1024 // 14 + 1))[: 1024 * 1024] + code = "var s='" + body + "';" + + start = time.perf_counter() + value = _literal_value(code) + elapsed = time.perf_counter() - start + + assert value == body + assert elapsed < 3.0, f'parse took {elapsed:.2f}s; quadratic scan likely regressed'