From 6bcabd2a7efcdb4cb752aa21e54ca9c6371da9eb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 20 Jul 2026 18:41:28 +0000 Subject: [PATCH 1/2] Initial plan From bc074d5bfa2daebdcbab968accf6d0385397d38e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 20 Jul 2026 18:46:21 +0000 Subject: [PATCH 2/2] Add deterministic parser perf benchmark infrastructure --- .github/workflows/perf.yml | 48 +++++ .github/workflows/python-publish.yml | 2 +- .github/workflows/python-tests.yml | 2 +- .gitignore | 1 + CONTRIBUTING.md | 15 +- pyproject.toml | 3 + tests/test_perf.py | 284 +++++++++++++++++++++++++++ 7 files changed, 352 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/perf.yml create mode 100644 tests/test_perf.py diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml new file mode 100644 index 0000000..10634fa --- /dev/null +++ b/.github/workflows/perf.yml @@ -0,0 +1,48 @@ +# Deterministic parser performance baseline workflow. +name: Parser Performance Baseline + +on: + workflow_dispatch: + schedule: + # weekly baseline collection every Sunday at 01:15 UTC + - cron: '15 1 * * 0' + +env: + PYTHON_VERSION_DEFAULT: "3.14" + RFC3987_PERF_LOOPS: "3" + RFC3987_PERF_WARMUP_LOOPS: "1" + RFC3987_PERF_OUTPUT_JSON: "${{ github.workspace }}/artifacts/perf/perf-results.json" + +permissions: + contents: read + +jobs: + perf-benchmark: + name: PerfBenchmark + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + # see https://github.com/actions/checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Setup Python Environment + # see https://github.com/actions/setup-python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: ${{ env.PYTHON_VERSION_DEFAULT }} + architecture: 'x64' + - name: Install env dependencies + run: python -m pip install --upgrade pip + - name: Install project + run: python -m pip install --user -e ".[testing]" + - name: Run perf tests + run: python -m pytest -m perf tests/test_perf.py -q + - name: Upload benchmark artifacts + if: always() + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 + with: + name: parser-perf-benchmark + path: artifacts/perf + if-no-files-found: error diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index b413f04..a33d412 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -44,7 +44,7 @@ jobs: - name: Install project run: python -m pip install --user -e ".[testing]" - name: Run Tests - run: python -m pytest tests + run: python -m pytest -m "not perf" tests publish: name: Publish needs: diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index e0262eb..154e181 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -104,4 +104,4 @@ jobs: - name: Install project run: pip install --user -e ".[testing]" - name: Run Tests - run: python -m pytest tests \ No newline at end of file + run: python -m pytest -m "not perf" tests \ No newline at end of file diff --git a/.gitignore b/.gitignore index 473a0b7..d6291ff 100644 --- a/.gitignore +++ b/.gitignore @@ -226,4 +226,5 @@ __marimo__/ #### CUSTOM +.pytest_perf/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 92cbf03..7415a9c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,7 +55,20 @@ Pull requests are welcome! Please keep the following in mind: Tests are written with [pytest](https://docs.pytest.org/). ```sh -python -m pytest tests +python -m pytest -m "not perf" tests +``` + +Performance benchmark tests are separated behind the `perf` marker: + +```sh +python -m pytest -m perf tests/test_perf.py +``` + +By default, perf tests run with one measured loop (`RFC3987_PERF_LOOPS=1`) to keep CI-safe runtimes. +You can increase loops and control the JSON output file path via environment variables: + +```sh +RFC3987_PERF_LOOPS=1000 RFC3987_PERF_WARMUP_LOOPS=1 RFC3987_PERF_OUTPUT_JSON=.pytest_perf/perf-results.json python -m pytest -m perf tests/test_perf.py ``` ## Running Static Analysis diff --git a/pyproject.toml b/pyproject.toml index 874230d..89c33d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,9 @@ testing = [ [tool.pytest.ini_options] pythonpath = ["src"] +markers = [ + "perf: deterministic parser performance baseline tests", +] [tool.hatch.build.targets.sdist] exclude = [ diff --git a/tests/test_perf.py b/tests/test_perf.py new file mode 100644 index 0000000..ad31a6f --- /dev/null +++ b/tests/test_perf.py @@ -0,0 +1,284 @@ +# Copyright (c) 2026 Jan Kowalleck - modifications and maintenance +# SPDX-License-Identifier: MIT + +import json +import os +import platform +import sys +import time +import tracemalloc +from importlib import reload +from pathlib import Path +from typing import Callable, TypedDict + +from lark import exceptions +import pytest + +import rfc3987_syntax2 as sut +import rfc3987_syntax2.syntax_helpers as syntax_helpers + +from . import invalid_syntax_data, valid_syntax_data, SyntaxCase + + +pytestmark = pytest.mark.perf + +PERF_LOOPS = int(os.getenv("RFC3987_PERF_LOOPS", "1")) +PERF_WARMUP_LOOPS = int(os.getenv("RFC3987_PERF_WARMUP_LOOPS", "1")) +PERF_OUTPUT_JSON = Path( + os.getenv("RFC3987_PERF_OUTPUT_JSON", ".pytest_perf/perf-results.json") +) + +_PARSER_TERMS = ("iri", "iri_reference", "absolute_iri") +_BENCHMARKED_PUBLIC_PARSER_APIS = ( + "parse", + "is_valid_syntax", + "make_syntax_validator", + "RFC3987_SYNTAX_TERM_VALIDATORS[term]", +) + + +class _RuntimeMeasurement(TypedDict): + elapsed_seconds: float + parse_calls: int + memory_peak_bytes: int + + +def _measure_runtime(run_cb: Callable[[], int]) -> _RuntimeMeasurement: + for _ in range(PERF_WARMUP_LOOPS): + run_cb() + + tracemalloc.start() + start = time.perf_counter() + parse_calls = run_cb() + elapsed = time.perf_counter() - start + _, memory_peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + return { + "elapsed_seconds": elapsed, + "parse_calls": parse_calls, + "memory_peak_bytes": memory_peak, + } + + +def _run_parser_api(term: str, valid_cases: list[SyntaxCase], invalid_cases: list[SyntaxCase]) -> int: + parse_calls = 0 + for _ in range(PERF_LOOPS): + for case in valid_cases: + parse_calls += 1 + sut.parse(term=term, value=case["value"]) + for case in invalid_cases: + parse_calls += 1 + with pytest.raises(exceptions.LarkError): + sut.parse(term=term, value=case["value"]) + return parse_calls + + +def _run_is_valid_api( + validator: Callable[[str], bool], valid_cases: list[SyntaxCase], invalid_cases: list[SyntaxCase] +) -> int: + parse_calls = 0 + for _ in range(PERF_LOOPS): + for case in valid_cases: + parse_calls += 1 + assert validator(case["value"]) is True + for case in invalid_cases: + parse_calls += 1 + assert validator(case["value"]) is False + return parse_calls + + +def _measure_parse_build_init(valid_sample: str) -> float: + fresh = reload(syntax_helpers) + start = time.perf_counter() + fresh.parse(term="iri", value=valid_sample) + return time.perf_counter() - start + + +def _measure_is_valid_syntax_build_init(valid_sample: str) -> float: + fresh = reload(syntax_helpers) + start = time.perf_counter() + assert fresh.is_valid_syntax(term="iri", value=valid_sample) is True + return time.perf_counter() - start + + +def _measure_make_validator_build_init(term: str, valid_sample: str) -> float: + start = time.perf_counter() + validator = sut.make_syntax_validator(term) + assert validator(valid_sample) is True + return time.perf_counter() - start + + +def _write_results(payload: dict[str, object]) -> None: + PERF_OUTPUT_JSON.parent.mkdir(parents=True, exist_ok=True) + with PERF_OUTPUT_JSON.open("w", encoding="utf-8") as handle: + json.dump(payload, handle, indent=2, sort_keys=True) + handle.write("\n") + + +def test_parser_performance_baseline() -> None: + assert PERF_LOOPS > 0 + assert PERF_WARMUP_LOOPS >= 0 + + valid_data = valid_syntax_data() + invalid_data = invalid_syntax_data() + benchmark_terms = tuple( + term + for term in sut.RFC3987_SYNTAX_TERMS + if term in valid_data and term in invalid_data + ) + assert benchmark_terms + for parser_term in _PARSER_TERMS: + assert parser_term in benchmark_terms + benchmarked_term_validator_apis = tuple( + f"is_valid_syntax_{term}" for term in benchmark_terms + ) + valid_sample = valid_data["iri"][0]["value"] + + metrics: list[dict[str, object]] = [] + + metrics.append( + { + "kind": "build_init", + "api": "parse", + "term": "iri", + "elapsed_seconds": _measure_parse_build_init(valid_sample), + } + ) + metrics.append( + { + "kind": "build_init", + "api": "is_valid_syntax", + "term": "iri", + "elapsed_seconds": _measure_is_valid_syntax_build_init(valid_sample), + } + ) + + for term in benchmark_terms: + metrics.append( + { + "kind": "build_init", + "api": "make_syntax_validator", + "term": term, + "elapsed_seconds": _measure_make_validator_build_init( + term, valid_data[term][0]["value"] + ), + } + ) + + for term in _PARSER_TERMS: + valid_cases = valid_data[term] + invalid_cases = invalid_data[term] + + parse_runtime = _measure_runtime( + lambda term=term, valid_cases=valid_cases, invalid_cases=invalid_cases: _run_parser_api( + term=term, valid_cases=valid_cases, invalid_cases=invalid_cases + ) + ) + metrics.append( + { + "kind": "runtime", + "api": "parse", + "term": term, + "cases": len(valid_cases) + len(invalid_cases), + "loops": PERF_LOOPS, + **parse_runtime, + } + ) + + is_valid_runtime = _measure_runtime( + lambda term=term, valid_cases=valid_cases, invalid_cases=invalid_cases: _run_is_valid_api( + validator=lambda value, term=term: sut.is_valid_syntax(term=term, value=value), + valid_cases=valid_cases, + invalid_cases=invalid_cases, + ) + ) + metrics.append( + { + "kind": "runtime", + "api": "is_valid_syntax", + "term": term, + "cases": len(valid_cases) + len(invalid_cases), + "loops": PERF_LOOPS, + **is_valid_runtime, + } + ) + + for term in benchmark_terms: + valid_cases = valid_data[term] + invalid_cases = invalid_data[term] + + factory_validator = sut.make_syntax_validator(term) + assert factory_validator(valid_cases[0]["value"]) is True + factory_runtime = _measure_runtime( + lambda factory_validator=factory_validator, valid_cases=valid_cases, invalid_cases=invalid_cases: _run_is_valid_api( + validator=factory_validator, + valid_cases=valid_cases, + invalid_cases=invalid_cases, + ) + ) + metrics.append( + { + "kind": "runtime", + "api": "make_syntax_validator", + "term": term, + "cases": len(valid_cases) + len(invalid_cases), + "loops": PERF_LOOPS, + **factory_runtime, + } + ) + + validator_from_mapping = sut.RFC3987_SYNTAX_TERM_VALIDATORS[term] + mapping_runtime = _measure_runtime( + lambda validator_from_mapping=validator_from_mapping, valid_cases=valid_cases, invalid_cases=invalid_cases: _run_is_valid_api( + validator=validator_from_mapping, + valid_cases=valid_cases, + invalid_cases=invalid_cases, + ) + ) + metrics.append( + { + "kind": "runtime", + "api": "RFC3987_SYNTAX_TERM_VALIDATORS[term]", + "term": term, + "cases": len(valid_cases) + len(invalid_cases), + "loops": PERF_LOOPS, + **mapping_runtime, + } + ) + + public_api_name = f"is_valid_syntax_{term}" + public_validator = getattr(sut, public_api_name) + public_runtime = _measure_runtime( + lambda public_validator=public_validator, valid_cases=valid_cases, invalid_cases=invalid_cases: _run_is_valid_api( + validator=public_validator, + valid_cases=valid_cases, + invalid_cases=invalid_cases, + ) + ) + metrics.append( + { + "kind": "runtime", + "api": public_api_name, + "term": term, + "cases": len(valid_cases) + len(invalid_cases), + "loops": PERF_LOOPS, + **public_runtime, + } + ) + + payload = { + "benchmarked_public_parser_apis": [ + *_BENCHMARKED_PUBLIC_PARSER_APIS, + *benchmarked_term_validator_apis, + ], + "metadata": { + "loops": PERF_LOOPS, + "warmup_loops": PERF_WARMUP_LOOPS, + "python_version": sys.version, + "platform": platform.platform(), + }, + "metrics": metrics, + } + _write_results(payload) + + assert PERF_OUTPUT_JSON.exists()