From 1ac2699bbdc05fc364b337d6f50cb9bd00b1b4de Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Mon, 27 Jul 2026 19:06:23 -0700 Subject: [PATCH] docs(gfql): delete refuted and unverifiable benchmark numbers, and gate the rest Two published OLAP ratios are not stale, they are wrong. `gfql/performance.rst` published q8 at "5.0 ms vs 1,004 ms for embedded Kuzu (200x)"; a matched, row-validated re-run measures 5.06 ms vs 9.76 ms = 1.93x. It published q9 at 14.2x; measured, 66.61 vs 84.06 = 1.26x. An audit of every other published performance figure found most of them can be neither confirmed nor refuted: the reproducers wrote to /tmp or only printed, recorded no commit/host/timestamp, and their artifacts were never committed. Two are worse than unverifiable - the LadybugDB competitor column was a hardcoded `LADYBUG = {...}` literal with no citation, and the filter->PageRank SVG charts are generated from a results directory that has never existed in any commit. All of it is removed rather than restated. The mechanism that stops it recurring ships with the deletion: a single generated source of truth (docs/source/_data/gfql_benchmarks.json), a Sphinx extension that resolves `:bench:` against it and fails the build on a missing, stale, non-quotable or undisclosed number, and a standalone gate that adds commit-drift and hand-typed-literal checks. Deletion and mechanism only - no editorial rewriting. The prose rewrite is a separate PR. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_015YsqAZQLbqjSDrYSFz2GoB --- CHANGELOG.md | 10 +- bin/check_bench_numbers.py | 266 +++ docs/.rstcheck.cfg | 8 +- docs/source/10min.rst | 2 +- docs/source/_data/gfql_benchmarks.json | 1488 ++++++++++++ docs/source/_ext/gfql_bench.py | 347 +++ docs/source/_ext/gfql_bench_data.py | 420 ++++ docs/source/cheatsheet.md | 2 +- docs/source/conf.py | 5 + .../filter_pagerank/gplus_lifecycle.svg | 2103 ----------------- .../filter_pagerank/twitter_lifecycle.svg | 1991 ---------------- .../source/gfql/benchmark_filter_pagerank.rst | 97 +- docs/source/gfql/builtin_calls.rst | 2 +- docs/source/gfql/engines.rst | 179 +- docs/source/gfql/index.rst | 2 +- docs/source/gfql/index_adjacency.rst | 95 +- docs/source/gfql/indexing.rst | 40 +- docs/source/gfql/performance.rst | 255 +- docs/source/performance.rst | 2 +- docs/test_bench_numbers.py | 161 ++ 20 files changed, 2908 insertions(+), 4567 deletions(-) create mode 100755 bin/check_bench_numbers.py create mode 100644 docs/source/_data/gfql_benchmarks.json create mode 100644 docs/source/_ext/gfql_bench.py create mode 100644 docs/source/_ext/gfql_bench_data.py delete mode 100644 docs/source/gfql/_static/filter_pagerank/gplus_lifecycle.svg delete mode 100644 docs/source/gfql/_static/filter_pagerank/twitter_lifecycle.svg create mode 100644 docs/test_bench_numbers.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ac88700713..b159251c6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,8 +37,14 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - **Seeded property-RETURN dtype divergence on cuDF**: the lean projection applied the pandas rows-pivot artifact (int → float64, bool → object) on every engine, but cuDF's canonical pivot preserves the source dtypes — so the fast path returned `float64`/`object` where cuDF's own canonical path returns `int64`/`bool`. The cast rule is now engine-aware; the dtype-class decline guard is unchanged. ### Documentation -- **GFQL pay-as-you-go resident indexing user guide**: New :doc:`Pay-As-You-Go Resident Indexing ` page — the lifecycle guide to resident indexes (`gfql_index_all()` / `gfql_index_edges()` / `create_index()` / `show_indexes()` / `drop_index()`): what the node-id + CSR in/out adjacency sidecars are, what engages them on 0.58.0 (seeded typed-hop fast paths incl. property RETURNs and property-seeded lookups per #1768/#1770, direct `g.hop()`; the general polars chain traversal honestly noted as not yet covered), the staleness/validity contract (identity + fingerprint; rebind invalidates; declines are safe — identical results either way), engine notes (polars needs `gfql_index_all(engine='polars')` until #1767), 0.58.0-tag measured numbers, and a runnable end-to-end example. Wired into the GFQL toctree + recommended paths alongside :doc:`Seeded Traversal Indexes `. -- **GFQL performance docs: 0.58.0 release-tag-verified numbers, siloed in one page**: `gfql/performance.rst` is now the canonical benchmark-numbers page (alongside `gfql/index_adjacency.rst` for the index benchmarks) — a benchmark rerun updates it alone. It carries the 0.58.0 tag sweep (DGX Spark GB10, warm medians N=30; four-engine numbers cross-engine parity-verified, competitor pairs validated against expected result rows): seeded typed-hop fast path across all four engines (e.g. pandas 29.9→2.46ms, 12.1×), native chain form, resident-index covered-shape lookups (with the `gfql_index_all(engine='polars')` caveat / PR #1767), flat seeded-hop scaling on pandas (0.159–0.164ms from 0.25M to 32M edges), the one-keyword `engine='polars'` LDBC SNB SF1 seed-lookup win (1,299.6→106.1ms, 12.3×), LDBC SNB interactive SF1 vs Neo4j 5.26 same-box pairs (GFQL 4 of 5; Neo4j wins recent-replies — reported as-is), OLAP multi-join vs embedded Kuzu (q8 200×, q9 14.2×) with the honest inverse (Kuzu wins single-table aggregates 2–4×, seeded property-projection lookups 2.4–64×), plus the prior Orkut/LiveJournal bulk sweep (moved from `engines.rst`, dated once) and its methodology. All other pages — `engines.rst`, `quick.rst`, `about.rst`, `overview.rst`, `index.rst` — now carry stable qualitative claims (e.g. "often an order of magnitude faster on query-heavy workloads") that link into `performance.rst` instead of inline figures, replacing the stale "up to ~38×" headlines and avoiding scattered per-claim version labels. +- **GFQL pay-as-you-go resident indexing user guide**: New :doc:`Pay-As-You-Go Resident Indexing ` page — the lifecycle guide to resident indexes (`gfql_index_all()` / `gfql_index_edges()` / `create_index()` / `show_indexes()` / `drop_index()`): what the node-id + CSR in/out adjacency sidecars are, what engages them on 0.58.0 (seeded typed-hop fast paths incl. property RETURNs and property-seeded lookups per #1768/#1770, direct `g.hop()`; the general polars chain traversal honestly noted as not yet covered), the staleness/validity contract (identity + fingerprint; rebind invalidates; declines are safe — identical results either way), engine notes (polars needs `gfql_index_all(engine='polars')` until #1767), the index cost model, and a runnable end-to-end example. Wired into the GFQL toctree + recommended paths alongside :doc:`Seeded Traversal Indexes `. +- **GFQL performance docs: benchmark numbers siloed in one page**: `gfql/performance.rst` is the canonical benchmark-numbers page (alongside `gfql/index_adjacency.rst` for the index benchmarks) — a benchmark rerun updates it alone. All other pages — `engines.rst`, `quick.rst`, `about.rst`, `overview.rst`, `index.rst` — carry stable qualitative claims that link into `performance.rst` instead of inline figures, avoiding scattered per-claim version labels. (The tag-sweep figures this entry originally listed have since been withdrawn — see the provenance entries below.) + +- **Published benchmark numbers now come from a single machine-readable source of truth, and a stale or missing one FAILS the docs build.** Every performance figure in the docs used to be transcribed by hand from a run report. That is how a number becomes authoritative and then gets re-copied after the board underneath it has moved. Docs no longer restate a number; they reference it. `docs/source/_data/gfql_benchmarks.json` holds every publishable cell with the run that produced it (measurement date, host, whether the perf lock was held and the host was quiet, rep protocol, the frozen pygraphistry commit, the pyg-bench commit, runtime image, dataset, competitor + version, raw artifact path), and it is **generated** by `scripts/export_docs_numbers.py` in [graphistry/pyg-bench](https://github.com/graphistry/pyg-bench) from artifacts committed there — so regenerating the docs' numbers needs no GPU box and no benchmark re-run. A new Sphinx extension (`docs/source/_ext/gfql_bench.py`) resolves ``:bench:`key``` against that file and **breaks the build** on: a key the source of truth does not contain; a run older than `policy.max_age_days`; a cell that is not board-quotable published as a bare number (it must use `:bench-diag:`, which labels it diagnostic-only); and a page that references a cell without rendering that cell's `.. bench-provenance::` or `.. bench-disclosures::` — because a bare ratio stripped of its asterisk launders the caveat, which is worse than manual transcription. `bin/check_bench_numbers.py` runs the same gate without Sphinx and adds two checks a docs build cannot make: **commit drift** (how many commits touching `graphistry/compute` have landed since a published run was measured), and a **hand-typed-literal guard** over the managed pages, since a number typed in by hand is by construction not a reference to the source of truth. Negative tests in `docs/test_bench_numbers.py` pin all four rejections. + +- **Two published OLAP ratios were REFUTED by a matched re-run and are removed.** `gfql/performance.rst` published "q8 runs in 5.0 ms vs 1,004 ms for embedded Kuzu (200×)"; a matched, row-validated re-run of the same q8 on the same host measures GFQL-polars 5.06 ms vs Kuzu 9.76 ms — **1.93×**, off by roughly a hundredfold. The same page published q9 at **14.2×**; measured, it is 66.61 ms vs 84.06 ms — **1.26×**. The 1,004 ms Kuzu figure traces to a historical board that pyg-bench's own results file records as "did not reproduce and cannot be reproduced". + +- **Benchmark numbers that could not be traced to a surviving measurement have been REMOVED rather than restated.** A provenance audit of every published performance figure found most of them can be neither confirmed nor refuted: the reproducers wrote their results to `/tmp` (or only printed them), recorded no commit, host or timestamp, and their raw artifacts were never committed and no longer exist. Withdrawn on those grounds: the four-engine Orkut/LiveJournal bulk table and its CPU-crossover ratios (`gfql/performance.rst`, `gfql/engines.rst`), the 0.58.0 tag-sweep tables (`gfql/performance.rst`, `gfql/indexing.rst`), the seeded-index synthetic and vs-Kuzu/Neo4j tables and the prepared-Kuzu figures (`gfql/index_adjacency.rst`), the LDBC-vs-Neo4j table (every row that maps onto a query id the current lane emits differs materially from the current board on both sides, and `recent-replies` maps onto no current query id at all), the LadybugDB head-to-head (whose competitor column came from a hardcoded `LADYBUG = {...}` literal in `benchmarks/gfql/bench_ladybug_cypher.py` with no URL, version or citation — a literal, not a measurement), the filter→PageRank-vs-Neo4j results and their two committed SVG charts (the chart generator reads `plans/gfql-gpu-pagerank-benchmark/results/`, a path that has never existed in any commit, so the charts cannot be regenerated by anyone), and every downstream echo of those figures (`~38x`, `9-28x vs Kuzu/Neo4j`, `43X+`, `10-50x`, `100X+`, `10X+`). `gfql/benchmark_graphframes.rst` is **kept intact** — its raw results are committed at `docs/source/gfql/_static/graphframes/results.json` and every headline cell round-trips against it. Removing an unverifiable number is the correct outcome, not a regression — but it is a large removal, so it is called out here rather than buried. ## [0.58.0 - 2026-07-22] diff --git a/bin/check_bench_numbers.py b/bin/check_bench_numbers.py new file mode 100755 index 0000000000..6a72fbb885 --- /dev/null +++ b/bin/check_bench_numbers.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 +"""Gate published benchmark numbers on the single source-of-truth. + +Runs the same data validation as the ``gfql_bench`` Sphinx extension, without +Sphinx and without re-running any benchmark, plus two checks a docs build cannot +make: + +1. **Commit drift** — how many commits touching the query engine have landed + since a published run was measured. This is the check that catches "the board + moved and nobody re-measured": a docs build is perfectly happy to render a + number taken on a six-PRs-ago tree. +2. **Hand-typed literals** — the managed pages may not contain a bare latency or + speedup literal. A number typed by hand is by construction not a reference to + the source-of-truth, which is how a figure becomes authoritative and then gets + re-copied. + +Exit status is non-zero on any finding, so it can gate CI and pre-commit. + +Usage:: + + bin/check_bench_numbers.py # full gate + bin/check_bench_numbers.py --no-git # skip the commit-drift check + bin/check_bench_numbers.py --today 2027-01-01 # what will break, and when +""" + +from __future__ import annotations + +import argparse +import datetime +import os +import re +import subprocess +import sys +from typing import Dict, List, Optional, Sequence, Tuple + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +DOCS_SOURCE = os.path.join(REPO_ROOT, 'docs', 'source') +sys.path.insert(0, os.path.join(DOCS_SOURCE, '_ext')) + +from gfql_bench_data import ( # noqa: E402 + BenchData, + BenchDataError, + load_bench_data, +) + +#: Paths whose churn invalidates a published query-latency number. +ENGINE_PATHS = ( + 'graphistry/compute', + 'graphistry/Engine.py', + 'graphistry/models/gfql', +) + +_ROLE_RE = re.compile(r':(bench|bench-diag):`([^`]+)`') +_PROVENANCE_RE = re.compile(r'^\s*\.\.\s+bench-provenance::\s*(\S+)\s*$', re.MULTILINE) +_DISCLOSURES_RE = re.compile(r'^\s*\.\.\s+bench-disclosures::\s*$', re.MULTILINE) + +#: A latency literal: "68 ms", "13.83s", "1.20s". +_LATENCY_RE = re.compile(r'(? None: + self.where = where + self.message = message + + def render(self) -> str: + return '{}: {}'.format(self.where, self.message) + + +def _read(path: str) -> str: + with open(path, 'r', encoding='utf-8') as handle: + return handle.read() + + +def _git(args: Sequence[str]) -> Optional[str]: + try: + completed = subprocess.run( + ['git'] + list(args), + cwd=REPO_ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + except OSError: + return None + if completed.returncode != 0: + return None + return completed.stdout.decode('utf-8', 'replace') + + +def check_reference_integrity(data: BenchData) -> List[Finding]: + """Every ``:bench:`` reference resolves, and its page carries the caveats.""" + findings: List[Finding] = [] + for rel in data.policy.managed_docs: + path = os.path.join(DOCS_SOURCE, rel) + if not os.path.isfile(path): + findings.append(Finding(rel, 'managed doc is listed in policy.managed_docs but does not exist')) + continue + text = _read(path) + keys = [match.group(2) for match in _ROLE_RE.finditer(text)] + declared_runs = set(_PROVENANCE_RE.findall(text)) + has_disclosures = bool(_DISCLOSURES_RE.search(text)) + + needed_runs: List[str] = [] + needs_disclosure = False + for key in keys: + if key not in data.cells: + findings.append(Finding(rel, 'references unknown benchmark key {!r}'.format(key))) + continue + cell = data.cells[key] + if cell.run_id not in needed_runs: + needed_runs.append(cell.run_id) + if cell.disclosures: + needs_disclosure = True + for run_id in needed_runs: + if run_id not in declared_runs: + findings.append(Finding( + rel, 'publishes run {!r} without a `.. bench-provenance:: {}` block'.format(run_id, run_id))) + if needs_disclosure and not has_disclosures: + findings.append(Finding( + rel, 'publishes a disclosure-bearing number but has no `.. bench-disclosures::` block')) + return findings + + +def check_freshness(data: BenchData, today: datetime.date) -> List[Finding]: + findings: List[Finding] = [] + for run, age in data.stale_runs(today): + findings.append(Finding( + 'runs.{}'.format(run.run_id), + 'measured {} ({} days ago); policy.max_age_days is {}. Re-measure or drop the claim.'.format( + run.measured_at.isoformat(), age, data.policy.max_age_days))) + return findings + + +def check_commit_drift(data: BenchData) -> List[Finding]: + """Fail when the query engine has moved materially since a run was measured.""" + findings: List[Finding] = [] + limit = data.policy.max_compute_commit_drift + for run_id in sorted(data.runs): + run = data.runs[run_id] + exists = _git(['cat-file', '-e', run.pygraphistry_commit + '^{commit}']) + if exists is None: + findings.append(Finding( + 'runs.{}'.format(run_id), + 'pygraphistry_commit {} is not in this repository, so the run cannot be placed in ' + 'history and its numbers cannot be validated'.format(run.pygraphistry_commit))) + continue + log = _git(['log', '--oneline', '{}..HEAD'.format(run.pygraphistry_commit), '--'] + list(ENGINE_PATHS)) + if log is None: + findings.append(Finding( + 'runs.{}'.format(run_id), + 'could not compute commit drift from {}'.format(run.pygraphistry_commit))) + continue + drift = len([line for line in log.splitlines() if line.strip()]) + if drift > limit: + findings.append(Finding( + 'runs.{}'.format(run_id), + '{} commits touching {} have landed since {} was measured (policy max {}). ' + 'The published numbers describe a tree that no longer exists.'.format( + drift, '/'.join(ENGINE_PATHS), run.pygraphistry_commit, limit))) + return findings + + +def _strip_literal_directives(text: str) -> str: + """Blank out regions where a literal is not a published claim. + + Code blocks, shell recipes and inline literals hold reproduction commands and + API names, not board numbers. + """ + literal_start = re.compile(r'^\s*\.\.\s+(?:code-block|parsed-literal|literalinclude|math)::') + literal_paragraph = re.compile(r'(? List[Finding]: + findings: List[Finding] = [] + for rel in data.policy.managed_docs: + path = os.path.join(DOCS_SOURCE, rel) + if not os.path.isfile(path): + continue + allowed = data.policy.allowed_literals(rel) + text = _strip_literal_directives(_read(path)) + for lineno, line in enumerate(text.split('\n'), start=1): + without_roles = _ROLE_RE.sub('', line) + for pattern in (_LATENCY_RE, _SPEEDUP_RE): + for match in pattern.finditer(without_roles): + literal = match.group(0).strip() + if literal in allowed: + continue + findings.append(Finding( + '{}:{}'.format(rel, lineno), + 'hand-typed benchmark literal {!r}. Publish it as :bench:`` from the ' + 'source-of-truth, or add it to policy.literal_allowlist if it is not a ' + 'measured claim.'.format(literal))) + return findings + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--data', default=None, help='path to gfql_benchmarks.json') + parser.add_argument('--today', default=None, help='YYYY-MM-DD; override the clock for freshness checks') + parser.add_argument('--no-git', action='store_true', help='skip the commit-drift check') + args = parser.parse_args(argv) + + try: + data = load_bench_data(args.data) + except BenchDataError as exc: + sys.stderr.write('FAIL benchmark source-of-truth is invalid: {}\n'.format(exc)) + return 2 + + if args.today: + year, month, day = args.today.split('-') + today = datetime.date(int(year), int(month), int(day)) + else: + today = datetime.date.today() + + checks: List[Tuple[str, List[Finding]]] = [ + ('reference integrity', check_reference_integrity(data)), + ('freshness', check_freshness(data, today)), + ('hand-typed literals', check_hand_typed_literals(data)), + ] + if not args.no_git: + checks.append(('commit drift', check_commit_drift(data))) + + total = 0 + for name, findings in checks: + if findings: + total += len(findings) + sys.stderr.write('\nFAIL {} ({} finding(s)):\n'.format(name, len(findings))) + for finding in findings: + sys.stderr.write(' - {}\n'.format(finding.render())) + else: + sys.stdout.write('ok {}\n'.format(name)) + + if total: + sys.stderr.write( + '\n{} published benchmark number(s) cannot be defended from {}.\n'.format( + total, os.path.relpath(data.source_path, REPO_ROOT))) + return 1 + sys.stdout.write('\nAll published benchmark numbers trace to {} run(s) with full provenance.\n'.format( + len(data.runs))) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/docs/.rstcheck.cfg b/docs/.rstcheck.cfg index 90a6d5d857..322035f4f4 100644 --- a/docs/.rstcheck.cfg +++ b/docs/.rstcheck.cfg @@ -20,7 +20,9 @@ ignore_roles = py:attr, py:exc, py:obj, - py:data + py:data, + bench, + bench-diag # Ignore Sphinx-specific directives ignore_directives = @@ -43,7 +45,9 @@ ignore_directives = glossary, productionlist, include, - graphviz + graphviz, + bench-provenance, + bench-disclosures # Ignore common informational messages ignore_messages = (Hyperlink target "[^"]*" is not referenced\.$) diff --git a/docs/source/10min.rst b/docs/source/10min.rst index d8cfd79e14..4e85647d09 100644 --- a/docs/source/10min.rst +++ b/docs/source/10min.rst @@ -69,7 +69,7 @@ Loading Data Efficiently The Python data science ecosystem supports connecting to most databases and file type types -Many users start with CSV, JSON, and SQL database. We often see teams adopt formats like **Parquet** and **Apache Arrow**. Graphistry natively leverages these, so loading data with them can often be 10X+ faster than typical libraries. +Many users start with CSV, JSON, and SQL database. We often see teams adopt formats like **Parquet** and **Apache Arrow**. Graphistry natively leverages these, so loading data with them is typically much faster than with row-oriented formats. **Example: Loading Parquet Data** diff --git a/docs/source/_data/gfql_benchmarks.json b/docs/source/_data/gfql_benchmarks.json new file mode 100644 index 0000000000..9023e82fae --- /dev/null +++ b/docs/source/_data/gfql_benchmarks.json @@ -0,0 +1,1488 @@ +{ + "cells": { + "graphbench.100k.q1.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 153.16, + "workload": "graph-benchmark 100k q1" + }, + "graphbench.100k.q1.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 199.43, + "workload": "graph-benchmark 100k q1" + }, + "graphbench.100k.q1.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 0.77, + "workload": "graph-benchmark 100k q1 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.100k.q1.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 42.14, + "workload": "graph-benchmark 100k q1" + }, + "graphbench.100k.q1.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 3.63, + "workload": "graph-benchmark 100k q1 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.100k.q1.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 4.7, + "workload": "graph-benchmark 100k q1 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.100k.q2.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 281.91, + "workload": "graph-benchmark 100k q2" + }, + "graphbench.100k.q2.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 208.88, + "workload": "graph-benchmark 100k q2" + }, + "graphbench.100k.q2.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 1.35, + "workload": "graph-benchmark 100k q2 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.100k.q2.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 55.76, + "workload": "graph-benchmark 100k q2" + }, + "graphbench.100k.q2.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 5.06, + "workload": "graph-benchmark 100k q2 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.100k.q2.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 3.7, + "workload": "graph-benchmark 100k q2 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.100k.q3.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 28.76, + "workload": "graph-benchmark 100k q3" + }, + "graphbench.100k.q3.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 73.72, + "workload": "graph-benchmark 100k q3" + }, + "graphbench.100k.q3.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 0.39, + "workload": "graph-benchmark 100k q3 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.100k.q3.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 14.74, + "workload": "graph-benchmark 100k q3" + }, + "graphbench.100k.q3.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 1.95, + "workload": "graph-benchmark 100k q3 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.100k.q3.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 5.0, + "workload": "graph-benchmark 100k q3 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.100k.q4.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 11.56, + "workload": "graph-benchmark 100k q4" + }, + "graphbench.100k.q4.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 64.09, + "workload": "graph-benchmark 100k q4" + }, + "graphbench.100k.q4.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 0.18, + "workload": "graph-benchmark 100k q4 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.100k.q4.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 12.13, + "workload": "graph-benchmark 100k q4" + }, + "graphbench.100k.q4.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 0.95, + "workload": "graph-benchmark 100k q4 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.100k.q4.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 5.3, + "workload": "graph-benchmark 100k q4 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.100k.q5.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 12.88, + "workload": "graph-benchmark 100k q5" + }, + "graphbench.100k.q5.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 410.05, + "workload": "graph-benchmark 100k q5" + }, + "graphbench.100k.q5.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 0.03, + "workload": "graph-benchmark 100k q5 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.100k.q5.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 17.52, + "workload": "graph-benchmark 100k q5" + }, + "graphbench.100k.q5.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 0.74, + "workload": "graph-benchmark 100k q5 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.100k.q5.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 23.4, + "workload": "graph-benchmark 100k q5 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.100k.q6.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 20.73, + "workload": "graph-benchmark 100k q6" + }, + "graphbench.100k.q6.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 412.3, + "workload": "graph-benchmark 100k q6" + }, + "graphbench.100k.q6.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 0.05, + "workload": "graph-benchmark 100k q6 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.100k.q6.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 21.99, + "workload": "graph-benchmark 100k q6" + }, + "graphbench.100k.q6.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 0.94, + "workload": "graph-benchmark 100k q6 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.100k.q6.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 18.8, + "workload": "graph-benchmark 100k q6 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.100k.q7.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 8.85, + "workload": "graph-benchmark 100k q7" + }, + "graphbench.100k.q7.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 119.63, + "workload": "graph-benchmark 100k q7" + }, + "graphbench.100k.q7.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 0.07, + "workload": "graph-benchmark 100k q7 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.100k.q7.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 16.12, + "workload": "graph-benchmark 100k q7" + }, + "graphbench.100k.q7.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 0.55, + "workload": "graph-benchmark 100k q7 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.100k.q7.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 7.4, + "workload": "graph-benchmark 100k q7 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.100k.q8.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 9.76, + "workload": "graph-benchmark 100k q8" + }, + "graphbench.100k.q8.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 91.6, + "workload": "graph-benchmark 100k q8" + }, + "graphbench.100k.q8.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 0.11, + "workload": "graph-benchmark 100k q8 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.100k.q8.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 5.06, + "workload": "graph-benchmark 100k q8" + }, + "graphbench.100k.q8.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 1.93, + "workload": "graph-benchmark 100k q8 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.100k.q8.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 18.1, + "workload": "graph-benchmark 100k q8 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.100k.q9.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 84.06, + "workload": "graph-benchmark 100k q9" + }, + "graphbench.100k.q9.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 212.0, + "workload": "graph-benchmark 100k q9" + }, + "graphbench.100k.q9.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 0.4, + "workload": "graph-benchmark 100k q9 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.100k.q9.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 66.61, + "workload": "graph-benchmark 100k q9" + }, + "graphbench.100k.q9.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 1.26, + "workload": "graph-benchmark 100k q9 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.100k.q9.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 3.2, + "workload": "graph-benchmark 100k q9 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.20k.q1.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 14.51, + "workload": "graph-benchmark 20k q1" + }, + "graphbench.20k.q1.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 35.46, + "workload": "graph-benchmark 20k q1" + }, + "graphbench.20k.q1.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.41, + "workload": "graph-benchmark 20k q1 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.20k.q1.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 13.13, + "workload": "graph-benchmark 20k q1" + }, + "graphbench.20k.q1.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 1.11, + "workload": "graph-benchmark 20k q1 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.20k.q1.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 2.7, + "workload": "graph-benchmark 20k q1 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.20k.q2.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 31.62, + "workload": "graph-benchmark 20k q2" + }, + "graphbench.20k.q2.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 38.05, + "workload": "graph-benchmark 20k q2" + }, + "graphbench.20k.q2.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.83, + "workload": "graph-benchmark 20k q2 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.20k.q2.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 17.59, + "workload": "graph-benchmark 20k q2" + }, + "graphbench.20k.q2.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 1.8, + "workload": "graph-benchmark 20k q2 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.20k.q2.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 2.2, + "workload": "graph-benchmark 20k q2 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.20k.q3.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 5.94, + "workload": "graph-benchmark 20k q3" + }, + "graphbench.20k.q3.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 11.68, + "workload": "graph-benchmark 20k q3" + }, + "graphbench.20k.q3.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.51, + "workload": "graph-benchmark 20k q3 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.20k.q3.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 7.66, + "workload": "graph-benchmark 20k q3" + }, + "graphbench.20k.q3.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.78, + "workload": "graph-benchmark 20k q3 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.20k.q3.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 1.5, + "workload": "graph-benchmark 20k q3 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.20k.q4.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 3.42, + "workload": "graph-benchmark 20k q4" + }, + "graphbench.20k.q4.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 9.45, + "workload": "graph-benchmark 20k q4" + }, + "graphbench.20k.q4.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.36, + "workload": "graph-benchmark 20k q4 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.20k.q4.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 6.04, + "workload": "graph-benchmark 20k q4" + }, + "graphbench.20k.q4.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.57, + "workload": "graph-benchmark 20k q4 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.20k.q4.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 1.6, + "workload": "graph-benchmark 20k q4 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.20k.q5.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 5.75, + "workload": "graph-benchmark 20k q5" + }, + "graphbench.20k.q5.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 81.03, + "workload": "graph-benchmark 20k q5" + }, + "graphbench.20k.q5.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.07, + "workload": "graph-benchmark 20k q5 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.20k.q5.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 8.9, + "workload": "graph-benchmark 20k q5" + }, + "graphbench.20k.q5.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.65, + "workload": "graph-benchmark 20k q5 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.20k.q5.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 9.1, + "workload": "graph-benchmark 20k q5 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.20k.q6.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 8.23, + "workload": "graph-benchmark 20k q6" + }, + "graphbench.20k.q6.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 81.25, + "workload": "graph-benchmark 20k q6" + }, + "graphbench.20k.q6.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.1, + "workload": "graph-benchmark 20k q6 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.20k.q6.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 10.43, + "workload": "graph-benchmark 20k q6" + }, + "graphbench.20k.q6.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.79, + "workload": "graph-benchmark 20k q6 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.20k.q6.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 7.8, + "workload": "graph-benchmark 20k q6 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.20k.q7.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 5.19, + "workload": "graph-benchmark 20k q7" + }, + "graphbench.20k.q7.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 19.97, + "workload": "graph-benchmark 20k q7" + }, + "graphbench.20k.q7.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.26, + "workload": "graph-benchmark 20k q7 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.20k.q7.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 10.16, + "workload": "graph-benchmark 20k q7" + }, + "graphbench.20k.q7.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.51, + "workload": "graph-benchmark 20k q7 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.20k.q7.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 2.0, + "workload": "graph-benchmark 20k q7 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.20k.q8.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 2.52, + "workload": "graph-benchmark 20k q8" + }, + "graphbench.20k.q8.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 10.46, + "workload": "graph-benchmark 20k q8" + }, + "graphbench.20k.q8.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.24, + "workload": "graph-benchmark 20k q8 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.20k.q8.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 2.07, + "workload": "graph-benchmark 20k q8" + }, + "graphbench.20k.q8.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 1.22, + "workload": "graph-benchmark 20k q8 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.20k.q8.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 5.1, + "workload": "graph-benchmark 20k q8 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.20k.q9.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 10.85, + "workload": "graph-benchmark 20k q9" + }, + "graphbench.20k.q9.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 28.29, + "workload": "graph-benchmark 20k q9" + }, + "graphbench.20k.q9.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.38, + "workload": "graph-benchmark 20k q9 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.20k.q9.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 13.83, + "workload": "graph-benchmark 20k q9" + }, + "graphbench.20k.q9.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.78, + "workload": "graph-benchmark 20k q9 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.20k.q9.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 2.0, + "workload": "graph-benchmark 20k q9 \u2014 GFQL-polars vs GFQL-pandas" + }, + "snb.sf1.is3.single_query.polars": { + "board_quotable": false, + "comparison_allowed": false, + "decimals": 2, + "disclosures": [ + "The A/B configuration this was measured under declares comparison_allowed: false, so it is a diagnostic against the previous GFQL form only, never a competitor result." + ], + "engine": "polars", + "run": "snb-is3-single-query-20260727", + "status": "ok", + "unit": "ms", + "value": 18.05, + "workload": "LDBC SNB IS3 (SF1), single-query form" + }, + "snb.sf1.is3.two_pass.polars": { + "board_quotable": false, + "comparison_allowed": false, + "decimals": 2, + "disclosures": [ + "Answered by a two-pass adapter workaround with a Python-side join inside the timed region; superseded, kept only as the baseline for the single-query improvement." + ], + "engine": "polars", + "run": "snb-is3-single-query-20260727", + "status": "partial", + "unit": "ms", + "value": 75.22, + "workload": "LDBC SNB IS3 (SF1), superseded two-pass adapter form" + } + }, + "generated_at": "2026-07-28T01:02:35Z", + "generated_by": "graphistry/pyg-bench a681bd1f scripts/export_docs_numbers.py", + "policy": { + "literal_allowlist": {}, + "managed_docs": [ + "gfql/performance.rst", + "gfql/index_adjacency.rst", + "gfql/indexing.rst", + "gfql/engines.rst", + "gfql/benchmark_filter_pagerank.rst", + "gfql/index.rst", + "gfql/about.rst", + "gfql/overview.rst", + "gfql/quick.rst", + "gfql/builtin_calls.rst", + "cheatsheet.md", + "10min.rst", + "performance.rst" + ], + "max_age_days": 60, + "max_compute_commit_drift": 12 + }, + "runs": { + "graphbench-q1q9-100k-20260726": { + "artifact": "results/graphbench-matched-q1q9-20260726/gb100k", + "competitor": "Kuzu", + "competitor_version": "0.11.3 (embedded, CPU)", + "dataset": "prrao87 graph-benchmark, 100,000 persons / 2,417,738 FOLLOWS / 2,775,195 edges", + "host": "dgx-spark (NVIDIA GB10), 20 CPU", + "measured_at": "2026-07-26", + "perf_lock_held": true, + "pyg_bench_commit": "47f94ba", + "pygraphistry_commit": "84be35fb", + "quiet_host": true, + "reps": "2 warmups + 5 timed runs per slot; slot median, then median across 4 position-balanced slots (K G G K K G G K)", + "row_validation": "every cell emits a canonical, column-order-independent row set; a GFQL cell is only publishable when its rows equal Kuzu's on every slot", + "runtime": "graphistry/test-rapids-official:26.02-gfql-polars, polars 1.35.2; Kuzu in a host venv on CPU" + }, + "graphbench-q1q9-20k-20260726": { + "artifact": "results/graphbench-matched-q1q9-20260726/gb20k", + "competitor": "Kuzu", + "competitor_version": "0.11.3 (embedded, CPU)", + "dataset": "prrao87 graph-benchmark, 20,000 persons / 199,939 FOLLOWS / 260,107 edges", + "host": "dgx-spark (NVIDIA GB10), 20 CPU", + "measured_at": "2026-07-26", + "perf_lock_held": true, + "pyg_bench_commit": "47f94ba", + "pygraphistry_commit": "84be35fb", + "quiet_host": true, + "reps": "3 warmups + 7 timed runs per slot; slot median, then median across 4 position-balanced slots (K G I I G K K G I I G K)", + "row_validation": "every cell emits a canonical, column-order-independent row set; a GFQL cell is only publishable when its rows equal Kuzu's on every slot", + "runtime": "graphistry/test-rapids-official:26.02-gfql-polars, polars 1.35.2; Kuzu in a host venv on CPU" + }, + "snb-is3-single-query-20260727": { + "artifact": "results/is3-single-query-r{1,2,3} and results/is3-twopass-base-r{1,2,3}", + "competitor": "Neo4j", + "competitor_version": "not re-paired in this configuration", + "dataset": "LDBC SNB Interactive SF1, official format", + "host": "dgx-spark (NVIDIA GB10), 20 CPU", + "measured_at": "2026-07-27", + "perf_lock_held": true, + "pyg_bench_commit": "64864b8", + "pygraphistry_commit": "233b64c8", + "quiet_host": true, + "reps": "3 repetitions of configs/suites/ab-is1-is7-sf1-polars.yaml, each already a median of 5 measured repeats after 1 warmup; median across repetitions", + "row_validation": "920/920 rows, value-identical to the independently derived expected fixture in all 3 repetitions", + "runtime": "graphistry/test-rapids-official:26.02-gfql-polars, polars 1.35.2" + } + }, + "schema_version": 1 +} diff --git a/docs/source/_ext/gfql_bench.py b/docs/source/_ext/gfql_bench.py new file mode 100644 index 0000000000..b10ebfc654 --- /dev/null +++ b/docs/source/_ext/gfql_bench.py @@ -0,0 +1,347 @@ +"""Sphinx extension: publish GFQL benchmark numbers from the source-of-truth. + +Docs never restate a measured number. They *reference* it: + +.. code-block:: rst + + * - 1-hop from 10K seeds + - :bench:`orkut.hop1_10k.pandas` + - :bench:`orkut.hop1_10k.polars` + +and every page that references a cell must also carry that cell's provenance and +disclosures: + +.. code-block:: rst + + .. bench-provenance:: orkut-4engine-20260703 + .. bench-disclosures:: + +Failure modes are build failures, never silent text: + +- a key the source-of-truth does not contain -> build fails +- a run older than ``policy.max_age_days`` -> build fails +- a non-board-quotable cell used as a bare number -> build fails +- a page that drops a cell's provenance/disclosure -> build fails + +``bin/check_bench_numbers.py`` runs the same data checks without Sphinx, plus a +commit-drift check against the current tree and a hand-typed-literal guard over +the managed pages, so CI can gate on staleness a pure docs build cannot see. +""" + +from __future__ import annotations + +import datetime +import os +from typing import Dict, List, Optional, Tuple, Union + +from docutils import nodes +from docutils.parsers.rst import Directive +from docutils.parsers.rst.states import Inliner +from sphinx.application import Sphinx +from sphinx.environment import BuildEnvironment +from sphinx.errors import SphinxError +from sphinx.util import logging as sphinx_logging + +from gfql_bench_data import ( + BenchCell, + BenchData, + BenchDataError, + BenchRun, + load_bench_data, +) + +logger = sphinx_logging.getLogger(__name__) + +RoleResult = Tuple[List[nodes.Node], List[nodes.system_message]] + + +class BenchNumberError(SphinxError): + category = 'GFQL benchmark number check failed' + + +class _State: + """Loaded once per build: the data, the clock, and the problems found.""" + + def __init__(self, data: BenchData, today: datetime.date) -> None: + self.data = data + self.today = today + self.problems: List[str] = [] + #: docname -> benchmark keys referenced by that page, in source order + self.refs: Dict[str, List[str]] = {} + #: docname -> run ids whose provenance the page renders + self.provenance: Dict[str, List[str]] = {} + #: docnames carrying a ``bench-disclosures`` block + self.disclosed: List[str] = [] + + def fail(self, message: str) -> None: + self.problems.append(message) + logger.warning('[gfql-bench] %s', message) + + def forget(self, docname: str) -> None: + self.refs.pop(docname, None) + self.provenance.pop(docname, None) + if docname in self.disclosed: + self.disclosed.remove(docname) + + +_STATE: Optional[_State] = None + + +def _state() -> _State: + if _STATE is None: + raise BenchNumberError('gfql_bench extension used before builder-inited') + return _STATE + + +def _bench_role_impl(key: str, rawtext: str, lineno: int, inliner: Inliner, diagnostic: bool) -> RoleResult: + state = _state() + env: BuildEnvironment = inliner.document.settings.env + docname = env.docname + state.refs.setdefault(docname, []).append(key) + + try: + cell: Optional[BenchCell] = state.data.cell(key) + except BenchDataError as exc: + state.fail('{}:{}: {}'.format(docname, lineno, exc)) + cell = None + if cell is None: + return [nodes.strong(rawtext, '[MISSING BENCHMARK NUMBER: {}]'.format(key))], [] + + if diagnostic and cell.board_quotable: + state.fail( + '{}:{}: {!r} is board-quotable; use :bench: rather than :bench-diag:'.format(docname, lineno, key) + ) + if not diagnostic and not cell.board_quotable: + state.fail( + '{}:{}: {!r} is NOT board-quotable (status={}, comparison_allowed={}); it may only be ' + 'published via :bench-diag:, which labels it diagnostic-only'.format( + docname, lineno, key, cell.status, cell.comparison_allowed) + ) + + text = cell.render() + rendered: nodes.Node = nodes.strong(rawtext, text) if cell.board_quotable else nodes.Text(text) + result: List[nodes.Node] = [rendered] + if diagnostic: + result.append(nodes.Text(' (diagnostic only — not a board result)')) + return result, [] + + +def bench_role( + name: str, + rawtext: str, + text: str, + lineno: int, + inliner: Inliner, + options: Optional[Dict[str, str]] = None, + content: Optional[List[str]] = None, +) -> RoleResult: + return _bench_role_impl(text.strip(), rawtext, lineno, inliner, diagnostic=False) + + +def bench_diag_role( + name: str, + rawtext: str, + text: str, + lineno: int, + inliner: Inliner, + options: Optional[Dict[str, str]] = None, + content: Optional[List[str]] = None, +) -> RoleResult: + return _bench_role_impl(text.strip(), rawtext, lineno, inliner, diagnostic=True) + + +def _field(label: str, value: str) -> nodes.definition_list_item: + item = nodes.definition_list_item() + item += nodes.term('', label) + definition = nodes.definition() + definition += nodes.paragraph('', value) + item += definition + return item + + +def _provenance_block(run: BenchRun) -> nodes.Element: + lock = 'perf lock held for the whole session' if run.perf_lock_held else 'PERF LOCK NOT HELD' + quiet = 'quiet host' if run.quiet_host else 'host contention not established' + listing = nodes.definition_list() + listing += _field('Measured', '{} on {} ({}, {})'.format( + run.measured_at.isoformat(), run.host, quiet, lock)) + listing += _field('pygraphistry', run.pygraphistry_commit) + listing += _field('Benchmark harness', 'graphistry/pyg-bench {}'.format(run.pyg_bench_commit)) + listing += _field('Runtime', run.runtime) + listing += _field('Dataset', run.dataset) + listing += _field('Protocol', run.reps) + listing += _field('Result validation', run.row_validation) + if run.competitor is not None: + listing += _field('Competitor', '{} {}'.format( + run.competitor, run.competitor_version or 'version unrecorded')) + listing += _field('Raw artifacts', 'graphistry/pyg-bench {}'.format(run.artifact)) + + admonition = nodes.admonition() + admonition['classes'] = ['note', 'gfql-bench-provenance'] + admonition += nodes.title('', 'Provenance: {}'.format(run.run_id)) + admonition += listing + return admonition + + +class BenchProvenanceDirective(Directive): + """``.. bench-provenance:: `` — render a run's full provenance.""" + + has_content = False + required_arguments = 1 + optional_arguments = 0 + + def run(self) -> List[nodes.Node]: + state = _state() + env: BuildEnvironment = self.state.document.settings.env + docname = env.docname + run_id = self.arguments[0].strip() + state.provenance.setdefault(docname, []).append(run_id) + run = state.data.runs.get(run_id) + if run is None: + state.fail('{}: unknown benchmark run {!r}'.format(docname, run_id)) + return [nodes.strong('', '[UNKNOWN BENCHMARK RUN: {}]'.format(run_id))] + return [_provenance_block(run)] + + +class BenchDisclosuresDirective(Directive): + """``.. bench-disclosures::`` — render every disclosure this page owes. + + The body is generated from the source-of-truth, so a caveat cannot be + paraphrased away or silently dropped when a number is refreshed. + """ + + has_content = False + required_arguments = 0 + optional_arguments = 0 + + def run(self) -> List[nodes.Node]: + state = _state() + env: BuildEnvironment = self.state.document.settings.env + docname = env.docname + if docname not in state.disclosed: + state.disclosed.append(docname) + placeholder = nodes.container() + placeholder['classes'] = ['gfql-bench-disclosures-placeholder'] + return [placeholder] + + +def _disclosure_lines(state: _State, docname: str) -> List[str]: + lines: List[str] = [] + for key in state.refs.get(docname, []): + cell = state.data.cells.get(key) + if cell is None: + continue + for disclosure in cell.disclosures: + line = '{}: {}'.format(cell.workload, disclosure) + if line not in lines: + lines.append(line) + return lines + + +def _fill_disclosures(app: Sphinx, doctree: nodes.document, docname: str) -> None: + state = _state() + for container in list(doctree.findall(nodes.container)): + if 'gfql-bench-disclosures-placeholder' not in container['classes']: + continue + admonition = nodes.admonition() + admonition['classes'] = ['important', 'gfql-bench-disclosures'] + admonition += nodes.title('', 'Disclosures that travel with these numbers') + lines = _disclosure_lines(state, docname) + if lines: + bullet = nodes.bullet_list() + for line in lines: + item = nodes.list_item() + item += nodes.paragraph('', line) + bullet += item + admonition += bullet + else: + admonition += nodes.paragraph( + '', 'Every number on this page is a clean, row-validated result with no caveat.') + container.replace_self(admonition) + + +def _check_consistency(app: Sphinx, env: BuildEnvironment) -> None: + state = _state() + for docname in sorted(state.refs): + keys = state.refs[docname] + if not keys: + continue + needed_runs: List[str] = [] + needs_disclosure = False + for key in keys: + cell = state.data.cells.get(key) + if cell is None: + continue + if cell.run_id not in needed_runs: + needed_runs.append(cell.run_id) + if cell.disclosures: + needs_disclosure = True + have_runs = state.provenance.get(docname, []) + for run_id in needed_runs: + if run_id not in have_runs: + state.fail('{}: publishes numbers from run {!r} without a ' + '`.. bench-provenance:: {}` block'.format(docname, run_id, run_id)) + run = state.data.runs[run_id] + age = run.age_days(state.today) + if age > state.data.policy.max_age_days: + state.fail('{}: run {!r} was measured {} days ago (policy max {}); ' + 're-measure it or remove the claim'.format( + docname, run_id, age, state.data.policy.max_age_days)) + if needs_disclosure and docname not in state.disclosed: + state.fail('{}: publishes a number that carries a disclosure but has no ' + '`.. bench-disclosures::` block — a bare ratio without its asterisk ' + 'launders the caveat'.format(docname)) + + if state.problems: + raise BenchNumberError( + 'benchmark numbers failed validation ({} problem(s)):\n - {}'.format( + len(state.problems), '\n - '.join(state.problems))) + + +def _builder_inited(app: Sphinx) -> None: + global _STATE + configured = str(app.config.gfql_bench_data_path) + path: Optional[str] = configured or None + if path is not None and not os.path.isabs(path): + path = os.path.join(str(app.srcdir), path) + data = load_bench_data(path) + today_setting = str(app.config.gfql_bench_today) + if today_setting: + year, month, day = today_setting.split('-') + today = datetime.date(int(year), int(month), int(day)) + else: + today = datetime.date.today() + _STATE = _State(data, today) + + # Freshness is checked for EVERY run in the file, not just the ones this build + # happens to re-read: an incremental build can serve a cached page whose numbers + # have aged out, and that page must not be publishable either. + stale = data.stale_runs(today) + if stale: + raise BenchNumberError( + 'benchmark run(s) past policy.max_age_days={}:\n - {}'.format( + data.policy.max_age_days, + '\n - '.join( + '{} measured {} ({} days ago)'.format( + run.run_id, run.measured_at.isoformat(), age) + for run, age in stale))) + + +def _purge_doc(app: Sphinx, env: BuildEnvironment, docname: str) -> None: + if _STATE is not None: + _STATE.forget(docname) + + +def setup(app: Sphinx) -> Dict[str, Union[str, bool]]: + app.add_config_value('gfql_bench_data_path', '', 'env') + app.add_config_value('gfql_bench_today', '', 'env') + app.add_role('bench', bench_role) + app.add_role('bench-diag', bench_diag_role) + app.add_directive('bench-provenance', BenchProvenanceDirective) + app.add_directive('bench-disclosures', BenchDisclosuresDirective) + app.connect('builder-inited', _builder_inited) + app.connect('env-purge-doc', _purge_doc) + app.connect('doctree-resolved', _fill_disclosures) + app.connect('env-check-consistency', _check_consistency) + # Bookkeeping lives in module state, not the pickled env: single-process reads only. + return {'version': '1', 'parallel_read_safe': False, 'parallel_write_safe': True} diff --git a/docs/source/_ext/gfql_bench_data.py b/docs/source/_ext/gfql_bench_data.py new file mode 100644 index 0000000000..dc269d7dcf --- /dev/null +++ b/docs/source/_ext/gfql_bench_data.py @@ -0,0 +1,420 @@ +"""Typed loader + validator for the GFQL benchmark source-of-truth. + +Single machine-readable source of every benchmark number published in the docs +(``docs/source/_data/gfql_benchmarks.json``), produced by the pyg-bench exporter +``scripts/export_docs_numbers.py``. + +This module is deliberately Sphinx-free so that ``bin/check_bench_numbers.py`` +can validate the data (and the docs that reference it) without building the +docs, and without re-running any benchmark. + +Design rules enforced here: + +- A cell may only be published if it is attached to a run with full provenance. +- A cell that is not board-quotable (``comparison_allowed`` false, or a + non-``ok`` status that has not been dispositioned) cannot be rendered by the + plain reference form; the caller must ask for the diagnostic form explicitly. +- A cell's disclosures travel with it: the consumer is told which disclosures a + page owes, and it is an error to publish a disclosed cell without them. +""" + +from __future__ import annotations + +import datetime +import json +import os +import re +from typing import Dict, List, Optional, Sequence, Tuple, Union + +__all__ = [ + 'BenchDataError', + 'BenchCell', + 'BenchRun', + 'BenchPolicy', + 'BenchData', + 'load_bench_data', + 'default_data_path', +] + +JsonScalar = Union[str, int, float, bool, None] +JsonValue = Union[JsonScalar, List['JsonValue'], Dict[str, 'JsonValue']] +JsonObject = Dict[str, 'JsonValue'] + +SCHEMA_VERSION = 1 + +#: Statuses a cell may carry. Anything other than ``ok`` is a disclosure-bearing +#: result and may never be published as a bare number. +VALID_STATUSES = ( + 'ok', # clean result, values verified + 'partial', # answered, but with a documented workaround + 'adapter_workaround', # the harness, not GFQL, shaped the query + 'result_mismatch', # rows disagree with the reference => unquotable + 'unsupported', # the engine declined the query +) + +#: Statuses that are still publishable on a board, provided their disclosures +#: are rendered alongside. +BOARD_STATUSES = ('ok', 'partial', 'adapter_workaround') + +VALID_UNITS = ('ms', 's', 'x', 'rows') + +_KEY_RE = re.compile(r'^[a-z0-9]+(?:[._-][a-z0-9]+)*$') +_DATE_RE = re.compile(r'^\d{4}-\d{2}-\d{2}$') +_SHA_RE = re.compile(r'^[0-9a-f]{7,40}$') + + +class BenchDataError(Exception): + """Raised when the benchmark source-of-truth is malformed or unpublishable.""" + + +def _obj(value: JsonValue, where: str) -> JsonObject: + if not isinstance(value, dict): + raise BenchDataError('{}: expected an object, got {}'.format(where, type(value).__name__)) + return value + + +def _req_str(obj: JsonObject, key: str, where: str) -> str: + value = obj.get(key) + if not isinstance(value, str) or not value: + raise BenchDataError('{}: missing/invalid string field {!r}'.format(where, key)) + return value + + +def _opt_str(obj: JsonObject, key: str, where: str) -> Optional[str]: + value = obj.get(key) + if value is None: + return None + if not isinstance(value, str) or not value: + raise BenchDataError('{}: field {!r} must be a non-empty string or absent'.format(where, key)) + return value + + +def _req_bool(obj: JsonObject, key: str, where: str) -> bool: + value = obj.get(key) + if not isinstance(value, bool): + raise BenchDataError('{}: missing/invalid boolean field {!r}'.format(where, key)) + return value + + +def _req_int(obj: JsonObject, key: str, where: str) -> int: + value = obj.get(key) + if not isinstance(value, int) or isinstance(value, bool): + raise BenchDataError('{}: missing/invalid integer field {!r}'.format(where, key)) + return value + + +def _req_number(obj: JsonObject, key: str, where: str) -> float: + value = obj.get(key) + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise BenchDataError('{}: missing/invalid numeric field {!r}'.format(where, key)) + return float(value) + + +def _str_list(value: JsonValue, where: str) -> List[str]: + if not isinstance(value, list): + raise BenchDataError('{}: expected a list of strings'.format(where)) + out: List[str] = [] + for i, item in enumerate(value): + if not isinstance(item, str) or not item: + raise BenchDataError('{}[{}] must be a non-empty string'.format(where, i)) + out.append(item) + return out + + +def _req_str_list(obj: JsonObject, key: str, where: str) -> List[str]: + if key not in obj: + raise BenchDataError('{}: missing list field {!r}'.format(where, key)) + return _str_list(obj[key], '{}.{}'.format(where, key)) + + +class BenchRun: + """Provenance for one measurement session. Every field is mandatory. + + A run without a commit, a host, a perf-lock disposition and a rep count is + not a measurement anyone can defend, so the loader refuses to construct one. + """ + + def __init__( + self, + run_id: str, + measured_at: datetime.date, + host: str, + perf_lock_held: bool, + quiet_host: bool, + reps: str, + pygraphistry_commit: str, + pyg_bench_commit: str, + runtime: str, + dataset: str, + artifact: str, + row_validation: str, + competitor: Optional[str], + competitor_version: Optional[str], + ) -> None: + self.run_id = run_id + self.measured_at = measured_at + self.host = host + self.perf_lock_held = perf_lock_held + self.quiet_host = quiet_host + self.reps = reps + self.pygraphistry_commit = pygraphistry_commit + self.pyg_bench_commit = pyg_bench_commit + self.runtime = runtime + self.dataset = dataset + self.artifact = artifact + self.row_validation = row_validation + self.competitor = competitor + self.competitor_version = competitor_version + + def age_days(self, today: datetime.date) -> int: + return (today - self.measured_at).days + + @staticmethod + def from_json(run_id: str, raw: JsonValue) -> 'BenchRun': + where = 'runs.{}'.format(run_id) + obj = _obj(raw, where) + measured_at_str = _req_str(obj, 'measured_at', where) + if not _DATE_RE.match(measured_at_str): + raise BenchDataError('{}: measured_at must be YYYY-MM-DD, got {!r}'.format(where, measured_at_str)) + year, month, day = measured_at_str.split('-') + commit = _req_str(obj, 'pygraphistry_commit', where) + if not _SHA_RE.match(commit): + raise BenchDataError('{}: pygraphistry_commit must be a git sha, got {!r}'.format(where, commit)) + bench_commit = _req_str(obj, 'pyg_bench_commit', where) + if not _SHA_RE.match(bench_commit): + raise BenchDataError('{}: pyg_bench_commit must be a git sha, got {!r}'.format(where, bench_commit)) + return BenchRun( + run_id=run_id, + measured_at=datetime.date(int(year), int(month), int(day)), + host=_req_str(obj, 'host', where), + perf_lock_held=_req_bool(obj, 'perf_lock_held', where), + quiet_host=_req_bool(obj, 'quiet_host', where), + reps=_req_str(obj, 'reps', where), + pygraphistry_commit=commit, + pyg_bench_commit=bench_commit, + runtime=_req_str(obj, 'runtime', where), + dataset=_req_str(obj, 'dataset', where), + artifact=_req_str(obj, 'artifact', where), + row_validation=_req_str(obj, 'row_validation', where), + competitor=_opt_str(obj, 'competitor', where), + competitor_version=_opt_str(obj, 'competitor_version', where), + ) + + +class BenchCell: + """One published number, with the disclosures that must travel with it.""" + + def __init__( + self, + key: str, + run_id: str, + workload: str, + engine: str, + value: float, + unit: str, + decimals: int, + status: str, + comparison_allowed: bool, + board_quotable: bool, + disclosures: List[str], + competitor: Optional[str], + ) -> None: + self.key = key + self.run_id = run_id + self.workload = workload + self.engine = engine + self.value = value + self.unit = unit + self.decimals = decimals + self.status = status + self.comparison_allowed = comparison_allowed + self.board_quotable = board_quotable + self.disclosures = disclosures + self.competitor = competitor + + def render(self) -> str: + """The published text for this cell, including its unit.""" + number = '{:.{}f}'.format(self.value, self.decimals) + if self.unit == 'x': + return number + '×' + if self.unit == 'rows': + return '{:,} rows'.format(int(self.value)) + return '{} {}'.format(number, self.unit) + + @staticmethod + def from_json(key: str, raw: JsonValue) -> 'BenchCell': + where = 'cells.{}'.format(key) + if not _KEY_RE.match(key): + raise BenchDataError('{}: key must be lowercase dotted/underscored ascii'.format(where)) + obj = _obj(raw, where) + unit = _req_str(obj, 'unit', where) + if unit not in VALID_UNITS: + raise BenchDataError('{}: unit {!r} not in {}'.format(where, unit, VALID_UNITS)) + status = _req_str(obj, 'status', where) + if status not in VALID_STATUSES: + raise BenchDataError('{}: status {!r} not in {}'.format(where, status, VALID_STATUSES)) + comparison_allowed = _req_bool(obj, 'comparison_allowed', where) + board_quotable = _req_bool(obj, 'board_quotable', where) + disclosures = _req_str_list(obj, 'disclosures', where) + + if board_quotable and not comparison_allowed: + raise BenchDataError( + '{}: board_quotable=true is impossible with comparison_allowed=false ' + '(a diagnostic-only pairing is never board-quotable)'.format(where) + ) + if board_quotable and status not in BOARD_STATUSES: + raise BenchDataError( + '{}: board_quotable=true is impossible with status={!r}'.format(where, status) + ) + if board_quotable and status != 'ok' and not disclosures: + raise BenchDataError( + '{}: status={!r} must carry at least one disclosure'.format(where, status) + ) + return BenchCell( + key=key, + run_id=_req_str(obj, 'run', where), + workload=_req_str(obj, 'workload', where), + engine=_req_str(obj, 'engine', where), + value=_req_number(obj, 'value', where), + unit=unit, + decimals=_req_int(obj, 'decimals', where), + status=status, + comparison_allowed=comparison_allowed, + board_quotable=board_quotable, + disclosures=disclosures, + competitor=_opt_str(obj, 'competitor', where), + ) + + +class BenchPolicy: + """Build-breaking thresholds, versioned with the data itself.""" + + def __init__( + self, + max_age_days: int, + max_compute_commit_drift: int, + managed_docs: List[str], + literal_allowlist: Dict[str, List[str]], + ) -> None: + self.max_age_days = max_age_days + self.max_compute_commit_drift = max_compute_commit_drift + self.managed_docs = managed_docs + #: doc path -> literals that are structural, not measured claims. + self.literal_allowlist = literal_allowlist + + def allowed_literals(self, doc: str) -> List[str]: + return self.literal_allowlist.get(doc, []) + + @staticmethod + def from_json(raw: JsonValue) -> 'BenchPolicy': + where = 'policy' + obj = _obj(raw, where) + allowlist: Dict[str, List[str]] = {} + raw_allowlist = obj.get('literal_allowlist') + if raw_allowlist is not None: + allow_obj = _obj(raw_allowlist, where + '.literal_allowlist') + for doc in sorted(allow_obj): + allowlist[doc] = _str_list( + allow_obj[doc], '{}.literal_allowlist.{}'.format(where, doc)) + return BenchPolicy( + max_age_days=_req_int(obj, 'max_age_days', where), + max_compute_commit_drift=_req_int(obj, 'max_compute_commit_drift', where), + managed_docs=_req_str_list(obj, 'managed_docs', where), + literal_allowlist=allowlist, + ) + + +class BenchData: + def __init__( + self, + source_path: str, + generated_by: str, + policy: BenchPolicy, + runs: Dict[str, BenchRun], + cells: Dict[str, BenchCell], + ) -> None: + self.source_path = source_path + self.generated_by = generated_by + self.policy = policy + self.runs = runs + self.cells = cells + + def cell(self, key: str) -> BenchCell: + try: + return self.cells[key] + except KeyError: + raise BenchDataError( + 'unknown benchmark key {!r}; it is not in {} ' + '(a number the source-of-truth does not contain must not be published)'.format( + key, os.path.basename(self.source_path) + ) + ) + + def run_for(self, cell: BenchCell) -> BenchRun: + return self.runs[cell.run_id] + + def stale_runs(self, today: datetime.date) -> List[Tuple[BenchRun, int]]: + """Runs older than the policy threshold, newest-first.""" + out: List[Tuple[BenchRun, int]] = [] + for run in self.runs.values(): + age = run.age_days(today) + if age > self.policy.max_age_days: + out.append((run, age)) + out.sort(key=lambda pair: pair[1]) + return out + + def referenced_runs(self, keys: Sequence[str]) -> List[BenchRun]: + seen: List[str] = [] + for key in keys: + run_id = self.cell(key).run_id + if run_id not in seen: + seen.append(run_id) + return [self.runs[run_id] for run_id in seen] + + +def default_data_path() -> str: + """``docs/source/_data/gfql_benchmarks.json``, resolved from this file.""" + here = os.path.dirname(os.path.abspath(__file__)) + return os.path.join(os.path.dirname(here), '_data', 'gfql_benchmarks.json') + + +def load_bench_data(path: Optional[str] = None) -> BenchData: + data_path = path if path is not None else default_data_path() + if not os.path.isfile(data_path): + raise BenchDataError('benchmark source-of-truth not found: {}'.format(data_path)) + with open(data_path, 'r', encoding='utf-8') as handle: + raw_value: JsonValue = json.load(handle) + raw = _obj(raw_value, 'root') + + schema_version = _req_int(raw, 'schema_version', 'root') + if schema_version != SCHEMA_VERSION: + raise BenchDataError( + 'schema_version {} is not supported by this checkout (expected {})'.format( + schema_version, SCHEMA_VERSION + ) + ) + + policy = BenchPolicy.from_json(raw.get('policy')) + + raw_runs = _obj(raw.get('runs'), 'runs') + runs: Dict[str, BenchRun] = {} + for run_id in sorted(raw_runs): + runs[run_id] = BenchRun.from_json(run_id, raw_runs[run_id]) + + raw_cells = _obj(raw.get('cells'), 'cells') + cells: Dict[str, BenchCell] = {} + for key in sorted(raw_cells): + cell = BenchCell.from_json(key, raw_cells[key]) + if cell.run_id not in runs: + raise BenchDataError( + 'cells.{}: references run {!r}, which has no provenance record'.format(key, cell.run_id) + ) + cells[key] = cell + + return BenchData( + source_path=data_path, + generated_by=_req_str(raw, 'generated_by', 'root'), + policy=policy, + runs=runs, + cells=cells, + ) diff --git a/docs/source/cheatsheet.md b/docs/source/cheatsheet.md index f909303ae9..5544b06b03 100644 --- a/docs/source/cheatsheet.md +++ b/docs/source/cheatsheet.md @@ -179,7 +179,7 @@ It is easy to turn arbitrary data into insightful graphs. PyGraphistry comes wit g2.plot() ``` - Enable GFQL's optional automatic GPU acceleration for 43X+ speedups: + Enable GFQL's optional automatic GPU acceleration on larger graphs: ```python # Switch from Pandas CPU dataframes to RAPIDS GPU dataframes diff --git a/docs/source/conf.py b/docs/source/conf.py index 11c48eb758..ae6e530e34 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,6 +17,8 @@ sys.path.insert(0, os.path.abspath("../..")) +# Local Sphinx extensions (gfql_bench: benchmark numbers from the source-of-truth) +sys.path.insert(0, os.path.abspath("_ext")) import graphistry @@ -54,6 +56,9 @@ "sphinx.ext.ifconfig", #"sphinx_autodoc_typehints", "sphinx_copybutton", + # Resolves :bench:`key` against docs/source/_data/gfql_benchmarks.json and fails + # the build on a missing, stale, non-quotable or undisclosed benchmark number. + "gfql_bench", ] diff --git a/docs/source/gfql/_static/filter_pagerank/gplus_lifecycle.svg b/docs/source/gfql/_static/filter_pagerank/gplus_lifecycle.svg deleted file mode 100644 index bff1957c3e..0000000000 --- a/docs/source/gfql/_static/filter_pagerank/gplus_lifecycle.svg +++ /dev/null @@ -1,2103 +0,0 @@ - - - - - - - - 2026-03-16T21:03:05.702756 - image/svg+xml - - - Matplotlib v3.10.8, https://matplotlib.org/ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/source/gfql/_static/filter_pagerank/twitter_lifecycle.svg b/docs/source/gfql/_static/filter_pagerank/twitter_lifecycle.svg deleted file mode 100644 index cd0a12668a..0000000000 --- a/docs/source/gfql/_static/filter_pagerank/twitter_lifecycle.svg +++ /dev/null @@ -1,1991 +0,0 @@ - - - - - - - - 2026-03-16T21:03:05.678530 - image/svg+xml - - - Matplotlib v3.10.8, https://matplotlib.org/ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/source/gfql/benchmark_filter_pagerank.rst b/docs/source/gfql/benchmark_filter_pagerank.rst index d35a5c5ef7..369a27358d 100644 --- a/docs/source/gfql/benchmark_filter_pagerank.rst +++ b/docs/source/gfql/benchmark_filter_pagerank.rst @@ -10,30 +10,19 @@ Run Cypher graph queries and analytics directly on Python dataframes — no database required. This benchmark compares **Graphistry's local Cypher** (CPU and GPU) against **Neo4j + GDS** on the same end-to-end pipeline. -.. list-table:: - :header-rows: 1 - :widths: 30 20 20 20 20 - - * - - - Neo4j + GDS - - GFQL Cypher (CPU) - - GFQL Cypher (GPU) - - GPU speedup vs Neo4j - * - **Twitter** (2.4M edges) - - 13.83s - - 2.55s - - **0.30s** - - **46x** - * - **GPlus** (30M edges) - - >187s - - 75.78s - - **3.33s** - - **>56x** - -*Pipeline time (search + PageRank + search), warm median of 5 runs, 2 warmup iterations. DGX -dgx-spark, GB10 GPU. The per-graph sections below report full-lifecycle totals that also include -one-time ETL/load — hence the slightly larger numbers there (e.g. GPlus GPU 3.33s pipeline vs -~7.1s lifecycle).* +.. warning:: + **The figures previously published on this page have been withdrawn.** The raw + measurement artifacts for that run no longer exist anywhere — nothing was committed, + and the chart generator (``benchmarks/gfql/filter_pagerank/presentation.py``) reads a + results directory that is not present in this repository, so even the rendered charts + cannot be regenerated. That makes those numbers impossible to confirm *or* refute, so + they are treated as unpublishable rather than assumed correct. + + What remains below is the part that is still verifiable: what the benchmark measures, + the exact pipeline, the Neo4j+GDS analog, and how to run it yourself. The figures will + be republished once this pipeline runs under the provenance-carrying harness described + on :doc:`performance` — committed per-slot artifacts, recorded commit/host/perf-lock/ + reps, and results validated against the competitor before any ratio is published. The pipeline ------------ @@ -77,34 +66,25 @@ The same pipeline shape, different backends: The Neo4j equivalent requires ~30 lines of Cypher + GDS projection + batched writes (see :ref:`neo4j-analog` below). -Twitter (2.4M edges): exact 3-way comparison --------------------------------------------- - -.. image:: _static/filter_pagerank/twitter_lifecycle.svg - :alt: Twitter end-to-end: Neo4j vs GFQL Cypher CPU vs GFQL Cypher GPU - -Stacked by workload phase: **ETL** (load + shape), **Search** (graph queries), **Analytics** (PageRank). - -- Neo4j total lifecycle: ~21.6s (6.0s import + 1.7s prep + 13.8s pipeline) -- GFQL Cypher CPU: ~2.8s — **8x faster than Neo4j** -- GFQL Cypher GPU: ~0.4s — **54x faster than Neo4j** +What is measured +---------------- -GPlus (30M edges): larger graph -------------------------------- +Two comparisons, on two SNAP graphs (Twitter, 2.4M edges; GPlus, 30M edges): -.. image:: _static/filter_pagerank/gplus_lifecycle.svg - :alt: GPlus: Neo4j (lower bound) vs GFQL Cypher CPU vs GFQL Cypher GPU +- **Pipeline time** — search + PageRank + search, with the graph already loaded. +- **Full lifecycle** — the same pipeline plus the one-time cost of getting the data into + each system: import and preparation for Neo4j, load and shaping for GFQL. This is the + number that reflects an analyst starting from files. -- Neo4j: **>187s** (lower bound — the transaction did not finish) -- GFQL Cypher CPU: ~85.5s — still faster than Neo4j's incomplete run -- GFQL Cypher GPU: ~7.1s — **>26x faster than Neo4j** +Both are broken out by workload phase — **ETL** (load + shape), **Search** (graph +queries), **Analytics** (PageRank) — so the cost is attributable rather than a single +opaque total. Why this matters ---------------- -The CPU path already beats Neo4j without a GPU. You get Cypher-style graph -search + PageRank directly on your dataframe, no database to stand up or -maintain. +You get Cypher-style graph search + PageRank directly on your dataframe, with no database +to stand up or maintain, and the whole pipeline stays in one process. The GPU path accelerates everything — ETL, search, and analytics — because ``cudf`` and ``cugraph`` are drop-in replacements for ``pandas`` and ``igraph`` @@ -176,7 +156,7 @@ pandas / cuDF). That is what makes the CPU-to-GPU switch a configuration flag (``engine="cudf"``) rather than a rewrite, and what keeps ETL, search, and analytics in the same in-process pipeline. -**Same answer on every engine.** The CPU and GPU timings above are comparable +**Same answer on every engine.** CPU and GPU timings of this pipeline are comparable because the query meaning is fixed: GFQL's engine contract is same result or pre-execution decline. Unsupported engine/query combinations are rejected during validation, compilation, or planning before query execution, rather than silently @@ -186,8 +166,8 @@ parity and static-safety contract. This page is one workload (a filter → PageRank → filter pipeline) against one external baseline (Neo4j+GDS). For the full four-engine picture — when Polars beats pandas on CPU, when the GPU pulls ahead, and how to choose — see -:doc:`engines`. For sub-millisecond *seeded* lookups that beat Kuzu and Neo4j -by 9–28×, see :doc:`index_adjacency`. +:doc:`engines`. For *seeded* lookups, where an index rather than an engine is the +lever, see :doc:`index_adjacency`. For more on the GFQL design and supported surface: @@ -197,17 +177,28 @@ For more on the GFQL design and supported surface: - :doc:`overview` — GFQL design, features, and GPU acceleration - :doc:`about` — 10-minute introduction to GFQL -Benchmark environment ---------------------- +Methodology +----------- - Host: ``dgx-spark``, GPU: ``GB10``, driver ``580.126.09`` - Container: ``graphistry/test-gpu:latest`` - Datasets: `SNAP `_ Twitter (2.4M edges) and GPlus (30M edges) -- Measurement: median of 5 runs after 2 warmup iterations -- Results rendered from saved JSON — this page does **not** rerun benchmarks +- Measurement: warm median of 5 timed runs after 2 warmup iterations +- Neo4j runs the analog Cypher + GDS pipeline below on the same host + +Reproduce +--------- + +Note the caveat at the top of this page: these reproducers print and plot, but do not +yet emit a provenance-carrying artifact, which is why their output is not published here. + +- ``benchmarks/gfql/filter_pagerank/load_prepare_cpu_gpu.py`` — load + shape the graphs +- ``benchmarks/gfql/filter_pagerank/filter_pagerank_pipeline_cpu_gpu.py`` — the GFQL CPU/GPU pipeline +- ``benchmarks/gfql/filter_pagerank/filter_pagerank_pipeline_neo4j.py`` — the Neo4j + GDS analog +- ``benchmarks/gfql/filter_pagerank/presentation.py`` — chart generation Notebook version ---------------- See ``demos/gfql/benchmark_filter_pagerank_cpu_gpu.ipynb`` for a notebook -version of this writeup using the same saved DGX results. +version of this writeup. diff --git a/docs/source/gfql/builtin_calls.rst b/docs/source/gfql/builtin_calls.rst index 8a47bb2ff8..f31bf4fb12 100644 --- a/docs/source/gfql/builtin_calls.rst +++ b/docs/source/gfql/builtin_calls.rst @@ -532,7 +532,7 @@ Current supported names include: filtering by component membership, not as stable semantic IDs. .. note:: - For graphs with millions of edges, consider using ``compute_cugraph`` with a GPU for 10-50x speedup, or :ref:`gfql-remote` if no local GPU is available. + For graphs with millions of edges, consider using ``compute_cugraph`` with a GPU, or :ref:`gfql-remote` if no local GPU is available. get_degrees ~~~~~~~~~~~ diff --git a/docs/source/gfql/engines.rst b/docs/source/gfql/engines.rst index ae2510aa66..2274f3b2df 100644 --- a/docs/source/gfql/engines.rst +++ b/docs/source/gfql/engines.rst @@ -33,16 +33,14 @@ engine is a one-keyword change — no GPU, same results: g.gfql(query) # engine='pandas' (default) g.gfql(query, engine='polars') # often much faster on query-heavy workloads, identical results -Switching is often an order-of-magnitude speedup on query-heavy workloads — the current -measured numbers (release-verified engine sweeps, cross-database pairs, OLAP) live in -:ref:`gfql-0580-numbers` on the :doc:`performance` page. +Switching is often a large speedup on query-heavy workloads. Your existing pandas, Polars, or cuDF graph works as-is: the input frames are accepted and coerced once; the only change is the keyword. The catch: a few exotic Cypher features still require ``engine='pandas'`` (they decline during validation, compilation, or planning rather than silently bridge), and the GPU engines only pay off on larger work. On CPU, Polars wins the common graph-query shapes (traversal, -``WHERE``/``ORDER``, aggregation) from ~10K edges up — see *When not to use Polars* below. +``WHERE``/``ORDER``, aggregation) — see *When not to use Polars* below. .. warning:: **Already a Polars user? Pass** ``engine='polars'`` **— the default does not.** With the @@ -103,28 +101,23 @@ never selects Polars or Polars-GPU** — they are explicit opt-in (see *Why opt- How the engines compare ----------------------- -The measured tables live on the :doc:`performance` page — the release-verified engine -sweeps plus the large-graph bulk comparison — and in :doc:`index_adjacency` for the -resident-index benchmarks. The stable pattern they show: +The stable pattern: - **Polars-CPU is the everyday win**: typically much faster than pandas on query-heavy - workloads (traversal, ``WHERE``/``ORDER``, aggregation) from roughly 10K edges up — - often by an order of magnitude on bulk traversal — with **no GPU**. It also frequently - beats eager cuDF on bulk work, because it builds **one fused lazy plan and collects - once** while cuDF pays a kernel launch and a materialized intermediate per op. + workloads (traversal, ``WHERE``/``ORDER``, aggregation), with **no GPU**. It also + frequently beats eager cuDF on bulk work, because it builds **one fused lazy plan and + collects once** while cuDF pays a kernel launch and a materialized intermediate per op. - **Polars-GPU leads heavy multi-hop and full-graph aggregation**: the same fused plan, executed on the GPU, once there is enough work to amortize kernel launches. -- **cuDF wins the extreme single materialization**: one very large join/output where raw +- **cuDF suits the extreme single materialization**: one very large join/output where raw GPU throughput dominates and the in-memory Polars-GPU executor comes under memory pressure. -- **pandas keeps trivial sub-millisecond operations**: a bare equality filter's boolean - mask beats Polars' plan overhead — immaterial at <1 ms. -- **Seeded / selective lookups are an indexing problem**, not an engine race: with the - opt-in resident index they run in low single-digit milliseconds on every engine and stay - flat in graph size — see [F5] below and :doc:`index_adjacency`. - -Current measured numbers: :ref:`gfql-0580-numbers` (release-verified sweep) and the bulk -engine comparison on the same page. +- **pandas keeps trivially small operations**: a bare equality filter's boolean + mask beats Polars' plan overhead — immaterial in absolute terms. +- **Seeded / selective lookups are an indexing problem**, not an engine race: the opt-in + resident index turns the ``O(E)`` scan into an ``O(degree)`` gather on every engine, so + the cost tracks the seeds rather than the graph — see [F5] below and + :doc:`index_adjacency`. .. _gfql-vs-external-tools: @@ -136,8 +129,8 @@ cuDF frame in-process — no separate database to stand up, no ETL to load, no c databases (Neo4j, Kuzu) are a **system-of-record** you provision and ingest into first. The table below is deliberately conservative: wins are stated with their conditions, losses are reported as-is, and where we have no head-to-head we say **not benchmarked** rather than -guess. The measured pairs behind every claim live in :doc:`performance` (and -:doc:`index_adjacency` for the index benchmarks). +guess. Comparisons whose raw artifacts could not be recovered have been withdrawn rather +than restated. .. list-table:: :header-rows: 1 @@ -149,44 +142,32 @@ guess. The measured pairs behind every claim live in :doc:`performance` (and - Where it complements / GFQL doesn't claim * - **Neo4j + GDS** - Server + GDS library; stand up a DB and ETL your data in. - - **LDBC SNB interactive pairs** (same box, warm): GFQL wins most clean pairs — - seeded lookups, message content/creator projections, one-hop expands. **End-to-end - filter→PageRank→filter pipelines**: GFQL wins by large margins on CPU and - especially GPU, including a case where Neo4j did not finish. Measured pairs in - :ref:`gfql-0580-numbers`; pipeline benchmark in :doc:`benchmark_filter_pagerank`. - - **Neo4j wins some interactive shapes** (e.g. the recent-replies pair) — reported - as-is in :doc:`performance`. Neo4j remains the transactional system-of-record; run - the read-heavy analytics in GFQL. + - **End-to-end filter→PageRank→filter pipelines** stay in one in-process dataframe + call — no projection step, no write-back. The pipeline and its reproducer are in + :doc:`benchmark_filter_pagerank`; the head-to-head figures previously published + there are withdrawn pending a provenance-carrying re-run. + - Neo4j remains the transactional system-of-record; run the read-heavy analytics in + GFQL. No currently-publishable head-to-head latency comparison. * - **Kuzu** - Embedded graph DB; still a separate store to load + index. - - GFQL typically wins **multi-join OLAP** (by one to two orders of magnitude with - ``engine='polars'``), **seeded / selective index lookups**, and **bulk frontier - expansion**. Measured pairs in :ref:`gfql-0580-numbers`; index benchmarks in - :doc:`index_adjacency`. - - **Embedded Kuzu typically wins single-table aggregates and seeded - property-projection lookups** (shapes GFQL's covered fast paths don't yet reach) — - exact ratios in :doc:`performance`. GFQL's strengths are traversals, multi-join - OLAP, and covered seeded shapes. Also **not claimed:** cyclic / multi-way-join - patterns (triangles, cliques) where Kuzu's worst-case-optimal joins can win. Use - Kuzu as the store; GFQL for bulk read analytics. + - GFQL's strengths are **traversals**, **multi-join OLAP**, and **covered seeded + shapes**, with no separate store to provision, load and index. No + currently-publishable head-to-head latency comparison. + - **Not claimed:** cyclic / multi-way-join patterns (triangles, cliques) where Kuzu's + worst-case-optimal joins can win. Use Kuzu as the store; GFQL for bulk read + analytics. * - **LadybugDB** - Actively-maintained **Kuzu fork** (Kuzu is archived); embedded C++, strongly-typed - Cypher, opt-in ART *or* hash indexing, zero-copy Arrow/CSR scans, and **out-of-core - billion-scale** (query a 1.8B-edge graph in <8 GB RAM). - - Against **LadybugDB's published numbers** for their own 5M-node / 20M-edge suite - (their figures, their hardware; GFQL measured separately on the same Cypher row - pipeline, each engine on its **native** frames — a cross-machine comparison, so - read it as indicative): GFQL **wins the scan-shaped ops** — full node scans and - relationship property/rowid scans — by large margins. **Point lookups** are close - in absolute terms but favor Ladybug's B-tree/hash **index seek** over a columnar - scan; a resident GFQL node-id index (tracked in issue #1676) should close that. - Ladybug wins the two ops backed by persistent structure: point lookups and a - cached relationship ``COUNT(*)`` (an O(1) count vs GFQL's O(E) endpoint-validated - scan — a dataframe has no referential integrity). GFQL's angle is - dataframe-native, in-process, and GPU-accelerated with no separate store to - load/index. Details and reproducer in the :doc:`performance` methodology notes. - - **Complement:** Ladybug is a durable embedded store with an out-of-core mode - (billion-scale in <8 GB RAM); GFQL is a query engine over your dataframes. GFQL's + Cypher, opt-in ART *or* hash indexing, zero-copy Arrow/CSR scans, and an + **out-of-core** mode. + - **No publishable head-to-head.** The comparison that used to sit here has been + withdrawn: its competitor column was an uncited constant hardcoded in the benchmark + script rather than a measurement, and the GFQL side has no surviving artifact. + Structurally, GFQL's angle is dataframe-native, in-process, and GPU-accelerated + with no separate store to load or index; a resident GFQL node-id index (tracked in + issue #1676) targets the point-lookup shape. + - **Complement:** Ladybug is a durable embedded store with an out-of-core mode; + GFQL is a query engine over your dataframes. GFQL's *default* is in-memory, but it is **not limited to it** — Polars streaming (``GFQL_POLARS_CPU_STREAMING=1``, disk-spill) and the cudf-polars streaming executor (``GFQL_POLARS_GPU_EXECUTOR=streaming``) are larger-than-memory paths @@ -205,7 +186,7 @@ guess. The measured pairs behind every claim live in :doc:`performance` (and - Fine for small/interactive graphs; GFQL is the columnar/GPU path when they grow. * - **Spark GraphFrames** - *Distributed* graph engine on a Spark cluster; provision + tune the cluster. - - GFQL is *single-node* (CPU or one GPU): 100M+ edges in-process on **one machine**, + - GFQL is *single-node* (CPU or one GPU): a large graph in-process on **one machine**, no cluster to stand up, interactive latency — and a single node often matches or beats Spark on read-heavy filter/traversal even on CPU, while the GPU engine wins PageRank by an order of magnitude; on CPU, PageRank via igraph is *slower* than GraphFrames. @@ -223,12 +204,10 @@ GFQL **complements** a graph database more than it replaces one: keep Neo4j or K system-of-record, and do the read-heavy search + analytics in GFQL so ETL, traversal, and scoring stay in one in-process dataframe pipeline. Route by shape — **selective** seeded lookups favor the GFQL index, **multi-join OLAP** favors Polars, and **bulk** frontier -expansion and full pipelines favor Polars / GPU (measured pairs in :doc:`performance` and -:doc:`index_adjacency`). The inverse holds too: embedded engines win single-table -aggregates and seeded property-projection lookups. +expansion and full pipelines favor Polars / GPU. Against the **distributed** engines the axis is different: GFQL trades horizontal scale-out for zero cluster/warehouse setup and interactive latency — -choose it below the single-machine ceiling (100M+ edges fit in-process; a cluster is only +choose it below the single-machine ceiling (a cluster is only needed once the graph genuinely exceeds one node's memory), and complement PuppyGraph's zero-ETL warehouse graph with GFQL's GPU analytics. The one case we explicitly **do not** claim is cyclic / multi-way-join patterns (triangles, cliques): they **run**, but Kuzu's @@ -247,45 +226,45 @@ Decision matrix - Recommended engine - Notes * - Filter / ``WHERE`` / aggregation - - > ~10K + - past small/interactive - CPU - ``polars`` - - wins from ~10K; gap grows with size (up to order-of-magnitude) [F1] + - takes over past small graphs; gap grows with size [F1] * - Bulk 1-hop frontier expansion - - > ~10K + - past small/interactive - CPU - ``polars`` - - wins from ~10K; gap grows to order-of-magnitude at scale [F1] + - takes over past small graphs; gap grows with size [F1] * - Heavy multi-hop (2-hop+) - large - GPU - ``polars-gpu`` - fastest until extreme materialization [F3]; GPU-or-error [F4] * - Full-graph aggregation - - 100M+ + - very large - GPU - ``polars-gpu`` / ``cudf`` - GPU work-bound [F2] * - One very large single materialization - - 80M+ output rows + - huge output row count - GPU - ``cudf`` - Polars-GPU can hit memory pressure here [F3] - * - Trivial sub-ms op (bare equality filter) + * - Trivially small op (bare equality filter) - any - CPU - ``pandas`` - - boolean mask beats Polars plan overhead; immaterial (<1 ms) [F1] + - boolean mask beats Polars plan overhead; immaterial [F1] * - Selective / seeded traversal - any - CPU - ``pandas``/``polars`` + **CSR index** - O(degree), not an engine choice [F5] -**[F1] CPU crossover is ~10K, not ~1M.** For the common graph-query shapes (traversal, -``WHERE``/``ORDER``, aggregation) CPU Polars beats pandas from ~10K edges up (measured -crossover in :doc:`performance`). -Pandas only edges out on a trivial sub-millisecond operation (a bare equality mask), +**[F1] The CPU crossover is early, not exotic.** For the common graph-query shapes +(traversal, ``WHERE``/``ORDER``, aggregation) CPU Polars takes over from pandas once +graphs get past small/interactive sizes. +Pandas only edges out on a trivially small operation (a bare equality mask), where the absolute difference is immaterial. The real small-size floor is **GPU-only** — cuDF / Polars-GPU need enough work to amortize kernel launch ([F2]). @@ -293,10 +272,9 @@ cuDF / Polars-GPU need enough work to amortize kernel launch ([F2]). its millisecond-scale kernel-launch floor: big frontiers, dense joins, full-graph aggregation. Tiny or seeded work finishes faster on CPU. -**[F3] Polars-GPU memory pressure.** On an extreme single materialization (tens of millions -of output rows from one join) raw ``cudf`` leads and ``polars-gpu`` slips as its in-memory -GPU executor comes under memory pressure (measured in :doc:`performance`). Prefer ``cudf`` -for that regime. +**[F3] Polars-GPU memory pressure.** On an extreme single materialization (a huge output +row count from one join) raw ``cudf`` leads and ``polars-gpu`` slips as its in-memory +GPU executor comes under memory pressure. Prefer ``cudf`` for that regime. **[F4] Polars-GPU is GPU-or-error.** It never silently falls back to CPU and reports the result as a GPU run (see *Honesty* below). @@ -304,11 +282,11 @@ result as a GPU run (see *Honesty* below). **[F5] Selective traversal is an indexing problem, not an engine choice.** A seeded ``hop`` from a few nodes is fastest with the opt-in **CSR adjacency index** (``g.gfql_index_all()`` / ``g.create_index(...)``, ``index_policy=``), which turns the O(E) scan into an O(degree) -gather — flat in graph size (measured flat-scaling numbers in :ref:`gfql-0580-numbers`; -index routing for the native seeded ``g.hop()`` currently engages on pandas, not yet the -Polars hop path). It works on all four engines, but seeded work is so small that **CPU -wins** — sub-millisecond on pandas/Polars vs the GPU kernel-launch floor on cuDF — the -clean inverse of bulk, where the GPU pulls ahead. So pick the index for selective +gather — a complexity-class change, so the cost tracks the seeds' neighborhood rather than +the graph (index routing for the native seeded ``g.hop()`` currently engages on pandas, not +yet the Polars hop path). It works on all four engines, but seeded work is so small that +**CPU wins** — the gather is tiny work on pandas/Polars, below the GPU kernel-launch floor +on cuDF — the clean inverse of bulk, where the GPU pulls ahead. So pick the index for selective traversal and a CPU engine to drive it. See :doc:`index_adjacency` for the full guide. Switching engines @@ -513,25 +491,21 @@ When **not** to use Polars Honesty matters more than a bigger number: -- **Trivial sub-millisecond operations** (a bare node-equality filter): pandas' boolean mask - beats Polars' plan overhead — but at <1 ms it is immaterial. For traversal / ``WHERE`` / - ``ORDER`` / aggregation, CPU Polars wins from ~10K edges up (footnote F1). The real small-size - caveat is **GPU-only** (cuDF / Polars-GPU need larger work — footnote F2). +- **Trivially small operations** (a bare node-equality filter): pandas' boolean mask + beats Polars' plan overhead — but in absolute terms it is immaterial. For traversal / + ``WHERE`` / ``ORDER`` / aggregation, CPU Polars takes over past small/interactive sizes + (footnote F1). The real small-size caveat is **GPU-only** (cuDF / Polars-GPU need larger + work — footnote F2). - **A few exotic Cypher features** are not yet native on Polars (e.g. cross-entity same-path ``WHERE``, some temporal/entity-text forms). GFQL rejects those shapes during validation, compilation, or planning before query execution and points at ``engine='pandas'`` — it **never** silently bridges Polars to pandas, because that would misreport pandas performance as Polars (see *Honesty*). -- **One extreme materialization (80M+ output rows):** prefer ``cudf`` over ``polars-gpu`` - (footnote F3). -- **vs graph databases:** GFQL-Polars typically beats embedded kuzu on multi-join OLAP and - bulk frontier expansion by one to two orders of magnitude (measured pairs in - :doc:`performance`; reproducer ``benchmarks/gfql/index_vs_kuzu_prepared.py``); it - separately beats Neo4j+GDS end-to-end (:doc:`benchmark_filter_pagerank`). The honest - boundary: embedded kuzu **wins single-table aggregates and seeded property-projection - lookups** (exact ratios in :doc:`performance`), and its worst-case-optimal joins - target **cyclic / multi-way join** patterns (triangles, cliques) that we have **not** - yet benchmarked, and kuzu may lead there. +- **One extreme materialization (a huge output row count):** prefer ``cudf`` over + ``polars-gpu`` (footnote F3). +- **vs graph databases:** kuzu's worst-case-optimal joins target **cyclic / multi-way + join** patterns (triangles, cliques) that we have **not** yet benchmarked, and kuzu may + lead there. Parity and honesty ------------------ @@ -558,10 +532,9 @@ Parity and honesty Methodology ----------- -The measured tables and their full methodology — hosts, datasets, warm-median protocol, -cross-engine parity guards, and reproducer scripts — live with the numbers on the -:doc:`performance` page (release-verified tag sweeps plus the prior large-graph bulk -sweep) and in :doc:`index_adjacency` for the resident-index benchmarks. +Figures whose originating run could not be reproduced from a committed artifact have been +removed from these docs rather than restated. See :doc:`performance` for how a published +benchmark number is now sourced and gated. Install ------- @@ -592,9 +565,9 @@ picks them). The main reason is robustness, not speed: a few exotic Cypher featu require ``engine='pandas'`` and are **rejected before execution** rather than silently bridge, so auto-selecting Polars would turn queries that work today on pandas into hard errors. (Performance is rarely the -downside — CPU Polars wins common graph queries from ~10K edges; only trivial sub-millisecond -operations favor pandas, immaterially.) Opting in keeps the default behavior unchanged and -guarantees a working result. +downside — CPU Polars wins common graph queries past small/interactive sizes; only +trivially small operations favor pandas, immaterially.) Opting in keeps the default +behavior unchanged and guarantees a working result. See also -------- diff --git a/docs/source/gfql/index.rst b/docs/source/gfql/index.rst index 2fc8d99574..dbd93935e8 100644 --- a/docs/source/gfql/index.rst +++ b/docs/source/gfql/index.rst @@ -38,7 +38,7 @@ Recommended paths: - Running Cypher syntax in GFQL: :doc:`cypher` -> :doc:`quick` -> :doc:`return` -> :doc:`spec/cypher_mapping` - Faster on CPU (no GPU): :doc:`engines` -> :doc:`performance` (one keyword, ``engine='polars'``, often an order of magnitude faster on query-heavy workloads) - Performance path (intro -> engine choice -> GPU -> remote GPU): :doc:`about` -> :doc:`engines` -> :doc:`performance` -> :doc:`remote` -- Fast seeded lookups (start from known nodes, like a DB index): :doc:`indexing` (build once with ``gfql_index_all()``, reused automatically) -> :doc:`index_adjacency` (O(degree), flat in graph size, 9-28x vs Kuzu/Neo4j) +- Fast seeded lookups (start from known nodes, like a DB index): :doc:`indexing` (build once with ``gfql_index_all()``, reused automatically) -> :doc:`index_adjacency` (O(degree) instead of O(E), so cost tracks the seeds, not the graph) - Translating existing Cypher to native GFQL: :doc:`spec/cypher_mapping` - Building agents/integrations: :doc:`spec/language` + :doc:`spec/python_embedding` + :doc:`spec/wire_protocol` diff --git a/docs/source/gfql/index_adjacency.rst b/docs/source/gfql/index_adjacency.rst index fb7cffe43c..7c9bc6db48 100644 --- a/docs/source/gfql/index_adjacency.rst +++ b/docs/source/gfql/index_adjacency.rst @@ -5,8 +5,8 @@ A **seeded** graph query starts from a known set of nodes — "the neighbors of 50 accounts", "2 hops out from this device" — rather than scanning the whole graph. By default GFQL answers a seeded ``hop`` with an ``O(E)`` pass over every edge. With an opt-in **CSR adjacency index**, the same hop becomes an ``O(degree)`` gather: its cost -depends on how many edges the *seeds* touch, not on how big the graph is. The result is -**flat in graph size** — and it beats embedded graph databases on selective lookups. +depends on how many edges the *seeds* touch, not on how big the graph is — so a seeded +lookup stays interactive as the graph grows. Nothing changes about the answer. The index is a pay-as-you-go accelerator: a query either uses a resident index or falls back to the scan, and any feature the index does not cover @@ -19,7 +19,8 @@ When to use it ring's known members) and hop out 1–3 steps. - **Repeated queries** against the same graph: build the index once, amortize it over many seeded lookups. -- **Interactive / point-lookup latency**: sub-millisecond neighbor expansion. +- **Interactive / point-lookup latency**: neighbor expansion whose cost tracks the + seeds rather than the graph. It does **not** help a full-graph scan (a property filter over every node, a global PageRank). For those, choose an *engine* instead — see :doc:`engines`. @@ -89,67 +90,35 @@ remote ``gfql_remote`` call can carry the same index intent. Performance ----------- -**Flat in graph size.** A seeded 1-hop stays sub-millisecond as the graph grows 10×, while -the ``O(E)`` scan grows linearly. Synthetic power-law graphs, GFQL-pandas, warm median, -every cell guarded so the index path was taken *and* the indexed result equals the scan -result: - -.. list-table:: - :header-rows: 1 - :widths: 40 30 30 - - * - Seeded 1-hop - - 0.8M nodes / 6.4M edges - - 8M nodes / 64M edges - * - **Indexed (O(degree))** - - **0.124 ms** - - **0.122 ms** *(flat)* - * - Scan (O(E)) - - 105 ms - - 1045 ms - -The same holds on real power-law graphs: a typical-seed 1-hop is ~0.13 ms on LiveJournal -(35M edges) and ~0.14 ms on Orkut (117M edges), versus an ``O(E)`` scan of 367 ms → 1208 ms. - -**Beats embedded graph databases on selective lookups.** Same graph (0.8M nodes / 6.4M -edges), matched result counts, warm median. GFQL is CPU-pandas with the index; Kuzu and -Neo4j use their native indexes: - -.. list-table:: - :header-rows: 1 - :widths: 24 22 18 18 18 - - * - Task - - GFQL (indexed) - - Kuzu - - Neo4j - - GFQL speedup - * - 1-hop seeded - - **0.123 ms** - - 1.15 ms - - 1.45 ms - - 9.4× / 11.8× - * - 1–2-hop seeded - - **0.150 ms** - - 4.25 ms - - 2.54 ms - - 28× / 16.9× - -On a fairer, fully-prepared, in-process Kuzu re-run (LiveJournal 35M), GFQL is still -**17×** on a typical seed (0.126 ms vs 2.13 ms) and **6×** on a hub seed (3.76 ms vs -22.6 ms). *(Kuzu's worst-case-optimal joins can win on cyclic / multi-way-join patterns — -triangles, cliques — which these forward-expansion lookups do not exercise; we do not -claim those.)* - -**Selective traversal is CPU's game.** The indexed hop is tiny work, so the GPU's -kernel-launch floor (~3 ms on cuDF) loses to a ~0.13 ms pandas / ~0.16 ms Polars -``searchsorted`` — the clean inverse of *bulk* analytics, where the GPU pulls ahead -(see :doc:`engines`). Pick the index for selective traversal and a **CPU engine** to -drive it. - -Reproduce: ``benchmarks/gfql/index_takeover_bench.py``, +**What the index changes is the complexity class**, and that part is structural +rather than measured: an indexed seeded hop is an ``O(degree)`` gather into a +sorted adjacency, so its cost tracks the seeds' neighborhood size, while the +default scan is ``O(E)`` and grows with the whole graph. The bigger the graph +relative to the seeds' neighborhood, the larger that difference gets. + +**Selective traversal is CPU's game.** The indexed hop is tiny work, so a GPU +engine's kernel-launch floor dominates it and a CPU engine (pandas or Polars, +both backed by a ``searchsorted`` gather) wins — the clean inverse of *bulk* +analytics, where the GPU pulls ahead (see :doc:`engines`). Pick the index for +selective traversal and a **CPU engine** to drive it. + +.. warning:: + **The measured figures that used to appear here have been withdrawn.** They + were transcribed by hand from runs whose raw artifacts no longer exist + anywhere — the reproducers wrote to ``/tmp`` (or only printed), recorded no + commit, host or timestamp, and were never committed. That makes those numbers + impossible to confirm *or* refute, so they are treated as unpublishable rather + than assumed correct. The comparisons against Kuzu and Neo4j are withdrawn on + the same grounds. + + They will return once the seeded-index lane runs under the provenance-carrying + harness described on :doc:`performance` — committed per-slot artifacts, recorded + commit/host/perf-lock/reps, and result rows validated against the competitor before + any ratio is published. + +Reproduce (note the caveat above — these reproducers do not yet emit a +provenance-carrying artifact): ``benchmarks/gfql/index_takeover_bench.py``, ``benchmarks/gfql/index_vs_dbs.py``, ``benchmarks/gfql/index_vs_kuzu_prepared.py``. -Hardware: DGX ``dgx-spark``, GB10 GPU. Honesty and cost ---------------- diff --git a/docs/source/gfql/indexing.rst b/docs/source/gfql/indexing.rst index d63dc535db..d3c85c64b1 100644 --- a/docs/source/gfql/indexing.rst +++ b/docs/source/gfql/indexing.rst @@ -200,38 +200,28 @@ Engines What it costs, what it buys --------------------------- -All numbers below: 0.58.0 tag sweep on DGX Spark, warm medians over 30 runs. +**Build (the "pay" side)**: one-time and ``O(E log E)`` — a sort over the edge frame, +amortized across every subsequent seeded query. ``index_policy='auto'`` only pays it when +the planner predicts a selective query will earn it back. -**Build (the "pay" side)**: one-time; on a 30.6M-edge graph the full -``gfql_index_all()`` build is about 5.7s. +**Seeded lookup (the "go" side)**: on a covered shape, the seeded lookup drops from the +general path to the fast path, and again with the index resident, on both CPU engines. -**Seeded lookup (the "go" side)**: a covered-shape seeded Cypher lookup: +**Flat in graph size**: a direct seeded ``g.hop()`` with the index resident turns the +``O(E)`` scan into an ``O(degree)`` gather, so its cost tracks the seeds' neighborhood +rather than the graph. -.. list-table:: - :header-rows: 1 - :widths: 40 30 30 - - * - Seeded typed lookup - - pandas - - Polars - * - General path (no fast path) - - 29.9 ms - - 13.8 ms - * - Fast path, no index - - 2.46 ms - - 2.28 ms - * - **Fast path + resident index** - - **1.74 ms** - - **1.59 ms** - -**Flat in graph size**: a direct seeded ``g.hop()`` with the index resident holds -0.159–0.164 ms from 0.25M to 32M edges (pandas) — cost tracks seed degree, not graph -size. +.. warning:: + **The tag-sweep latencies that used to appear in this section have been withdrawn.** + Their raw artifacts were not committed anywhere, so they cannot be confirmed or + refuted, and re-measuring them is the only honest way to restore them. Measured + figures now live only on :doc:`performance` and :doc:`index_adjacency`, referenced + from a provenance-carrying source of truth rather than transcribed. See also -------- - :doc:`Seeded Traversal Indexes ` — the planner (``index_policy``), - Cypher DDL / wire protocol forms, and competitive benchmarks vs Kuzu / Neo4j. + Cypher DDL / wire protocol forms, and the index cost model. - :doc:`engines` — choosing pandas / Polars / cuDF / Polars-GPU. - :doc:`performance` — the vectorization + GPU design behind GFQL. diff --git a/docs/source/gfql/performance.rst b/docs/source/gfql/performance.rst index ee535f9599..01f6a61c7d 100644 --- a/docs/source/gfql/performance.rst +++ b/docs/source/gfql/performance.rst @@ -7,14 +7,23 @@ This page is the **canonical home for GFQL benchmark numbers** — the measured here (and, for the resident-index benchmarks, in :doc:`index_adjacency`), while the rest of the docs make stable qualitative claims and link back here. +.. note:: + **Every number published on this page is now referenced, not transcribed.** Figures come + from a single machine-readable source of truth + (``docs/source/_data/gfql_benchmarks.json``) that is generated from committed benchmark + artifacts, and the docs build fails rather than render a figure the source of truth does + not contain or whose run has gone stale. See `How a number gets published here`_. + + Figures whose originating run could not be reproduced from a committed artifact have + been **removed** from these docs rather than restated. Removing an unverifiable number + is the correct outcome, not a regression. + Engine speedups at a glance --------------------------- GFQL runs the **same query** on four interchangeable engines — ``pandas`` (default), ``polars`` (CPU, columnar), ``cudf`` (NVIDIA GPU), and ``polars-gpu`` (GPU) — and returns -**identical results** on each (differential parity is a release gate; every four-engine -number on this page was kept only after the result rows were verified identical across -engines, and the cross-database pairs were validated against expected result rows). +**identical results** on each (differential parity is a release gate). Unsupported engine/query combinations are declined before execution during validation, compilation, or planning rather than silently falling back. The biggest, easiest win is one keyword, **no GPU required**: @@ -26,226 +35,33 @@ keyword, **no GPU required**: g.gfql(query) # engine='pandas' (default) g.gfql(query, engine='polars') # often much faster on query-heavy workloads, same results -For example, in the release-verified sweep below, the LDBC SNB SF1 seed-lookup drops from -**1,299.6 ms** on eager pandas to **106.1 ms** with ``engine='polars'`` — a **12.3×** -one-keyword speedup, no GPU, results identical. - -.. _gfql-0580-numbers: - -Release-verified numbers (0.58.0) ---------------------------------- - -All numbers in this section were measured on the **0.58.0 release tag** on an NVIDIA DGX -Spark (GB10), warm medians over N=30 runs. The four-engine numbers (seeded fast paths, -resident index, scaling) were kept only after the result rows were asserted identical -across engines; the competitor pairs (vs Neo4j, vs Kuzu) were validated against expected -result rows and cross-database value/row-count checks. - -Seeded typed-hop fast path -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -A seeded typed hop — Cypher ``MATCH (m {id: ...})-[:T]->(p) RETURN p`` on a 50k-node / -200k-edge graph — is the bread-and-butter selective lookup. The release's fast path speeds -it up on every engine (before → after within the sweep): - -.. list-table:: - :header-rows: 1 - :widths: 25 20 20 15 - - * - Engine - - Before - - After (fast path) - - Speedup - * - ``pandas`` - - 29.9 ms - - **2.46 ms** - - 12.1× - * - ``polars`` - - 13.8 ms - - **2.28 ms** - - 6.1× - * - ``cudf`` - - 30.1 ms - - **4.89 ms** - - 6.1× - * - ``polars-gpu`` - - 25.2 ms - - **2.49 ms** - - 10.1× - -The native chain form of the same query is faster still: pandas 21.1 → **1.65 ms** -(12.8×), cuDF 23.2 → **3.84 ms** (6.0×). - -With a resident index -~~~~~~~~~~~~~~~~~~~~~ - -Building the opt-in resident index once (``g.gfql_index_all()``) makes the covered-shape -seeded lookup faster again — pandas **1.74 ms**, polars **1.59 ms**, polars-gpu -**1.91 ms**, cudf **5.78 ms**. - -.. warning:: - **Polars + index: pass** ``engine='polars'`` **when building.** Polars frames currently - need ``g.gfql_index_all(engine='polars')`` explicitly — an AUTO build swaps Polars - frames to pandas. The fix is tracked in PR #1767. - -Scaling: flat in graph size -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -With the resident adjacency index, a native seeded 1-hop ``g.hop()`` on pandas stays -**flat at 0.159–0.164 ms from 0.25M to 32M edges** (constant average degree 4): the index -turns the O(E) scan into an O(degree) gather, so seeded latency does not grow with the -graph. (Pandas-only today — the Polars hop path is not yet index-routed.) - -vs Neo4j (LDBC SNB interactive SF1) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Same box, warm, against Neo4j 5.26 — GFQL wins **4 of 5** clean pairs: - -.. list-table:: - :header-rows: 1 - :widths: 30 20 20 15 - - * - Query - - GFQL - - Neo4j 5.26 - - Winner - * - seed-lookup - - **106.1 ms** - - 143.7 ms - - GFQL - * - message-content - - **7.1 ms** - - 23.0 ms - - GFQL - * - message-creator - - **6.8 ms** - - 27.7 ms - - GFQL - * - one-hop-expand - - **111.9 ms** - - 180.7 ms - - GFQL - * - recent-replies - - 209.6 ms - - **104.0 ms** - - Neo4j - -The message-creator flip shipped in this release via property-seeded resident-index -gathers (PR #1770). Neo4j still wins recent-replies — reported as-is. - -OLAP multi-join -~~~~~~~~~~~~~~~ - -On the graph-benchmark OLAP multi-join queries at 100k-node scale with -``engine='polars'``: **q8 runs in 5.0 ms vs 1,004 ms for embedded Kuzu (200×)**; q9 is -**14.2×**. - -When not to use GFQL -~~~~~~~~~~~~~~~~~~~~ - -Honesty matters more than a bigger number. In the same cross-DB sweep, **embedded Kuzu -wins single-table aggregates (2–4×) and seeded property-projection lookups (2.4–64×)**. -GFQL's strengths are **traversals, multi-join OLAP, and covered seeded shapes** — route by -workload, and keep a database as the system-of-record where one fits. - -.. _gfql-bulk-sweep: - -Bulk engine comparison (prior sweep) ------------------------------------- - -The numbers in this section are from an earlier, pre-0.58.0 bulk sweep on SNAP -**com-LiveJournal** (35M edges) and **com-Orkut** (117M edges) — retained as the -bulk-workload reference until rerun on a current tag. - -Same query, same answers, four engines — warm-median latency on Orkut (3.1M nodes / -117M edges), measured on a single machine: - -.. list-table:: - :header-rows: 1 - :widths: 34 16 16 16 16 - - * - Workload (Orkut, 117M edges) - - ``pandas`` - - ``polars`` - - ``cudf`` - - ``polars-gpu`` - * - 1-hop from 10K seeds - - 2613 ms - - **68 ms** - - 1005 ms - - 63 ms - * - 2-hop from 10K seeds - - 18161 ms - - 2695 ms - - 2774 ms - - **1518 ms** - * - Full out-degree aggregation - - 799 ms - - 205 ms - - 314 ms - - **167 ms** - * - 2-hop from 100K seeds (~85M output rows) - - 28822 ms - - 8215 ms - - **6002 ms** - - 8559 ms - -Reading the table: - -- **Polars-CPU beat pandas up to ~38x** on bulk traversal and ~4x on aggregation — **with - no GPU**. On the 1-hop workload it was ~38x faster than pandas (68 ms vs 2613 ms). -- **Polars-CPU also beat cuDF** on these shapes (68 ms vs 1005 ms on 1-hop). cuDF runs - GFQL *eagerly*, op by op (a kernel launch + a materialized intermediate per hop), while - Polars builds **one fused lazy plan and collects once**. The fused plan wins until the - work is large enough to amortize GPU launch costs. -- **Polars-GPU was fastest on heavy multi-hop** (2-hop from 10K seeds: 1518 ms) and on - aggregation — the same fused plan, executed on the GPU. -- **cuDF won the one extreme case** — a 2-hop from 100K seeds materializing ~85M output - rows (6.0 s) — where raw GPU throughput on a single massive join overtakes everything - and Polars-GPU comes under memory pressure. -- On LiveJournal (35M edges) the pattern held: 1-hop from 10K seeds was pandas 1129 ms → - polars **37 ms** (~30x). -- The CPU crossover is early: on LiveJournal subsampled (CPU, warm-median), 1-hop - traversal was 2.7× / 4.5× / 7.6× and ``WHERE``+``ORDER`` 3.0× / 3.0× / 18× over pandas - at 10K / 100K / 1M edges. The only case pandas edged out was a trivial sub-millisecond - operation (a bare node-equality filter), where its boolean mask beats Polars' plan - overhead — immaterial at <1 ms. Reproducer: ``benchmarks/gfql/index_crossover_bench.py``. - -Methodology (prior sweep) -~~~~~~~~~~~~~~~~~~~~~~~~~ - -- Host: NVIDIA DGX Spark (GB10 Grace-Blackwell, unified memory — the memory-pressure - boundary above is partly a property of this box), RAPIDS container - ``graphistry/test-rapids-official:26.02-gfql-polars``. -- Datasets: `SNAP `_ **com-LiveJournal** (35M edges), - **com-Orkut** (117M edges). -- Measurement: **warm median** after 2 warmups (5 timed runs on Orkut, 8 on LiveJournal); - every reported cell is **guarded** — the result rows are verified identical across - engines before any timing is kept. -- Reproduce: ``benchmarks/gfql/index_bulk_olap_bench.py`` (engine comparison), - ``benchmarks/gfql/pandas_vs_polars.py``, and ``benchmarks/gfql/index_vs_kuzu_prepared.py`` - (vs kuzu). Numbers on this page are rendered from saved runs; the page does not re-run - them. -- **LadybugDB comparison** (referenced qualitatively in :doc:`engines`): the Ladybug - figures are **their published results on their hardware**; the GFQL side ran on the host - above via ``benchmarks/gfql/bench_ladybug_cypher.py`` (5M/20M synthetic per their suite - shape, native frames per engine, warm medians) — a cross-machine comparison, disclosed - as such. GFQL won the scan-shaped ops by large margins (full node scan ~65×, - relationship property/rowid scans ~3.5–3.7×); Ladybug won the two ops backed by - persistent structure — point lookups (index seek vs columnar scan) and a cached - relationship ``COUNT(*)``. - -There is **no universal winner**: ``polars`` typically takes over from ~10K edges up -(``pandas`` still wins trivial sub-millisecond operations), and the right GPU +There is **no universal winner**: ``polars`` typically takes over past small/interactive +sizes (``pandas`` still wins trivially small operations), and the right GPU engine depends on the workload. See :doc:`engines` for the full decision matrix, the honest -"when *not* to use Polars", and the cuDF-vs-Polars-GPU comparison. The end-to-end -CPU/GPU-vs-Neo4j pipeline benchmark is in :doc:`benchmark_filter_pagerank`, and the +"when *not* to use Polars", and the cuDF-vs-Polars-GPU comparison. The Spark GraphFrames head-to-head is in :doc:`benchmark_graphframes`. +How a number gets published here +-------------------------------- + +1. The benchmark harness lives in `graphistry/pyg-bench + `_, which commits its raw per-slot artifacts + alongside the results. +2. ``scripts/export_docs_numbers.py`` in that repository turns those committed artifacts + into ``docs/source/_data/gfql_benchmarks.json`` here — deriving each median, each ratio + and each cell's publishability from the artifacts rather than from anyone's notes. **No + GPU box and no benchmark re-run is needed** to regenerate the docs from stored results. +3. The docs build resolves every ``:bench:`` reference against that file, and fails if a + key is missing, if a run has aged past the policy threshold, or if a page drops a + number's provenance or disclosures. +4. ``bin/check_bench_numbers.py`` runs the same gate outside Sphinx and adds a + commit-drift check: when the query engine has moved materially since a published run was + measured, the number is treated as stale and the check fails. + How GFQL is fast ---------------- -Three design choices explain the numbers above: +Three design choices explain the speedups: **Collection-oriented execution.** GFQL evaluates whole collections of nodes and edges at once (set-at-a-time), rather than walking one path at a time like traditional Cypher/Gremlin @@ -254,7 +70,7 @@ engines. A traversal advances by joining edge tables, so the work vectorizes. **Vectorized columnar processing.** Data is processed in columnar batches on top of `Apache Arrow `_, which keeps the CPU path fast and makes moving data between systems cheap. The ``polars`` engine additionally builds **one fused lazy plan -and collects once**, which is why it outruns both pandas and eager cuDF on bulk work. +and collects once**, which is why it outruns eager engines on bulk work. **Massive parallelism on GPUs.** On an NVIDIA GPU (``cudf`` / ``polars-gpu``), the same vectorized work saturates tens of thousands of threads — paying off when there is enough @@ -271,7 +87,6 @@ Next Steps ---------- - **Choose an engine**: :doc:`engines` — the full decision matrix and qualitative guidance. -- **Selective lookups**: :doc:`index_adjacency` — the resident index behind the flat-scaling numbers. -- **End-to-end benchmark**: :doc:`benchmark_filter_pagerank` — CPU/GPU vs Neo4j+GDS. +- **Selective lookups**: :doc:`index_adjacency` — the CSR adjacency index. - **Explore GFQL**: :ref:`10min-gfql`. **Get started**: :ref:`10min-pygraphistry`. - **Ecosystem**: `Apache Arrow `_ and `NVIDIA RAPIDS `_. diff --git a/docs/source/performance.rst b/docs/source/performance.rst index 6815613573..ba29d41262 100644 --- a/docs/source/performance.rst +++ b/docs/source/performance.rst @@ -6,7 +6,7 @@ CPU & GPU Acceleration in PyGraphistry Why PyGraphistry is Fast ------------------------ -PyGraphistry is designed for speed. By focusing on **vectorized processing**, it outperforms most graph libraries on standard CPUs. When you leverage GPUs and AI models, PyGraphistry can become **100X+ faster**, enabling real-time analytics and machine learning at scale. We regularly use it on datasets with millions and billions of rows. +PyGraphistry is designed for speed. By focusing on **vectorized processing**, it outperforms most graph libraries on standard CPUs. When you leverage GPUs and AI models, PyGraphistry can become **dramatically faster**, enabling real-time analytics and machine learning at scale. We regularly use it on datasets with millions and billions of rows. Just as Apache Spark used in-memory processing to replace racks of Hadoop servers with faster and smaller multicore ones, the PyGraphistry ecosystem uses GPU acceleration to increase speeds and decrease costs even further. diff --git a/docs/test_bench_numbers.py b/docs/test_bench_numbers.py new file mode 100644 index 0000000000..0e5c7efe06 --- /dev/null +++ b/docs/test_bench_numbers.py @@ -0,0 +1,161 @@ +"""The benchmark-number gate must FAIL on the ways a published figure goes wrong. + +These are negative tests on purpose: a gate that has never been shown to reject +anything is indistinguishable from no gate. Each case here corresponds to a real +failure this repository has had — a number that outlived its measurement, a +diagnostic-only cell quoted as a competitor result, a caveat dropped in transit, +and a figure typed in by hand. +""" + +from __future__ import annotations + +import datetime +import json +import os +import sys +from typing import Dict, List + +import pytest + +_DOCS = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(_DOCS, 'source', '_ext')) +sys.path.insert(0, os.path.join(os.path.dirname(_DOCS), 'bin')) + +from gfql_bench_data import ( # noqa: E402 + BenchDataError, + load_bench_data, + default_data_path, +) + +import check_bench_numbers as gate # noqa: E402 + + +def _write(tmp_path: str, payload: Dict[str, object]) -> str: + path = os.path.join(tmp_path, 'bench.json') + with open(path, 'w', encoding='utf-8') as handle: + json.dump(payload, handle) + return path + + +def _base_payload() -> Dict[str, object]: + return { + 'schema_version': 1, + 'generated_by': 'test', + 'policy': { + 'max_age_days': 60, + 'max_compute_commit_drift': 12, + 'managed_docs': [], + }, + 'runs': { + 'r1': { + 'measured_at': '2026-07-26', + 'host': 'dgx-spark', + 'perf_lock_held': True, + 'quiet_host': True, + 'reps': '3 warmups + 7 timed', + 'pygraphistry_commit': '84be35fb', + 'pyg_bench_commit': '47f94ba', + 'runtime': 'rapids 26.02', + 'dataset': 'graph-benchmark 20k', + 'artifact': 'results/x', + 'row_validation': 'rows equal the competitor on every slot', + }, + }, + 'cells': { + 'a.b.polars': { + 'run': 'r1', 'workload': 'q1', 'engine': 'polars', 'value': 13.13, + 'unit': 'ms', 'decimals': 2, 'status': 'ok', + 'comparison_allowed': True, 'board_quotable': True, 'disclosures': [], + }, + }, + } + + +def test_the_shipped_source_of_truth_loads() -> None: + data = load_bench_data(default_data_path()) + assert data.runs, 'the shipped source-of-truth must contain at least one run' + for cell in data.cells.values(): + assert cell.run_id in data.runs + + +def test_unknown_key_is_refused(tmp_path: object) -> None: + data = load_bench_data(_write(str(tmp_path), _base_payload())) + with pytest.raises(BenchDataError) as excinfo: + data.cell('a.b.polars_new') + assert 'must not be published' in str(excinfo.value) + + +def test_a_diagnostic_cell_cannot_claim_to_be_board_quotable(tmp_path: object) -> None: + payload = _base_payload() + cells: Dict[str, Dict[str, object]] = payload['cells'] # type: ignore[assignment] + cells['a.b.polars']['comparison_allowed'] = False + with pytest.raises(BenchDataError) as excinfo: + load_bench_data(_write(str(tmp_path), payload)) + assert 'diagnostic-only pairing is never board-quotable' in str(excinfo.value) + + +def test_a_non_ok_status_must_carry_a_disclosure(tmp_path: object) -> None: + payload = _base_payload() + cells: Dict[str, Dict[str, object]] = payload['cells'] # type: ignore[assignment] + cells['a.b.polars']['status'] = 'partial' + with pytest.raises(BenchDataError) as excinfo: + load_bench_data(_write(str(tmp_path), payload)) + assert 'must carry at least one disclosure' in str(excinfo.value) + + +def test_a_run_without_provenance_is_not_loadable(tmp_path: object) -> None: + payload = _base_payload() + runs: Dict[str, Dict[str, object]] = payload['runs'] # type: ignore[assignment] + del runs['r1']['perf_lock_held'] + with pytest.raises(BenchDataError) as excinfo: + load_bench_data(_write(str(tmp_path), payload)) + assert 'perf_lock_held' in str(excinfo.value) + + +def test_freshness_gate_fires_once_a_run_ages_out(tmp_path: object) -> None: + data = load_bench_data(_write(str(tmp_path), _base_payload())) + fresh = gate.check_freshness(data, datetime.date(2026, 8, 1)) + assert fresh == [] + stale = gate.check_freshness(data, datetime.date(2026, 12, 1)) + assert len(stale) == 1 + assert 'Re-measure or drop the claim' in stale[0].render() + + +def test_hand_typed_literal_is_rejected_in_a_managed_doc(tmp_path: object) -> None: + source = os.path.join(str(tmp_path), 'source') + os.makedirs(source) + with open(os.path.join(source, 'managed.rst'), 'w', encoding='utf-8') as handle: + handle.write('Seeded lookups run in 0.124 ms and are 9.4x faster.\n') + payload = _base_payload() + policy: Dict[str, object] = payload['policy'] # type: ignore[assignment] + policy['managed_docs'] = ['managed.rst'] + data = load_bench_data(_write(str(tmp_path), payload)) + + previous = gate.DOCS_SOURCE + try: + gate.DOCS_SOURCE = source + findings: List[gate.Finding] = gate.check_hand_typed_literals(data) + finally: + gate.DOCS_SOURCE = previous + rendered = ' '.join(finding.render() for finding in findings) + assert '0.124 ms' in rendered + assert '9.4x' in rendered + + +def test_allowlisted_literal_is_permitted(tmp_path: object) -> None: + source = os.path.join(str(tmp_path), 'source') + os.makedirs(source) + with open(os.path.join(source, 'managed.rst'), 'w', encoding='utf-8') as handle: + handle.write('The graph grows 10x between the two columns.\n') + payload = _base_payload() + policy: Dict[str, object] = payload['policy'] # type: ignore[assignment] + policy['managed_docs'] = ['managed.rst'] + policy['literal_allowlist'] = {'managed.rst': ['10x']} + data = load_bench_data(_write(str(tmp_path), payload)) + + previous = gate.DOCS_SOURCE + try: + gate.DOCS_SOURCE = source + assert gate.check_hand_typed_literals(data) == [] + finally: + gate.DOCS_SOURCE = previous