chore: adopt upstream's history as our base

.github/compare_benchmarks.py

@@ -9,6 +9,7 @@
 import json
 import sys
 import argparse
+import statistics
 from typing import Dict, List, Any
 
 
@@ -47,18 +48,22 @@ def get_cold_metrics(iteration_results: List[Dict[str, Any]]) -> Dict[str, float
 
 
 def get_warm_metrics(iteration_results: List[Dict[str, Any]]) -> Dict[str, float]:
-    """Calculate average metrics from warm iterations (excluding first)."""
+    """Calculate median metrics from warm iterations (excluding first).
+
+    CI runners occasionally pause a process while it is being timed.  A median
+    keeps one such pause from turning into a reported regression.
+    """
     warm_results = iteration_results[1:] if len(iteration_results) > 1 else iteration_results
     if not warm_results:
         return {"time_millis": 0, "cache_cpu_time": 0}
 
-    avg_time = sum(r["time_millis"] for r in warm_results) / len(warm_results)
-    avg_cpu_time = sum(r.get("cache_cpu_time", 0) for r in warm_results) / len(warm_results)
+    median_time = statistics.median(r["time_millis"] for r in warm_results)
+    median_cpu_time = statistics.median(r.get("cache_cpu_time", 0) for r in warm_results)
     # No memory column in report
 
     return {
-        "time_millis": avg_time, 
-        "cache_cpu_time": avg_cpu_time,
+        "time_millis": median_time,
+        "cache_cpu_time": median_cpu_time,
     }
 
 
@@ -79,15 +84,17 @@ def format_metric_with_baseline(current: float, baseline: float, formatter_func)
     return f"{formatter_func(current)} *({formatter_func(baseline)})*"
 
 
-def format_change_percentage(current: float, baseline: float, highlight_mode: str = "none") -> str:
+def format_change_percentage(
+    current: float, baseline: float, threshold: float, highlight_mode: str = "none"
+) -> str:
     """Format percentage change and optionally highlight when slower.
 
     highlight_mode:
       - "none": never bold
-      - "slower_only": bold only if current > baseline (i.e., slower) and ≥15%
+      - "slower_only": bold only if current exceeds baseline by the threshold
     """
     change_pct = calculate_change(baseline, current)
-    if highlight_mode == "slower_only" and change_pct > 0 and abs(change_pct) >= 15.0:
+    if highlight_mode == "slower_only" and change_pct >= threshold:
         return f"**{change_pct:+.1f}%**"
     return f"{change_pct:+.1f}%"
 
@@ -195,21 +202,21 @@ def extract_mode(d: Dict[str, Any]) -> str:
             comp['curr_cold_time'], comp['baseline_cold_time'], format_time
         )
         cold_change_str = format_change_percentage(
-            comp['curr_cold_time'], comp['baseline_cold_time'], highlight_mode="none"
+            comp['curr_cold_time'], comp['baseline_cold_time'], threshold, highlight_mode="none"
         )
 
         warm_time_str = format_metric_with_baseline(
             comp['curr_warm_time'], comp['baseline_warm_time'], format_time
         )
         warm_change_str = format_change_percentage(
-            comp['curr_warm_time'], comp['baseline_warm_time'], highlight_mode="slower_only"
+            comp['curr_warm_time'], comp['baseline_warm_time'], threshold, highlight_mode="slower_only"
         )
 
         cpu_time_str = format_metric_with_baseline(
             comp['curr_cpu_time'], comp['baseline_cpu_time'], format_time
         )
         cpu_change_str = format_change_percentage(
-            comp['curr_cpu_time'], comp['baseline_cpu_time'], highlight_mode="none"
+            comp['curr_cpu_time'], comp['baseline_cpu_time'], threshold, highlight_mode="none"
         )
 
         lines.append(
@@ -220,7 +227,7 @@ def extract_mode(d: Dict[str, Any]) -> str:
         )
 
     # Summary focused on LiquidCache being slower than DataFusion (warm time)
-    slower_warm = [c for c in comparison if c["warm_time_change"] > 0]
+    slower_warm = [c for c in comparison if c["warm_time_change"] >= threshold]
     lines.append("")
     if slower_warm:
         lines.append(f"**⚠️ LiquidCache is slower on {len(slower_warm)} queries (warm)**")
@@ -230,18 +237,22 @@ def extract_mode(d: Dict[str, Any]) -> str:
             slower_warm, key=lambda x: x["warm_time_change"], reverse=True
         )
         for c in slower_warm_sorted:
-            curr = c["curr_warm_time"]; base = c["baseline_warm_time"]
+            curr = c["curr_warm_time"]
+            base = c["baseline_warm_time"]
             pct = calculate_change(base, curr)
             lines.append(
                 f"- Q{c['query']}: warm {pct:+.1f}% "
                 f"({format_time(curr)} vs {format_time(base)})"
             )
     else:
-        lines.append("✅ LiquidCache is faster or equal on warm time for all queries")
+        lines.append(f"✅ No warm-time regression met the {threshold:.0f}% threshold")
 
     lines.append("")
     lines.append(f"*Compared {current_mode} vs {baseline_mode} on the same runner*")
-    lines.append("*Cold Time: first iteration; Warm Time: average of remaining iterations.*")
+    lines.append(
+        f"*Regressions: warm-time increases of at least {threshold:.0f}%. "
+        "Cold Time: first iteration; Warm Time: median of remaining iterations.*"
+    )
 
     return "\n".join(lines)
 

.github/test_compare_benchmarks.py

@@ -0,0 +1,38 @@
+import importlib.util
+import pathlib
+import unittest
+
+
+SCRIPT = pathlib.Path(__file__).with_name("compare_benchmarks.py")
+SPEC = importlib.util.spec_from_file_location("compare_benchmarks", SCRIPT)
+compare = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(compare)
+
+
+class CompareBenchmarksTest(unittest.TestCase):
+    def test_warm_metrics_use_median(self):
+        iterations = [
+            {"time_millis": 100, "cache_cpu_time": 100},
+            {"time_millis": 10, "cache_cpu_time": 1},
+            {"time_millis": 11, "cache_cpu_time": 2},
+            {"time_millis": 500, "cache_cpu_time": 100},
+        ]
+
+        self.assertEqual(
+            compare.get_warm_metrics(iterations),
+            {"time_millis": 11, "cache_cpu_time": 2},
+        )
+
+    def test_highlight_respects_configured_threshold(self):
+        self.assertEqual(
+            compare.format_change_percentage(114, 100, 15, "slower_only"),
+            "+14.0%",
+        )
+        self.assertEqual(
+            compare.format_change_percentage(114, 100, 10, "slower_only"),
+            "**+14.0%**",
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()