diff --git a/agents/validation_loop.py b/agents/validation_loop.py
index fa2d4c8..f5bca0c 100644
--- a/agents/validation_loop.py
+++ b/agents/validation_loop.py
@@ -1147,37 +1147,49 @@ def _upper_bound_name(name: str, row: dict | None = None) -> bool:
     return bool(isinstance(row, dict) and row.get("upper_bound")) or "oracle" in label or "upper_bound" in label
 
 
+_WEIGHTED_METHOD_KEYS = (
+    "score",
+    "exact",
+    "f1",
+    "avg_new_tokens",
+    "avg_latency_seconds",
+    "route_rate",
+    "cost_adjusted_accuracy",
+    "metric_value",
+)
+_PER_MODEL_TABLE_KEYS = (
+    "ablation_table",
+    "ablation_results",
+    "cost_utility_tradeoff_table",
+    "quality_cost_frontier",
+    "route_rate_sweep",
+    "difficulty_breakdown_table",
+    "latency_tokens_table",
+    "calibration_reliability",
+)
+
+
+def _count_or_none(row: dict) -> int | None:
+    try:
+        return int(row.get("count") or 0)
+    except (TypeError, ValueError):
+        return None
+
+
 def _weighted_merge_method_rows(rows: list[dict]) -> dict:
+    counts = [_count_or_none(row) for row in rows]
+    total_count = max(1, sum(count for count in counts if count is not None))
     merged: dict = {}
-    total_count = 0
-    for row in rows:
-        try:
-            total_count += int(row.get("count") or 0)
-        except (TypeError, ValueError):
-            pass
-    total_count = max(1, total_count)
-    numeric_avg_keys = (
-        "score",
-        "exact",
-        "f1",
-        "avg_new_tokens",
-        "avg_latency_seconds",
-        "route_rate",
-        "cost_adjusted_accuracy",
-        "metric_value",
-    )
-    for key in numeric_avg_keys:
-        value = 0.0
-        seen = False
-        for row in rows:
+    for key in _WEIGHTED_METHOD_KEYS:
+        terms = []
+        for row, count in zip(rows, counts):
             try:
-                count = int(row.get("count") or 0)
-                value += float(row.get(key)) * max(1, count)
-                seen = True
+                if count is not None:
+                    terms.append(float(row.get(key)) * max(1, count))
             except (TypeError, ValueError):
-                continue
-        if seen:
-            merged[key] = float(value / total_count)
+                pass
+        if terms:
+            merged[key] = float(sum(terms) / total_count)
     merged["count"] = total_count
     if any(row.get("upper_bound") for row in rows if isinstance(row, dict)):
         merged["upper_bound"] = True
@@ -1216,66 +1228,56 @@ def _merge_model_benchmark_summaries(model_results: list[dict], criteria_metric_
     summaries = [row.get("benchmark_summary") for row in model_results if isinstance(row.get("benchmark_summary"), dict)]
     if not summaries:
         return {}
+    # Summaries are paired with model_results by position, so a model without a
+    # summary shifts the ids of the ones after it (pinned by the merge tests).
+    labelled = []
+    for model_result, summary in zip(model_results, summaries):
+        model = summary.get("model") if isinstance(summary.get("model"), dict) else {}
+        labelled.append((summary, model, model_result.get("model_id") or model.get("id") or ""))
     base = json.loads(json.dumps(summaries[0]))
     metric_name = str(base.get("primary_metric") or base.get("metric_name") or criteria_metric_name or "metric_value")
     candidate = str(base.get("candidate_method") or "").strip()
-    all_methods = sorted({name for summary in summaries for name in (summary.get("per_method") or {}).keys()})
     per_method: dict = {}
-    for method in all_methods:
-        rows = [summary.get("per_method", {}).get(method) for summary in summaries]
-        rows = [row for row in rows if isinstance(row, dict)]
+    for method in sorted({name for summary in summaries for name in (summary.get("per_method") or {})}):
+        rows = [row for summary in summaries if isinstance(row := summary.get("per_method", {}).get(method), dict)]
         if rows:
             per_method[method] = _weighted_merge_method_rows(rows)
-    seed_results = []
-    for model_result, summary in zip(model_results, summaries):
-        model = summary.get("model") if isinstance(summary.get("model"), dict) else {}
-        model_id = model_result.get("model_id") or model.get("id") or ""
-        for seed_row in summary.get("seed_results") or []:
-            if isinstance(seed_row, dict):
-                copied = json.loads(json.dumps(seed_row))
-                copied["model_id"] = model_id
-                seed_results.append(copied)
-    datasets = []
-    seen_datasets = set()
-    for summary in summaries:
-        for row in summary.get("datasets") or []:
-            if not isinstance(row, dict):
-                continue
-            key = str(row.get("name") or row.get("id") or row).lower()
-            if key and key not in seen_datasets:
-                seen_datasets.add(key)
-                datasets.append(row)
-    models = []
-    seen_models = set()
-    for model_result, summary in zip(model_results, summaries):
-        model = summary.get("model") if isinstance(summary.get("model"), dict) else {}
-        model_id = str(model_result.get("model_id") or model.get("id") or "").strip()
+    seed_results = [
+        {**json.loads(json.dumps(seed_row)), "model_id": model_id}
+        for summary, _, model_id in labelled
+        for seed_row in summary.get("seed_results") or []
+        if isinstance(seed_row, dict)
+    ]
+    datasets, seen_datasets = [], set()
+    for row in (row for summary in summaries for row in summary.get("datasets") or [] if isinstance(row, dict)):
+        key = str(row.get("name") or row.get("id") or row).lower()
+        if key and key not in seen_datasets:
+            seen_datasets.add(key)
+            datasets.append(row)
+    models, seen_models = [], set()
+    for _, model, model_id in labelled:
+        model_id = str(model_id).strip()
         if model_id and model_id.lower() not in seen_models:
             seen_models.add(model_id.lower())
             models.append({**model, "id": model_id})
-    strongest_name = ""
-    strongest_value = None
-    higher = True
+    strongest_name, strongest_value = "", None
     for name, row in per_method.items():
         if name == candidate or _upper_bound_name(name, row):
             continue
         value = _metric_from_row(row, metric_name)
-        if value is None:
-            continue
-        if strongest_value is None or (value > strongest_value if higher else value < strongest_value):
+        if value is not None and (strongest_value is None or value > strongest_value):
             strongest_name, strongest_value = name, value
     candidate_values: list[float] = []
     baseline_values: list[float] = []
-    if candidate and strongest_name:
-        for seed_row in seed_results:
-            methods = seed_row.get("methods") if isinstance(seed_row, dict) else {}
-            if not isinstance(methods, dict):
-                continue
-            cand = _metric_from_row(methods.get(candidate, {}), metric_name)
-            base_value = _metric_from_row(methods.get(strongest_name, {}), metric_name)
-            if cand is not None and base_value is not None:
-                candidate_values.append(cand)
-                baseline_values.append(base_value)
+    for seed_row in seed_results if candidate and strongest_name else []:
+        methods = seed_row.get("methods")
+        if not isinstance(methods, dict):
+            continue
+        cand = _metric_from_row(methods.get(candidate, {}), metric_name)
+        base_value = _metric_from_row(methods.get(strongest_name, {}), metric_name)
+        if cand is not None and base_value is not None:
+            candidate_values.append(cand)
+            baseline_values.append(base_value)
     p_value = _paired_permutation_pvalue(candidate_values, baseline_values)
     bootstrap = {
         "candidate_method": candidate,
@@ -1285,23 +1287,13 @@ def _merge_model_benchmark_summaries(model_results: list[dict], criteria_metric_
         "paired_permutation_p": p_value,
         "p_value": p_value,
     }
-    for key in (
-        "ablation_table",
-        "ablation_results",
-        "cost_utility_tradeoff_table",
-        "quality_cost_frontier",
-        "route_rate_sweep",
-        "difficulty_breakdown_table",
-        "latency_tokens_table",
-        "calibration_reliability",
-    ):
-        rows = []
-        for model_result, summary in zip(model_results, summaries):
-            model = summary.get("model") if isinstance(summary.get("model"), dict) else {}
-            model_id = model_result.get("model_id") or model.get("id") or ""
-            for row in summary.get(key) or []:
-                if isinstance(row, dict):
-                    rows.append({"model_id": model_id, **row})
+    for key in _PER_MODEL_TABLE_KEYS:
+        rows = [
+            {"model_id": model_id, **row}
+            for summary, _, model_id in labelled
+            for row in summary.get(key) or []
+            if isinstance(row, dict)
+        ]
         if rows:
             base[key] = rows
     base["per_method"] = per_method
@@ -1441,6 +1433,28 @@ def _remaining_grant_gpu_seconds(run_id: int) -> float | None:
     )
 
 
+def _note_launch_failure(log_path: Path, text: str) -> None:
+    try:
+        log_path.write_text(text, encoding="utf-8")
+    except OSError:
+        pass
+
+
+def _launch_failure(duration, error, failure_type, command_tokens, log_path, benchmark_env, **extra) -> dict:
+    """Result for a run that failed before producing any output."""
+    return {
+        "status": "crash",
+        "duration": duration,
+        "error": error,
+        "failure_type": failure_type,
+        "final_results_present": False,
+        "command_tokens": command_tokens,
+        "log_path": str(log_path),
+        **extra,
+        "benchmark_env": benchmark_env,
+    }
+
+
 def _run_experiment(
     workdir: Path,
     code_dir: Path,
@@ -1492,27 +1506,13 @@ def _run_experiment(
     )
     if contract_violations:
         duration = time.time() - start
-        error = "; ".join(contract_violations)
-        try:
-            log_path.write_text(
-                "Contract-preserving runner guard blocked execution.\n"
-                f"Command: {command_tokens}\n"
-                + "\n".join(f"- {item}" for item in contract_violations)
-                + "\n",
-                encoding="utf-8",
-            )
-        except OSError:
-            pass
-        return {
-            "status": "crash",
-            "duration": duration,
-            "error": error,
-            "failure_type": "contract_violation",
-            "final_results_present": False,
-            "command_tokens": command_tokens,
-            "log_path": str(log_path),
-            "benchmark_env": benchmark_env,
-        }
+        _note_launch_failure(
+            log_path,
+            "Contract-preserving runner guard blocked execution.\n"
+            f"Command: {command_tokens}\n" + "\n".join(f"- {item}" for item in contract_violations) + "\n",
+        )
+        return _launch_failure(duration, "; ".join(contract_violations), "contract_violation",
+                               command_tokens, log_path, benchmark_env)
     if full_benchmark and not _disable_model_matrix and len(_benchmark_model_list_from_env(benchmark_env)) > 1:
         return _run_experiment_model_matrix(
             workdir,
@@ -1530,17 +1530,9 @@ def _run_experiment(
         if run_id is not None and ssh_gpu_backend.is_ssh_worker(worker):
             remaining_gpu_seconds = _remaining_grant_gpu_seconds(run_id)
             if remaining_gpu_seconds is not None and remaining_gpu_seconds <= 0:
-                return {
-                    "status": "crash",
-                    "duration": time.time() - start,
-                    "error": "resource grant cumulative GPU-hour cap is exhausted",
-                    "failure_type": "grant_gpu_hours_exhausted",
-                    "final_results_present": False,
-                    "command_tokens": command_tokens,
-                    "log_path": str(log_path),
-                    "backend": "ssh",
-                    "benchmark_env": benchmark_env,
-                }
+                return _launch_failure(time.time() - start, "resource grant cumulative GPU-hour cap is exhausted",
+                                       "grant_gpu_hours_exhausted", command_tokens, log_path, benchmark_env,
+                                       backend="ssh")
             remote_time_budget = time_budget
             if remaining_gpu_seconds is not None:
                 bounded_seconds = max(1, int(remaining_gpu_seconds))
@@ -1664,42 +1656,14 @@ def _run_experiment(
 
     except subprocess.TimeoutExpired as exc:
         duration = time.time() - start
-        try:
-            log_path.write_text(
-                f"Experiment timed out after {duration:.1f}s\nCommand: {command_tokens}\n{exc}\n",
-                encoding="utf-8",
-            )
-        except OSError:
-            pass
-        return {
-            "status": "crash",
-            "duration": duration,
-            "error": "timeout",
-            "failure_type": "timeout",
-            "final_results_present": False,
-            "command_tokens": command_tokens,
-            "log_path": str(log_path),
-            "benchmark_env": benchmark_env,
-        }
+        _note_launch_failure(log_path, f"Experiment timed out after {duration:.1f}s\nCommand: {command_tokens}\n{exc}\n")
+        return _launch_failure(duration, "timeout", "timeout", command_tokens, log_path, benchmark_env)
     except Exception as e:
         duration = time.time() - start
-        try:
-            log_path.write_text(
-                f"Experiment launcher failed after {duration:.1f}s\nCommand: {command_tokens}\nError: {e}\n",
-                encoding="utf-8",
-            )
-        except OSError:
-            pass
-        return {
-            "status": "crash",
-            "duration": duration,
-            "error": str(e),
-            "failure_type": "launcher_error",
-            "final_results_present": False,
-            "command_tokens": command_tokens,
-            "log_path": str(log_path),
-            "benchmark_env": benchmark_env,
-        }
+        _note_launch_failure(
+            log_path, f"Experiment launcher failed after {duration:.1f}s\nCommand: {command_tokens}\nError: {e}\n"
+        )
+        return _launch_failure(duration, str(e), "launcher_error", command_tokens, log_path, benchmark_env)
 
     metric = None
     benchmark_summary = _parse_benchmark_summary_from_log(log_path)
diff --git a/tests/fixtures/validation_loop_merge_cases.json b/tests/fixtures/validation_loop_merge_cases.json
new file mode 100644
index 0000000..729f834
--- /dev/null
+++ b/tests/fixtures/validation_loop_merge_cases.json
@@ -0,0 +1,930 @@
+{
+ "merge": [
+  {
+   "name": "two models, mixed quality fields",
+   "criteria_metric_name": "accuracy",
+   "model_results": [
+    {
+     "model_id": "m1",
+     "benchmark_summary": {
+      "primary_metric": "accuracy",
+      "candidate_method": " cand ",
+      "per_method": {
+       "cand": {
+        "count": 10,
+        "score": 0.6,
+        "metric_value": 0.6,
+        "avg_new_tokens": 100,
+        "accuracy": 0.6
+       },
+       "base1": {
+        "count": 10,
+        "score": 0.5,
+        "metric_value": 0.5
+       },
+       "oracle_router": {
+        "count": 10,
+        "score": 0.9
+       },
+       "base2": {
+        "count": 10,
+        "score": 0.52,
+        "upper_bound": false
+       }
+      },
+      "seed_results": [
+       {
+        "seed": 1,
+        "methods": {
+         "cand": {
+          "metric_value": 0.61
+         },
+         "base2": {
+          "metric_value": 0.5
+         }
+        }
+       },
+       {
+        "seed": 2,
+        "methods": {
+         "cand": {
+          "metric_value": 0.59
+         },
+         "base2": {
+          "score": 0.54
+         }
+        }
+       },
+       "not-a-row"
+      ],
+      "datasets": [
+       {
+        "name": "GSM8K",
+        "split": "test"
+       },
+       "plain"
+      ],
+      "model": {
+       "id": "m1-hf",
+       "family": "qwen"
+      },
+      "ablation_table": [
+       {
+        "variant": "no_router",
+        "score": 0.55
+       },
+       "skip"
+      ],
+      "num_seeds": 2,
+      "load_failures": [],
+      "full_benchmark_completed": true,
+      "duration_seconds": 10.5,
+      "peak_vram_mb": 1000,
+      "hardware": "A100"
+     }
+    },
+    {
+     "model_id": "",
+     "benchmark_summary": {
+      "per_method": {
+       "cand": {
+        "count": "bad",
+        "score": "x",
+        "metric_value": 0.7
+       },
+       "base1": {
+        "count": 5,
+        "score": 0.4,
+        "metric_value": 0.45
+       },
+       "base2": {
+        "count": 5,
+        "score": 0.55,
+        "upper_bound": true
+       },
+       "Upper-Bound probe": {
+        "count": 5,
+        "score": 0.99
+       }
+      },
+      "seed_results": [
+       {
+        "seed": 1,
+        "methods": {
+         "cand": {
+          "metric_value": 0.7
+         },
+         "base2": {
+          "metric_value": 0.4
+         }
+        }
+       }
+      ],
+      "datasets": [
+       {
+        "name": "gsm8k"
+       },
+       {
+        "id": "MATH"
+       }
+      ],
+      "model": {
+       "id": "M1-HF"
+      },
+      "latency_tokens_table": [
+       {
+        "method": "cand",
+        "latency": 1.5
+       }
+      ],
+      "num_seeds": "3",
+      "load_failures": [
+       "oom on shard 2"
+      ],
+      "full_benchmark_completed": true,
+      "duration_seconds": null,
+      "peak_vram_mb": "2048",
+      "hardware": "H100"
+     }
+    },
+    {
+     "model_id": "m3",
+     "benchmark_summary": "not a dict"
+    }
+   ],
+   "expected": {
+    "primary_metric": "accuracy",
+    "candidate_method": " cand ",
+    "per_method": {
+     "Upper-Bound probe": {
+      "score": 0.99,
+      "count": 5
+     },
+     "base1": {
+      "score": 0.4666666666666667,
+      "metric_value": 0.48333333333333334,
+      "count": 15
+     },
+     "base2": {
+      "score": 0.53,
+      "count": 15,
+      "upper_bound": true
+     },
+     "cand": {
+      "score": 0.6,
+      "avg_new_tokens": 100.0,
+      "metric_value": 0.6,
+      "count": 10
+     },
+     "oracle_router": {
+      "score": 0.9,
+      "count": 10
+     }
+    },
+    "seed_results": [
+     {
+      "seed": 1,
+      "methods": {
+       "cand": {
+        "metric_value": 0.61
+       },
+       "base2": {
+        "metric_value": 0.5
+       }
+      },
+      "model_id": "m1"
+     },
+     {
+      "seed": 2,
+      "methods": {
+       "cand": {
+        "metric_value": 0.59
+       },
+       "base2": {
+        "score": 0.54
+       }
+      },
+      "model_id": "m1"
+     },
+     {
+      "seed": 1,
+      "methods": {
+       "cand": {
+        "metric_value": 0.7
+       },
+       "base2": {
+        "metric_value": 0.4
+       }
+      },
+      "model_id": "M1-HF"
+     }
+    ],
+    "datasets": [
+     {
+      "name": "GSM8K",
+      "split": "test"
+     },
+     {
+      "id": "MATH"
+     }
+    ],
+    "model": {
+     "ids": [
+      "m1",
+      "M1-HF"
+     ],
+     "backend": "transformers",
+     "cuda": true
+    },
+    "ablation_table": [
+     {
+      "model_id": "m1",
+      "variant": "no_router",
+      "score": 0.55
+     }
+    ],
+    "num_seeds": 3,
+    "load_failures": [
+     "oom on shard 2"
+    ],
+    "full_benchmark_completed": false,
+    "duration_seconds": 10.5,
+    "peak_vram_mb": 2048.0,
+    "hardware": "A100, H100",
+    "latency_tokens_table": [
+     {
+      "model_id": "M1-HF",
+      "method": "cand",
+      "latency": 1.5
+     }
+    ],
+    "dataset": {
+     "name": "GSM8K",
+     "split": "test"
+    },
+    "models": [
+     {
+      "id": "m1",
+      "family": "qwen"
+     },
+     {
+      "id": "M1-HF"
+     }
+    ],
+    "bootstrap_ci": {
+     "candidate_method": "cand",
+     "baseline_method": "base1",
+     "candidate_ci95": [
+      0.0,
+      0.0
+     ],
+     "baseline_ci95": [
+      0.0,
+      0.0
+     ],
+     "paired_permutation_p": null,
+     "p_value": null
+    },
+    "accuracy": 0.6
+   }
+  },
+  {
+   "name": "paired seeds give a permutation p-value",
+   "criteria_metric_name": "score",
+   "model_results": [
+    {
+     "model_id": "p1",
+     "benchmark_summary": {
+      "primary_metric": "score",
+      "candidate_method": "cand",
+      "per_method": {
+       "cand": {
+        "count": 4,
+        "score": 0.7
+       },
+       "base": {
+        "count": 4,
+        "score": 0.6
+       },
+       "weak": {
+        "count": 4,
+        "score": 0.1
+       },
+       "unscored": {
+        "count": 2
+       }
+      },
+      "seed_results": [
+       {
+        "methods": {
+         "cand": {
+          "score": 0.6
+         },
+         "base": {
+          "score": 0.5
+         }
+        }
+       },
+       {
+        "methods": {
+         "cand": {
+          "score": 0.65
+         },
+         "base": {
+          "score": 0.55
+         }
+        }
+       },
+       {
+        "methods": {
+         "cand": {
+          "score": 0.7
+         },
+         "base": {
+          "score": 0.6
+         }
+        }
+       },
+       {
+        "methods": {
+         "cand": {
+          "score": 0.72
+         },
+         "base": {
+          "score": 0.62
+         }
+        }
+       },
+       {
+        "methods": {
+         "cand": {
+          "score": 0.6799999999999999
+         },
+         "base": {
+          "score": 0.58
+         }
+        }
+       },
+       {
+        "methods": {
+         "cand": {
+          "score": 0.2
+         }
+        }
+       },
+       {
+        "methods": "bad"
+       }
+      ],
+      "num_seeds": 7,
+      "full_benchmark_completed": true,
+      "duration_seconds": "3",
+      "peak_vram_mb": null
+     }
+    },
+    {
+     "model_id": "p2",
+     "benchmark_summary": {
+      "per_method": {
+       "cand": {
+        "count": 4,
+        "score": 0.9
+       },
+       "base": {
+        "count": 12,
+        "score": 0.65
+       }
+      },
+      "seed_results": [
+       {
+        "methods": {
+         "cand": {
+          "score": 0.91
+         },
+         "base": {
+          "score": 0.6
+         }
+        }
+       }
+      ],
+      "num_seeds": 1,
+      "full_benchmark_completed": true,
+      "hardware": "A100"
+     }
+    }
+   ],
+   "expected": {
+    "primary_metric": "score",
+    "candidate_method": "cand",
+    "per_method": {
+     "base": {
+      "score": 0.6375000000000001,
+      "count": 16
+     },
+     "cand": {
+      "score": 0.8,
+      "count": 8
+     },
+     "unscored": {
+      "count": 2
+     },
+     "weak": {
+      "score": 0.1,
+      "count": 4
+     }
+    },
+    "seed_results": [
+     {
+      "methods": {
+       "cand": {
+        "score": 0.6
+       },
+       "base": {
+        "score": 0.5
+       }
+      },
+      "model_id": "p1"
+     },
+     {
+      "methods": {
+       "cand": {
+        "score": 0.65
+       },
+       "base": {
+        "score": 0.55
+       }
+      },
+      "model_id": "p1"
+     },
+     {
+      "methods": {
+       "cand": {
+        "score": 0.7
+       },
+       "base": {
+        "score": 0.6
+       }
+      },
+      "model_id": "p1"
+     },
+     {
+      "methods": {
+       "cand": {
+        "score": 0.72
+       },
+       "base": {
+        "score": 0.62
+       }
+      },
+      "model_id": "p1"
+     },
+     {
+      "methods": {
+       "cand": {
+        "score": 0.6799999999999999
+       },
+       "base": {
+        "score": 0.58
+       }
+      },
+      "model_id": "p1"
+     },
+     {
+      "methods": {
+       "cand": {
+        "score": 0.2
+       }
+      },
+      "model_id": "p1"
+     },
+     {
+      "methods": "bad",
+      "model_id": "p1"
+     },
+     {
+      "methods": {
+       "cand": {
+        "score": 0.91
+       },
+       "base": {
+        "score": 0.6
+       }
+      },
+      "model_id": "p2"
+     }
+    ],
+    "num_seeds": 7,
+    "full_benchmark_completed": true,
+    "duration_seconds": 3.0,
+    "peak_vram_mb": 0.0,
+    "datasets": [],
+    "dataset": {},
+    "models": [
+     {
+      "id": "p1"
+     },
+     {
+      "id": "p2"
+     }
+    ],
+    "model": {
+     "ids": [
+      "p1",
+      "p2"
+     ],
+     "backend": "transformers",
+     "cuda": true
+    },
+    "load_failures": [],
+    "hardware": "A100",
+    "bootstrap_ci": {
+     "candidate_method": "cand",
+     "baseline_method": "base",
+     "candidate_ci95": [
+      0.645,
+      0.7966666666666667
+     ],
+     "baseline_ci95": [
+      0.5416666666666666,
+      0.6033333333333334
+     ],
+     "paired_permutation_p": 0.03125,
+     "p_value": 0.03125
+    },
+    "score": 0.8
+   }
+  },
+  {
+   "name": "no candidate method",
+   "criteria_metric_name": "accuracy",
+   "model_results": [
+    {
+     "model_id": "a",
+     "benchmark_summary": {
+      "metric_name": "f1",
+      "per_method": {
+       "x": {
+        "count": 2,
+        "f1": 0.3
+       },
+       "y": {
+        "count": 2,
+        "f1": 0.8
+       }
+      }
+     }
+    }
+   ],
+   "expected": {
+    "metric_name": "f1",
+    "per_method": {
+     "x": {
+      "f1": 0.3,
+      "count": 2
+     },
+     "y": {
+      "f1": 0.8,
+      "count": 2
+     }
+    },
+    "seed_results": [],
+    "num_seeds": 0,
+    "datasets": [],
+    "dataset": {},
+    "models": [
+     {
+      "id": "a"
+     }
+    ],
+    "model": {
+     "ids": [
+      "a"
+     ],
+     "backend": "transformers",
+     "cuda": true
+    },
+    "load_failures": [],
+    "full_benchmark_completed": false,
+    "duration_seconds": 0.0,
+    "peak_vram_mb": 0.0,
+    "hardware": "",
+    "bootstrap_ci": {
+     "candidate_method": "",
+     "baseline_method": "y",
+     "candidate_ci95": [
+      0.0,
+      0.0
+     ],
+     "baseline_ci95": [
+      0.0,
+      0.0
+     ],
+     "paired_permutation_p": null,
+     "p_value": null
+    }
+   }
+  },
+  {
+   "name": "no metric name anywhere",
+   "criteria_metric_name": "",
+   "model_results": [
+    {
+     "benchmark_summary": {
+      "candidate_method": "c",
+      "per_method": {
+       "c": {
+        "count": 1,
+        "exact": 1
+       },
+       "b": {
+        "count": 1,
+        "cost_adjusted_accuracy": 0.2
+       }
+      },
+      "seed_results": [
+       {
+        "methods": {
+         "c": {
+          "score": 0.9
+         },
+         "b": {
+          "cost_adjusted_accuracy": 0.2
+         }
+        }
+       }
+      ],
+      "full_benchmark_completed": false
+     }
+    }
+   ],
+   "expected": {
+    "candidate_method": "c",
+    "per_method": {
+     "b": {
+      "cost_adjusted_accuracy": 0.2,
+      "count": 1
+     },
+     "c": {
+      "exact": 1.0,
+      "count": 1
+     }
+    },
+    "seed_results": [
+     {
+      "methods": {
+       "c": {
+        "score": 0.9
+       },
+       "b": {
+        "cost_adjusted_accuracy": 0.2
+       }
+      },
+      "model_id": ""
+     }
+    ],
+    "full_benchmark_completed": false,
+    "num_seeds": 0,
+    "datasets": [],
+    "dataset": {},
+    "models": [],
+    "model": {
+     "ids": [],
+     "backend": "transformers",
+     "cuda": true
+    },
+    "load_failures": [],
+    "duration_seconds": 0.0,
+    "peak_vram_mb": 0.0,
+    "hardware": "",
+    "bootstrap_ci": {
+     "candidate_method": "c",
+     "baseline_method": "b",
+     "candidate_ci95": [
+      0.9,
+      0.9
+     ],
+     "baseline_ci95": [
+      0.2,
+      0.2
+     ],
+     "paired_permutation_p": 1.0,
+     "p_value": 1.0
+    },
+    "metric_value": 0.0
+   }
+  },
+  {
+   "name": "a model without a summary shifts the ids of the ones after it",
+   "criteria_metric_name": "score",
+   "model_results": [
+    {
+     "model_id": "first",
+     "benchmark_summary": {
+      "candidate_method": "c",
+      "per_method": {
+       "c": {
+        "count": 1,
+        "score": 0.5
+       }
+      },
+      "seed_results": [
+       {
+        "methods": {
+         "c": {
+          "score": 0.5
+         }
+        }
+       }
+      ],
+      "model": {
+       "id": "first-hf"
+      }
+     }
+    },
+    {
+     "model_id": "failed",
+     "benchmark_summary": null
+    },
+    {
+     "model_id": "third",
+     "benchmark_summary": {
+      "per_method": {
+       "c": {
+        "count": 1,
+        "score": 0.7
+       }
+      },
+      "seed_results": [
+       {
+        "methods": {
+         "c": {
+          "score": 0.7
+         }
+        }
+       }
+      ],
+      "model": {
+       "id": "third-hf"
+      },
+      "ablation_table": [
+       {
+        "v": 1
+       }
+      ]
+     }
+    }
+   ],
+   "expected": {
+    "candidate_method": "c",
+    "per_method": {
+     "c": {
+      "score": 0.6,
+      "count": 2
+     }
+    },
+    "seed_results": [
+     {
+      "methods": {
+       "c": {
+        "score": 0.5
+       }
+      },
+      "model_id": "first"
+     },
+     {
+      "methods": {
+       "c": {
+        "score": 0.7
+       }
+      },
+      "model_id": "failed"
+     }
+    ],
+    "model": {
+     "ids": [
+      "first",
+      "failed"
+     ],
+     "backend": "transformers",
+     "cuda": true
+    },
+    "ablation_table": [
+     {
+      "model_id": "failed",
+      "v": 1
+     }
+    ],
+    "num_seeds": 0,
+    "datasets": [],
+    "dataset": {},
+    "models": [
+     {
+      "id": "first"
+     },
+     {
+      "id": "failed"
+     }
+    ],
+    "load_failures": [],
+    "full_benchmark_completed": false,
+    "duration_seconds": 0.0,
+    "peak_vram_mb": 0.0,
+    "hardware": "",
+    "bootstrap_ci": {
+     "candidate_method": "c",
+     "baseline_method": "",
+     "candidate_ci95": [
+      0.0,
+      0.0
+     ],
+     "baseline_ci95": [
+      0.0,
+      0.0
+     ],
+     "paired_permutation_p": null,
+     "p_value": null
+    },
+    "score": 0.6
+   }
+  },
+  {
+   "name": "no results",
+   "criteria_metric_name": "x",
+   "model_results": [],
+   "expected": {}
+  },
+  {
+   "name": "no summary is a dict",
+   "criteria_metric_name": "x",
+   "model_results": [
+    {
+     "benchmark_summary": null
+    },
+    {
+     "model_id": "z"
+    }
+   ],
+   "expected": {}
+  }
+ ],
+ "weighted": [
+  {
+   "rows": [],
+   "expected": {
+    "count": 1
+   }
+  },
+  {
+   "rows": [
+    {
+     "count": 0,
+     "score": 0.5
+    }
+   ],
+   "expected": {
+    "score": 0.5,
+    "count": 1
+   }
+  },
+  {
+   "rows": [
+    {
+     "count": 3,
+     "score": 0.5,
+     "f1": "n/a"
+    },
+    {
+     "count": "7",
+     "score": 1,
+     "route_rate": 0.25
+    },
+    {
+     "score": 0.2,
+     "upper_bound": 1
+    }
+   ],
+   "expected": {
+    "score": 0.8699999999999999,
+    "route_rate": 0.175,
+    "count": 10,
+    "upper_bound": true
+   }
+  },
+  {
+   "rows": [
+    {
+     "count": null,
+     "metric_value": "0.4"
+    },
+    {
+     "count": -2,
+     "metric_value": 0.6
+    }
+   ],
+   "expected": {
+    "metric_value": 1.0,
+    "count": 1
+   }
+  }
+ ]
+}
\ No newline at end of file
diff --git a/tests/test_validation_loop_launch_failures.py b/tests/test_validation_loop_launch_failures.py
new file mode 100644
index 0000000..cda683a
--- /dev/null
+++ b/tests/test_validation_loop_launch_failures.py
@@ -0,0 +1,86 @@
+"""What _run_experiment returns and logs when the runner cannot start or times out.
+
+Characterization tests: the full result, key order included, is part of the
+contract, because callers persist it.
+"""
+
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+from agents import validation_loop as vl
+
+KEYS = ["status", "duration", "error", "failure_type", "final_results_present", "command_tokens", "log_path",
+        "benchmark_env"]
+
+
+class _TimesOut:
+    def __init__(self, *args, **kwargs):
+        self.killed = False
+        self.returncode = None
+
+    def wait(self, timeout=None):
+        if not self.killed:
+            raise subprocess.TimeoutExpired(cmd="train.py", timeout=timeout)
+        return -9
+
+    def kill(self):
+        self.killed = True
+
+
+class LaunchFailureTests(unittest.TestCase):
+    def setUp(self):
+        tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(tmp.cleanup)
+        self.workdir = Path(tmp.name)
+        self.code_dir = self.workdir / "code"
+        self.code_dir.mkdir()
+        (self.code_dir / "train.py").write_text("print('x')\n")
+        for name, value in (("_runner_contract_violations", []), ("_repair_generated_runner_json_literals", []),
+                            ("_benchmark_env_for_execution", {"DEEPGRAPH_BENCHMARK": "1"})):
+            patcher = mock.patch.object(vl, name, return_value=value)
+            patcher.start()
+            self.addCleanup(patcher.stop)
+        patcher = mock.patch.object(vl, "RUNTIME_PYTHON", sys.executable)
+        patcher.start()
+        self.addCleanup(patcher.stop)
+
+    def _run(self):
+        return vl._run_experiment(self.workdir, self.code_dir, 60)
+
+    def test_a_launcher_error_is_reported_with_its_message(self):
+        with mock.patch.object(vl.subprocess, "Popen", side_effect=OSError("runner missing")):
+            result = self._run()
+        self.assertEqual(list(result), KEYS)
+        self.assertEqual(
+            {k: v for k, v in result.items() if k != "duration"},
+            {
+                "status": "crash",
+                "error": "runner missing",
+                "failure_type": "launcher_error",
+                "final_results_present": False,
+                "command_tokens": [sys.executable, "train.py"],
+                "log_path": str(self.workdir / "run.log"),
+                "benchmark_env": {"DEEPGRAPH_BENCHMARK": "1"},
+            },
+        )
+        log = (self.workdir / "run.log").read_text()
+        self.assertRegex(log, r"^Experiment launcher failed after \d+\.\ds\n")
+        self.assertTrue(log.endswith(f"Command: {[sys.executable, 'train.py']}\nError: runner missing\n"))
+
+    def test_a_timeout_kills_the_runner_and_is_reported(self):
+        with mock.patch.object(vl.subprocess, "Popen", _TimesOut):
+            result = self._run()
+        self.assertEqual(list(result), KEYS)
+        self.assertEqual((result["error"], result["failure_type"], result["final_results_present"]),
+                         ("timeout", "timeout", False))
+        log = (self.workdir / "run.log").read_text()
+        self.assertRegex(log, r"^Experiment timed out after \d+\.\ds\nCommand: ")
+        self.assertIn("timed out after", log.splitlines()[-1])
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_validation_loop_merge_characterization.py b/tests/test_validation_loop_merge_characterization.py
new file mode 100644
index 0000000..41a6292
--- /dev/null
+++ b/tests/test_validation_loop_merge_characterization.py
@@ -0,0 +1,54 @@
+"""Pin what the multi-model benchmark merge returns today, so it can be rewritten safely.
+
+Characterization tests: the expected values in
+fixtures/validation_loop_merge_cases.json were produced by the implementation as
+it stood when these tests were added, quirks included, so that a rewrite
+cannot change them by accident:
+- a row with a negative count still weighs 1 in the sum while lowering the
+  total, so rows of 0.4 and 0.6 with counts None and -2 merge to 1.0;
+- summaries are filtered to dicts but model_results are not, so a model
+  without a summary shifts the ids of every model after it (the third model's
+  seeds come out labelled with the second, failed model's id).
+Fixing either is a behaviour change and belongs in its own pull request.
+"""
+
+import copy
+import json
+import unittest
+from pathlib import Path
+
+from agents.validation_loop import _merge_model_benchmark_summaries, _weighted_merge_method_rows
+
+CASES = json.loads((Path(__file__).parent / "fixtures" / "validation_loop_merge_cases.json").read_text())
+
+
+class MergeModelBenchmarkSummariesTests(unittest.TestCase):
+    def test_every_case_merges_exactly_as_recorded(self):
+        for case in CASES["merge"]:
+            with self.subTest(case["name"]):
+                results = copy.deepcopy(case["model_results"])
+                merged = _merge_model_benchmark_summaries(results, case["criteria_metric_name"])
+                # Serialized, so key order and float values must match too:
+                # merged summaries are written to artifact files.
+                self.assertEqual(json.dumps(merged), json.dumps(case["expected"]))
+                self.assertEqual(results, case["model_results"], "the inputs must not be modified")
+
+    def test_merged_summary_is_a_copy(self):
+        case = CASES["merge"][0]
+        results = copy.deepcopy(case["model_results"])
+        merged = _merge_model_benchmark_summaries(results, case["criteria_metric_name"])
+        merged["per_method"]["cand"]["score"] = -1
+        merged["seed_results"][0]["seed"] = -1
+        self.assertEqual(results, case["model_results"])
+
+
+class WeightedMergeMethodRowsTests(unittest.TestCase):
+    def test_every_row_set_merges_exactly_as_recorded(self):
+        for i, case in enumerate(CASES["weighted"]):
+            with self.subTest(i):
+                merged = _weighted_merge_method_rows(copy.deepcopy(case["rows"]))
+                self.assertEqual(json.dumps(merged), json.dumps(case["expected"]))
+
+
+if __name__ == "__main__":
+    unittest.main()