From 340c1d35ecd34fe4e05a1bf88055f6acf76cfa28 Mon Sep 17 00:00:00 2001 From: Viraat Chandra Date: Thu, 13 Aug 2026 12:18:56 -0700 Subject: [PATCH 1/9] feat(config): consolidate all give-up deadlines into settings.timeouts; make --timeout a real run watchdog Reworked from PR #409 review feedback, rebuilt on latest main: - New frozen Timeouts model at settings.timeouts holds every give-up deadline: run_timeout_s (--timeout, whole-run watchdog), service-ready, per-phase drains (absorbs DrainConfig), metrics drain (0-sentinel killed; None = unlimited), and the worker lifecycle waits (moved off settings.client; carriers renamed *_s, excluded from dumps and CLI). - --timeout was consumed nowhere; it now aborts the run: session.stop() then SIGTERM the aggregator (INTERRUPTED final snapshot, first-wins), ExecutionError after finalization - a fired watchdog can never yield a COMPLETE result_summary.json. Deadline is captured before setup; the timer stays armed through the metrics drain. Timed-out runs skip accuracy scoring; audit phases map a fired watchdog to ExecutionError. - publish_final serialized with an asyncio.Lock: a SIGTERM racing the ENDED-driven finalize can no longer abandon a half-written snapshot. - runtime.min_duration_ms/--duration deleted: sample count is explicit (--num-samples) or the dataset issued once. max_duration_ms stays in runtime as the perf-phase workload cap (int|None, gt 0); reaching it is a normal end. MLPerf ruleset path (RuntimeSettings/UserConfig) keeps its internal duration fields. - ServiceLauncher.terminate(module): exact-match SIGTERM; MetricsPipeline.terminate_metrics_aggregator() is the narrow public face. - config/schema.py split into enums/audit/model_params/datasets/settings/ timeouts modules; schema.py keeps the root aggregate + re-export hub. SystemDefaults and TEMPLATE_TYPE_MAP deleted. - Examples, templates, and docs migrated; docs gain a YAML<->CLI time-knob table. Stale inert timeout: values dropped, warmup drain removed from examples. Breaking: bare configs (no --num-samples) now run the dataset once instead of deriving QPS x 10min samples; old YAML keys hard-error via extra=forbid. --- .pre-commit-config.yaml | 2 +- AGENTS.md | 18 +- docs/CLI_DESIGN.md | 3 +- docs/CLI_QUICK_REFERENCE.md | 44 +- docs/LOCAL_TESTING.md | 13 +- docs/config/DESIGN.md | 22 +- examples/02_ServerBenchmarking/README.md | 2 +- .../offline_llama3_8b_cnn.yaml | 4 +- .../online_llama2_70b_cnn.yaml | 4 +- examples/03_BenchmarkComparison/README.md | 22 +- .../compare_with_vllm.py | 10 +- examples/04_GPTOSS120B_Example/Readme.md | 8 +- .../gptoss_120b_example.yaml | 4 +- examples/04_GPTOSS120B_Example/run.py | 7 - .../sglang_gptoss_120b_example.yaml | 5 +- .../vllm_gptoss_120b_example.yaml | 5 +- ...m_gptoss_120b_per_dataset_osl_example.yaml | 8 +- examples/05_Llama_Examples/README.md | 6 +- .../offline_llama3_8b_cnn.yaml | 3 +- .../online_llama2_70b_orca.yaml | 4 +- .../online_llama3_8b_cnn.yaml | 3 +- ...ractive_qwen3_vl_235b_a22b_shopify_8k.yaml | 8 +- .../offline_qwen3_vl_235b_a22b_shopify.yaml | 14 +- .../server_qwen3_vl_235b_a22b_shopify.yaml | 9 +- .../offline_wan22_submission.yaml | 6 +- .../single_stream_wan22_submission.yaml | 6 +- .../kimi_agentic_benchmark.yaml | 3 - .../qwen_agentic_benchmark.yaml | 3 +- .../online_edge_full_run.yaml | 5 +- scripts/bench_drain_tokenize.py | 301 +++++ scripts/regenerate_templates.py | 2 - .../async_utils/services/launcher.py | 14 + .../services/metrics_aggregator/publisher.py | 38 +- src/inference_endpoint/commands/audit.py | 7 + .../commands/benchmark/cli.py | 10 +- .../commands/benchmark/execute.py | 164 ++- .../commands/benchmark/pipeline.py | 27 +- src/inference_endpoint/config/audit.py | 84 ++ src/inference_endpoint/config/datasets.py | 284 +++++ src/inference_endpoint/config/enums.py | 131 +++ src/inference_endpoint/config/model_params.py | 165 +++ .../config/rulesets/mlcommons/rules.py | 3 +- .../config/runtime_settings.py | 23 +- src/inference_endpoint/config/schema.py | 1037 ++--------------- src/inference_endpoint/config/settings.py | 343 ++++++ .../templates/concurrency_template.yaml | 2 - .../templates/concurrency_template_full.yaml | 25 +- .../config/templates/offline_template.yaml | 2 - .../templates/offline_template_full.yaml | 25 +- .../config/templates/online_template.yaml | 2 - .../templates/online_template_full.yaml | 25 +- .../config/templates/submission_template.yaml | 1 - src/inference_endpoint/config/timeouts.py | 145 +++ .../endpoint_client/config.py | 25 +- .../endpoint_client/worker_manager.py | 8 +- .../commands/test_accuracy_pipeline.py | 2 - .../commands/test_benchmark_command.py | 14 +- tests/integration/commands/test_cli.py | 10 +- .../integration/commands/test_run_timeout.py | 172 +++ tests/integration/commands/test_warmup.py | 2 +- .../async_utils/services/test_launcher.py | 46 + .../async_utils/transport/test_protocol.py | 113 ++ tests/unit/commands/test_benchmark.py | 164 +-- tests/unit/compliance/test_output_caching.py | 23 + tests/unit/config/test_schema.py | 27 +- tests/unit/config/test_timeouts.py | 294 +++++ tests/unit/config/test_yaml_loader.py | 15 +- .../scripts/test_metrics_preflight_tap.py | 410 +++++++ 68 files changed, 3095 insertions(+), 1341 deletions(-) create mode 100644 scripts/bench_drain_tokenize.py create mode 100644 src/inference_endpoint/config/audit.py create mode 100644 src/inference_endpoint/config/datasets.py create mode 100644 src/inference_endpoint/config/enums.py create mode 100644 src/inference_endpoint/config/model_params.py create mode 100644 src/inference_endpoint/config/settings.py create mode 100644 src/inference_endpoint/config/timeouts.py create mode 100644 tests/integration/commands/test_run_timeout.py create mode 100644 tests/unit/async_utils/services/test_launcher.py create mode 100644 tests/unit/async_utils/transport/test_protocol.py create mode 100644 tests/unit/config/test_timeouts.py create mode 100644 tests/unit/scripts/test_metrics_preflight_tap.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b5c0943ba..f13dbc726 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -58,7 +58,7 @@ repos: entry: uv run --no-sync python scripts/regenerate_templates.py language: system pass_filenames: false - files: ^(src/inference_endpoint/config/(schema\.py|templates/.*)|src/inference_endpoint/endpoint_client/config\.py|scripts/regenerate_templates\.py)$ + files: ^(src/inference_endpoint/config/((schema|enums|audit|model_params|datasets|settings|timeouts)\.py|templates/.*)|src/inference_endpoint/endpoint_client/config\.py|scripts/regenerate_templates\.py)$ - id: add-license-header name: Add license headers diff --git a/AGENTS.md b/AGENTS.md index 9ef6f83ef..4c41fde22 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -92,7 +92,7 @@ Dataset Manager --> Load Generator --> Endpoint Client --> External Endpoint | **Dataset Manager** | `src/inference_endpoint/dataset_manager/` | Loads JSONL, HuggingFace, CSV, JSON, Parquet datasets. `Dataset` base class with `load_sample()`/`num_samples()` interface | | **Metrics Aggregator** | `src/inference_endpoint/async_utils/services/metrics_aggregator/` | Subprocess. Subscribes to events, aggregates per-sample metrics into a `MetricsRegistry` (counters + HDR-histogram series + raw values), publishes `MetricsSnapshot` over IPC PUB at a configurable cadence (`SessionState`: `INITIALIZE` → `LIVE` → `DRAINING` → {`COMPLETE` \| `INTERRUPTED`}). Final snapshot is atomically written to `final_snapshot.json` as the **primary** Report source; the terminal pub/sub frame is a TUI "run finished" signal only. | | **Report** | `src/inference_endpoint/metrics/report.py` | `Report.from_snapshot(dict)` — pure-function builder consuming the dict form (`snapshot_to_dict`). Reads `final_snapshot.json` directly via `json.loads` (no Struct decode). Plumbs `complete = (state == "complete" and n_pending_tasks == 0)`; renders an explicit warning for `INTERRUPTED` runs. | -| **Config** | `src/inference_endpoint/config/`, `endpoint_client/config.py` | Pydantic-based YAML schema (`schema.py`), `HTTPClientConfig` (single Pydantic model for CLI/YAML/runtime), `RuntimeSettings` | +| **Config** | `src/inference_endpoint/config/`, `endpoint_client/config.py` | Pydantic-based YAML schema split into focused modules (`schema.py` = BenchmarkConfig + EndpointConfig + re-export hub; `enums.py`, `audit.py`, `model_params.py`, `datasets.py`, `settings.py`), `Timeouts` (`config/timeouts.py` — all give-up deadlines in one frozen model at `settings.timeouts`, incl. the whole-run `run_timeout_s` watchdog; the workload duration `runtime.max_duration_ms` stays in `settings.py`), `HTTPClientConfig` (single Pydantic model for CLI/YAML/runtime), `RuntimeSettings` | | **CLI** | `src/inference_endpoint/main.py`, `commands/benchmark/cli.py` | cyclopts-based, auto-generated from `schema.py` and `HTTPClientConfig` Pydantic models. Flat shorthands via `cyclopts.Parameter(alias=...)` | | **Async Utils** | `src/inference_endpoint/async_utils/` | `LoopManager` (uvloop + eager_task_factory), ZMQ transport layer, generic `MessageCodec[T]`-parametrized pub/sub, event publisher | | **OpenAI/SGLang** | `src/inference_endpoint/openai/`, `sglang/` | Protocol adapters and response accumulators for different API formats. `openai_completions` adapter (`completions_adapter.py`) sends pre-tokenized token IDs to `/v1/completions`, bypassing the server chat template — required for gpt-oss-120b on vLLM. `sglang` adapter sends to `/generate` via `input_ids`. Both apply `Harmonize()` client-side. | @@ -118,7 +118,7 @@ The aggregator is a separate process (`python -m inference_endpoint.async_utils. - **Series storage**: each `SeriesSampler` keeps three parallel views: O(1) cheap rollups (count/total/min/max/sum_sq, exact), an HDR Histogram (cheap live percentiles), and an in-memory `array.array` of raw values (for exact percentiles in the `COMPLETE` snapshot). Hot path is `registry.record(name, value)` — no allocation, no I/O. - **Counter API**: `registry.increment(name, delta=1)` for sample-event counters. `registry.set_counter(name, value)` only for the three derived-duration counters (`total_duration_ns` max-of-elapsed, `tracked_duration_ns` sum-of-blocks, `legacy_loadgen_window_duration_ns` first-issue→last-issued-completion span for LoadGen-parity QPS/TPS). -- **Lifecycle**: `INITIALIZE` (constructed, awaiting first `STARTED`) → `LIVE` (run in progress, ticking every `--publish-interval` seconds) → `DRAINING` (set on `ENDED`; tick continues; bounded by the `--drain-timeout` budget — schema default 0 = unlimited) → terminal: `COMPLETE` (clean end via `publish_final`, exact stats) **or** `INTERRUPTED` (signal-handler-triggered final via SIGTERM/SIGINT; best-effort partial stats). Drain timeout detected by consumers as `state == COMPLETE and n_pending_tasks > 0`; interrupted runs are detected as `state == INTERRUPTED` directly. +- **Lifecycle**: `INITIALIZE` (constructed, awaiting first `STARTED`) → `LIVE` (run in progress, ticking every `--publish-interval` seconds) → `DRAINING` (set on `ENDED`; tick continues; bounded by the `--drain-timeout` budget — argv 0 = unlimited; schema `settings.timeouts.metrics_drain_timeout_s` uses None = unlimited, converted at the argv boundary) → terminal: `COMPLETE` (clean end via `publish_final`, exact stats) **or** `INTERRUPTED` (signal-handler-triggered final via SIGTERM/SIGINT; best-effort partial stats). Drain timeout detected by consumers as `state == COMPLETE and n_pending_tasks > 0`; interrupted runs are detected as `state == INTERRUPTED` directly. - **Final delivery is dual-path with separated concerns**: `publish_final` atomically writes `final_snapshot.json` (`tmp + fsync(file) + rename + fsync(parent_dir)`) — this is the **primary** Report source — AND emits the terminal-state snapshot over pub/sub as a TUI shutdown signal. Each path is wrapped in its own try/except so one failure cannot suppress the other. Main process consumer reads `final_snapshot.json` (via `json.loads` to dict, no Struct decode); falls back to the subscriber's `latest` live snapshot only if the file is missing (e.g. SIGKILL / OOM before the signal handler ran). The dict form is the canonical consumer contract (see `snapshot_to_dict`). - **Early stopping (on by default)**: series registered with `register_series(..., tail_latency=True)` (today ttft/tpot/latency) get MLPerf early-stopping percentile estimates on the COMPLETE (exact) snapshot — a compact `early_stopping_percentiles` map in `result_summary.json` whose keys mirror the `percentiles` grid (≥ p50) with estimate-or-`null` values; rich detail is INFO-logged. On by default (cold-path only; the exact path shares one in-place sort between the percentile grid and the estimates); `settings.early_stopping.enabled: false` / `--no-early-stopping` opts out. Confidence/tolerance are LoadGen constants. Pure math in `metrics/early_stopping.py`; post-hoc recomputation from any run's `events.jsonl` via `scripts/early_stopping_estimate_from_events.py`. See docs/early_stopping.md. - **Histogram bucket edges are dynamic per snapshot**: log-spaced over the observed `[min, max]`. Bucket count is fixed at construction; consumers MUST re-render from the snapshot's `(lo, hi, count)` triples each frame and MUST NOT track bucket-by-index across snapshots. @@ -128,7 +128,7 @@ The aggregator is a separate process (`python -m inference_endpoint.async_utils. CLI is auto-generated from `config/schema.py` Pydantic models via cyclopts. Fields annotated with `cyclopts.Parameter(alias="--flag")` get flat shorthands; all other fields get auto-generated dotted flags (kebab-case). - **CLI mode** (`offline`/`online`): cyclopts constructs `OfflineBenchmarkConfig`/`OnlineBenchmarkConfig` (subclasses in `config/schema.py`) directly from CLI args. Type locked via `Literal`. `--dataset` is repeatable with TOML-style format `[perf|acc:][,key=value...]` (e.g. `--dataset data.csv,samples=500,parser.prompt=article`). Full accuracy support via `accuracy_config.eval_method=pass_at_1` etc. -- **YAML mode** (`from-config`): `BenchmarkConfig.from_yaml_file()` loads YAML, resolves env vars, and auto-selects the right subclass via Pydantic discriminated union. Optional `--timeout`/`--mode` overrides via `config.with_updates()`. +- **YAML mode** (`from-config`): `BenchmarkConfig.from_yaml_file()` loads YAML, resolves env vars, and auto-selects the right subclass via Pydantic discriminated union. Optional `--timeout` (maps to `settings.timeouts.run_timeout_s`)/`--mode` overrides via `config.with_updates()`. - **eval**: Not yet implemented (raises `CLIError` with a tracking issue link) ### Config Construction & Validation @@ -148,9 +148,9 @@ YAML from-config: from_yaml_file(path) → discriminated union → same subcl Validation is layered: -1. **Field-level** (Pydantic): `Field(ge=0)` on durations, `Field(ge=-1)` on workers, `Literal` on `benchmark_mode` +1. **Field-level** (Pydantic): `Field(gt=0)` on durations/deadlines, `Field(ge=-1)` on workers, `Literal` on `benchmark_mode` 2. **Field validators**: `workers != 0` check -3. **Model validator** (`_resolve_and_validate`): streaming AUTO resolution, model name from `submission_ref`, load pattern vs test type, cross-field duration check, duplicate datasets +3. **Model validator** (`_resolve_and_validate`): streaming AUTO resolution, model name from `submission_ref`, load pattern vs test type, duplicate datasets ### Load Patterns @@ -244,7 +244,13 @@ src/inference_endpoint/ │ ├── early_stopping.py # MLPerf LoadGen early-stopping percentile estimates (pure math; see docs/early_stopping.md) │ └── results_plots.py # Standardized run-artifact plots (matplotlib-guarded); CLI: scripts/plot_results.py ├── config/ -│ ├── schema.py # Single source of truth: Pydantic models + cyclopts annotations +│ ├── schema.py # BenchmarkConfig + EndpointConfig; re-export hub for the schema surface +│ ├── enums.py # Shared schema enums (TestType, LoadPatternType, StreamingMode, ...) +│ ├── audit.py # Audit config models (audit: YAML block) +│ ├── model_params.py # ModelParams, OSLDistribution, SubmissionReference +│ ├── datasets.py # Dataset, AccuracyConfig, AgenticInferenceConfig +│ ├── settings.py # Settings + Runtime/LoadPattern/Warmup/Profiling/EarlyStopping configs +│ ├── timeouts.py # Timeouts — all give-up deadlines (settings.timeouts) │ ├── runtime_settings.py # RuntimeSettings + SampleOrderSpec dataclasses │ ├── ruleset_base.py # BenchmarkSuiteRuleset base │ ├── ruleset_registry.py # Ruleset registry diff --git a/docs/CLI_DESIGN.md b/docs/CLI_DESIGN.md index da4799a14..474c81fb9 100644 --- a/docs/CLI_DESIGN.md +++ b/docs/CLI_DESIGN.md @@ -131,7 +131,6 @@ Validation is layered, executing in order: 1. cyclopts → required args? unknown flags? 2. Pydantic fields → type coercion, ge/le constraints 3. Sub-model validators: - ├── RuntimeConfig._validate_durations → max >= min duration ├── LoadPattern._validate_completeness → poisson needs qps, concurrency needs target └── HTTPClientConfig._workers_not_zero → num_workers != 0 4. BenchmarkConfig._resolve_and_validate: @@ -204,5 +203,5 @@ class HTTPClientConfig(WithUpdatesMixin, BaseModel): `BenchmarkConfig` is frozen. Use `with_updates()` to produce new instances with re-validation: ```python -config = config.with_updates(timeout=300, datasets=["new_data.jsonl"]) +config = config.with_updates(report_dir="results/run1", datasets=["new_data.jsonl"]) ``` diff --git a/docs/CLI_QUICK_REFERENCE.md b/docs/CLI_QUICK_REFERENCE.md index 2367e1c93..6f69d7b6b 100644 --- a/docs/CLI_QUICK_REFERENCE.md +++ b/docs/CLI_QUICK_REFERENCE.md @@ -96,8 +96,7 @@ Flag names shown as `--full.dotted.path --alias`. Both forms work. - `--model-params.max-new-tokens --max-output-tokens` - Max output tokens (default: 1024) - `--model-params.osl-distribution.min --min-output-tokens` - Min output tokens (default: 1) - `--model-params.streaming --streaming` - Streaming mode: auto/on/off (default: auto) -- `--runtime.min-duration-ms --duration` - Min duration: ms default, or with suffix (600s, 10m) (default: 600000) -- `--runtime.n-samples-to-issue --num-samples` - Explicit sample count override +- `--runtime.n-samples-to-issue --num-samples` - Explicit sample count (omit to issue the dataset once — the default) - `--client.num-workers --workers` - HTTP workers (-1=auto, default: -1) - `--client.max-connections --max-connections` - Max TCP connections (-1=unlimited) - `--endpoint-config.api-key --api-key` - API authentication @@ -106,7 +105,7 @@ Flag names shown as `--full.dotted.path --alias`. Both forms work. Note: applies to CLI-driven `benchmark offline` / `benchmark online`; `benchmark from-config` does not expose a CLI override for `report_dir`. Set it in the YAML only if you need to control the output location; otherwise a default report directory is used. -- `--timeout` - Global timeout in seconds +- `--timeout` - Whole-run watchdog in seconds (off by default). If it fires, the run is aborted, the report is marked INTERRUPTED, and the process exits non-zero. - `--enable-cpu-affinity / --no-cpu-affinity` - NUMA-aware CPU pinning (default: true) - `--no-early-stopping` - opt out of the MLPerf early-stopping percentile estimates in `result_summary.json` (default: on; see [early_stopping.md](early_stopping.md)) @@ -118,6 +117,35 @@ Flag names shown as `--full.dotted.path --alias`. Both forms work. **All other schema fields** are accessible via dotted paths (e.g., `--model-params.temperature`, `--model-params.top-k`, `--runtime.scheduler-random-seed`). Run `--help` to see the full list. +## Time Knobs + +All give-up deadlines live under `settings.timeouts`; the only workload duration is +`settings.runtime.max_duration_ms`. `null`/unset means "wait indefinitely" (or "off") everywhere. + +| YAML path | CLI flag | Semantics | +| --------------------------------------------------- | ----------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | +| `settings.runtime.max_duration_ms` | `--runtime.max-duration-ms` | Caps the performance phase (ms, or suffix: `600s`, `10m`); reaching it ends the phase NORMALLY — the report stays valid | +| `settings.timeouts.run_timeout_s` | `--timeout` | Whole-run watchdog; firing aborts the entire run — report marked INTERRUPTED, non-zero exit | +| `settings.timeouts.service_ready_timeout_s` | `--service-ready-timeout` | Wait for the metrics-aggregator/event-logger services to become ready (default 30) | +| `settings.timeouts.warmup_drain_timeout_s` | `--warmup-drain-timeout` | Bound on in-flight warmup requests after the warmup phase ends (default 240) | +| `settings.timeouts.performance_drain_timeout_s` | `--performance-drain-timeout` | Bound on in-flight performance requests after the phase ends (default: wait indefinitely) | +| `settings.timeouts.accuracy_drain_timeout_s` | `--accuracy-drain-timeout` | Bound on in-flight accuracy requests after the phase ends (default: wait indefinitely) | +| `settings.timeouts.metrics_drain_timeout_s` | `--metrics-drain-timeout` | Budget for the metrics aggregator to finish tokenizing buffered samples after the run ends (default: wait indefinitely) | +| `settings.timeouts.worker_initialization_timeout_s` | `--worker-initialization-timeout-s` | Wait for endpoint-client worker processes to start (default 60) | +| `settings.timeouts.worker_graceful_shutdown_wait_s` | `--worker-graceful-shutdown-wait-s` | Post-run wait for workers to exit gracefully (default 0.5) | +| `settings.timeouts.worker_force_kill_timeout_s` | `--worker-force-kill-timeout-s` | Wait after SIGTERM before SIGKILL during worker teardown (default 0.5) | + +How the knobs compose: + +1. **`--num-samples` / dataset-once defines the work.** An explicit `runtime.n_samples_to_issue` + sets the sample count; omitting it issues the performance dataset once. +2. **`runtime.max_duration_ms` caps the performance phase** and ends it normally — remaining + samples are not issued, the report is valid. +3. **`timeouts.run_timeout_s` aborts the whole run** (every phase, drains included) — the report + is marked INTERRUPTED and the process exits non-zero. +4. **Per-phase drain timeouts bound the post-phase wait** for requests still in flight after a + phase stops issuing. + ## Environment Variables **In YAML files** — use `${VAR}` or `${VAR:-default}` syntax: @@ -224,14 +252,13 @@ inference-endpoint benchmark online \ --report-dir production_report \ -v -# Or with duration (calculates samples from target_qps * duration) +# Without --num-samples, the dataset is issued once (the default) inference-endpoint benchmark online \ --endpoints https://api.production.com \ --model Qwen/Qwen3-8B \ --dataset prod_queries.jsonl \ --load-pattern poisson \ --target-qps 100 \ - --duration 5m \ --workers 16 \ --report-dir production_report \ -v @@ -290,8 +317,7 @@ datasets: settings: runtime: - min_duration_ms: 600000 # 10 minutes - n_samples_to_issue: null # Optional: explicit sample count (null = auto-calculate) + n_samples_to_issue: null # Optional: explicit sample count (null = issue the dataset once) scheduler_random_seed: 42 # For Poisson/distribution sampling dataloader_random_seed: 42 # For dataset shuffling load_pattern: @@ -331,8 +357,8 @@ Note: For submission configs, `model_params.name` is optional when `submission_r **Sample Count Control:** -- Priority: `--num-samples` > calculated (target_qps × duration) > dataset size -- Default duration: 600000ms (10 minutes) +- `--num-samples` sets an explicit sample count; without it the dataset is issued once +- Behavior change: bare configs (no `--num-samples`) now run the dataset once instead of deriving 10 minutes' worth of samples from the target QPS **Mode Requirements:** diff --git a/docs/LOCAL_TESTING.md b/docs/LOCAL_TESTING.md index b8883264e..b8bd59d03 100644 --- a/docs/LOCAL_TESTING.md +++ b/docs/LOCAL_TESTING.md @@ -74,8 +74,7 @@ Waiting for 5 responses... uv run inference-endpoint -v benchmark offline \ --endpoints http://localhost:8765 \ --model Qwen/Qwen3-8B \ - --dataset tests/assets/datasets/dummy_1k.jsonl \ - --duration 0 + --dataset tests/assets/datasets/dummy_1k.jsonl # Production test with custom params and report generation uv run inference-endpoint -v benchmark offline \ @@ -97,7 +96,7 @@ Loading: dummy_1k.jsonl Loaded 1000 samples Mode: TestMode.PERF, QPS: 10.0, Responses: False Streaming: disabled (auto, offline mode) -Min Duration: 0.0s, Expected samples: 1000 +Expected samples: 1000 Scheduler: MaxThroughputScheduler (pattern: max_throughput) Connecting: http://localhost:8765 Running... @@ -115,7 +114,6 @@ uv run inference-endpoint -v benchmark online \ --endpoints http://localhost:8765 \ --model Qwen/Qwen3-8B \ --dataset tests/assets/datasets/dummy_1k.jsonl \ - --duration 0 \ --load-pattern poisson \ --target-qps 100 \ --report-dir online_benchmark_report @@ -128,7 +126,7 @@ Loading: dummy_1k.jsonl Loaded 1000 samples Mode: TestMode.PERF, QPS: 100.0, Responses: False Streaming: enabled (auto, online mode) -Min Duration: 0.0s, Expected samples: 1000 +Expected samples: 1000 Scheduler: PoissonDistributionScheduler (pattern: poisson) Connecting: http://localhost:8765 Running... @@ -311,9 +309,8 @@ uv run inference-endpoint benchmark online \ **Sample Count Control:** -- Use `--duration 0` when you want a local test to stop after exhausting the dataset instead of running for the default timed duration -- Sample priority: `--num-samples` > dataset size (when `--duration 0`) > calculated (target_qps × duration) -- Default duration: 600000ms (10 minutes) +- By default (no `--num-samples`) a run stops after issuing the dataset once +- Use `--num-samples` for an explicit sample count **Testing & Debugging:** diff --git a/docs/config/DESIGN.md b/docs/config/DESIGN.md index 795efbb66..b8208c909 100644 --- a/docs/config/DESIGN.md +++ b/docs/config/DESIGN.md @@ -57,17 +57,17 @@ Key nested models: Immutable snapshot of all parameters needed to execute a run. -| Field | Type | Source | -| -------------------- | -------------- | --------------------------------------- | -| `load_pattern` | `LoadPattern` | config | -| `n_samples_to_issue` | `int` | calculated: QPS × duration, or explicit | -| `min_duration_ms` | `int` | runtime config | -| `max_duration_ms` | `int` | runtime config | -| `min_sample_count` | `int` | current default / future ruleset hook | -| `metric_target` | `Metric` | primary target driving scheduler logic | -| `reported_metrics` | `list[Metric]` | metrics validated after the run | -| `rng_sched` | `Random` | seeded from `scheduler_random_seed` | -| `rng_sample_index` | `Random` | seeded from `dataloader_random_seed` | +| Field | Type | Source | +| -------------------- | -------------- | ----------------------------------------- | +| `load_pattern` | `LoadPattern` | config | +| `n_samples_to_issue` | `int` | explicit, or dataset size (issue once) | +| `min_duration_ms` | `int \| None` | ruleset override path only (`UserConfig`) | +| `max_duration_ms` | `int \| None` | runtime config | +| `min_sample_count` | `int` | current default / future ruleset hook | +| `metric_target` | `Metric` | primary target driving scheduler logic | +| `reported_metrics` | `list[Metric]` | metrics validated after the run | +| `rng_sched` | `Random` | seeded from `scheduler_random_seed` | +| `rng_sample_index` | `Random` | seeded from `dataloader_random_seed` | Once constructed, `RuntimeSettings` cannot be modified. All consumers receive the same instance. diff --git a/examples/02_ServerBenchmarking/README.md b/examples/02_ServerBenchmarking/README.md index bfb8e5b95..797b1c88e 100644 --- a/examples/02_ServerBenchmarking/README.md +++ b/examples/02_ServerBenchmarking/README.md @@ -81,6 +81,6 @@ dataset["train"].to_json("cnn_dailymail_train.json") And then launch the example template. ``` -uv run inference-endpoint benchmark from-config -c examples/02_ServerBenchmarking/online_llama2_70b_cnn.yaml --timeout 600 +uv run inference-endpoint benchmark from-config -c examples/02_ServerBenchmarking/online_llama2_70b_cnn.yaml ``` diff --git a/examples/02_ServerBenchmarking/offline_llama3_8b_cnn.yaml b/examples/02_ServerBenchmarking/offline_llama3_8b_cnn.yaml index bc5b92f1d..5420d4521 100644 --- a/examples/02_ServerBenchmarking/offline_llama3_8b_cnn.yaml +++ b/examples/02_ServerBenchmarking/offline_llama3_8b_cnn.yaml @@ -19,8 +19,8 @@ datasets: settings: runtime: - min_duration_ms: 6000 # 6 seconds - max_duration_ms: 60000 # 1 minute + max_duration_ms: 60000 # 1 minute cap on the performance phase + n_samples_to_issue: 1000 # ≈ ceil(10 QPS × 6 s × 1.1) rounded up to the 1000-sample dataset (replaces the duration-derived count) scheduler_random_seed: 137 # For Poisson/distribution sampling dataloader_random_seed: 111 # For dataset shuffling diff --git a/examples/02_ServerBenchmarking/online_llama2_70b_cnn.yaml b/examples/02_ServerBenchmarking/online_llama2_70b_cnn.yaml index d16035447..3dac9b4bb 100644 --- a/examples/02_ServerBenchmarking/online_llama2_70b_cnn.yaml +++ b/examples/02_ServerBenchmarking/online_llama2_70b_cnn.yaml @@ -19,8 +19,8 @@ datasets: settings: runtime: - min_duration_ms: 60000 # 1 minute - max_duration_ms: 180000 # 3 minutes + max_duration_ms: 180000 # 3 minute cap on the performance phase + n_samples_to_issue: 1000 # ≈ ceil(10 QPS × 60 s × 1.1) = 660 rounded up to the 1000-sample dataset (replaces the duration-derived count) scheduler_random_seed: 42 # For Poisson/distribution sampling dataloader_random_seed: 42 # For dataset shuffling diff --git a/examples/03_BenchmarkComparison/README.md b/examples/03_BenchmarkComparison/README.md index fbb60b9f4..d79b32176 100644 --- a/examples/03_BenchmarkComparison/README.md +++ b/examples/03_BenchmarkComparison/README.md @@ -33,17 +33,17 @@ uv run python compare_with_vllm.py --model "Qwen/Qwen2.5-0.5B-Instruct" --endpoi ### Options -| Option | Description | Default | -| --------------------- | -------------------------------- | ----------------------- | -| `--model`, `-m` | Model name (required) | - | -| `--num-prompts`, `-n` | Number of prompts | 100 | -| `--endpoint` | Server URL | `http://localhost:8000` | -| `--max-output-tokens` | Max output tokens | 2000 | -| `--timeout` | Timeout in seconds | 900 | -| `--workers` | Number of workers | 1 | -| `--verbose`, `-v` | Show full output from each run | - | -| `--dry` | Print commands without executing | - | -| `--vllm-venv-dir` | Path to vLLM virtualenv | `./vllm_venv` | +| Option | Description | Default | +| --------------------- | -------------------------------------------------------------------------------- | ----------------------- | +| `--model`, `-m` | Model name (required) | - | +| `--num-prompts`, `-n` | Number of prompts | 100 | +| `--endpoint` | Server URL | `http://localhost:8000` | +| `--max-output-tokens` | Max output tokens | 2000 | +| `--timeout` | Whole-run watchdog (seconds) passed to inference-endpoint; firing aborts the run | 900 | +| `--workers` | Number of workers | 1 | +| `--verbose`, `-v` | Show full output from each run | - | +| `--dry` | Print commands without executing | - | +| `--vllm-venv-dir` | Path to vLLM virtualenv | `./vllm_venv` | ### Example diff --git a/examples/03_BenchmarkComparison/compare_with_vllm.py b/examples/03_BenchmarkComparison/compare_with_vllm.py index fc1e6c069..64a28c7f5 100644 --- a/examples/03_BenchmarkComparison/compare_with_vllm.py +++ b/examples/03_BenchmarkComparison/compare_with_vllm.py @@ -129,7 +129,6 @@ def generate_ie_config( num_requests: int, max_output_tokens: int, workers: int, - timeout: int, report_dir: Path, config_path: Path, ) -> None: @@ -148,7 +147,6 @@ def generate_ie_config( num_requests: Number of requests to send max_output_tokens: Maximum output tokens per request workers: Number of parallel http-client workers - timeout: Timeout in seconds report_dir: Directory to save reports config_path: Path to write the config file """ @@ -177,8 +175,6 @@ def generate_ie_config( ], "settings": { "runtime": { - "min_duration_ms": 0, - "max_duration_ms": timeout * 1000, "n_samples_to_issue": num_requests, }, "load_pattern": {"type": "max_throughput"}, @@ -186,7 +182,6 @@ def generate_ie_config( }, "endpoint_config": {"endpoints": [endpoint_url]}, "report_dir": str(report_dir), - "timeout": timeout, } with open(config_path, "w") as f: @@ -227,7 +222,7 @@ def parse_args() -> argparse.Namespace: "--timeout", type=int, default=900, - help="Timeout in seconds for inference-endpoint (default: 900)", + help="Whole-run watchdog in seconds passed to inference-endpoint (default: 900)", ) parser.add_argument( "--workers", @@ -278,7 +273,7 @@ def run_inference_endpoint( endpoint_url: Server endpoint URL num_requests: Number of requests to send max_output_tokens: Maximum output tokens per request - timeout: Timeout in seconds + timeout: Whole-run watchdog in seconds passed via --timeout workers: Number of parallel http-client workers temp_dir: Temporary directory to save report and config dry_run: If True, print command without executing it @@ -301,7 +296,6 @@ def run_inference_endpoint( num_requests=num_requests, max_output_tokens=max_output_tokens, workers=workers, - timeout=timeout, report_dir=report_dir, config_path=config_path, ) diff --git a/examples/04_GPTOSS120B_Example/Readme.md b/examples/04_GPTOSS120B_Example/Readme.md index f9666a1bf..68be9906c 100644 --- a/examples/04_GPTOSS120B_Example/Readme.md +++ b/examples/04_GPTOSS120B_Example/Readme.md @@ -49,8 +49,7 @@ docker run --runtime nvidia --gpus all \ ```bash uv run inference-endpoint benchmark from-config \ - -c examples/04_GPTOSS120B_Example/vllm_gptoss_120b_example.yaml \ - --timeout 60 + -c examples/04_GPTOSS120B_Example/vllm_gptoss_120b_example.yaml ``` The config uses `api_type: openai_completions`, which routes to `/v1/completions` with pre-tokenized @@ -171,8 +170,7 @@ LiveCodeBench accuracy at concurrency 512: ```bash uv run inference-endpoint benchmark from-config \ - -c examples/04_GPTOSS120B_Example/sglang_gptoss_120b_example.yaml \ - --timeout 60 + -c examples/04_GPTOSS120B_Example/sglang_gptoss_120b_example.yaml ``` For a performance-only run, use [`gptoss_120b_example.yaml`](gptoss_120b_example.yaml). It is @@ -210,7 +208,6 @@ cd examples/04_GPTOSS120B_Example python run.py \ --report-dir ./results \ --num-repeats 1 \ - --min-duration 10 \ --max-duration 600 ``` @@ -218,7 +215,6 @@ python run.py \ | -------------------- | ------------------------ | ------------------------------------ | | `--report-dir` | `sglang_accuracy_report` | Directory to save results | | `--num-repeats` | `1` | Repeats per dataset | -| `--min-duration` | `10` | Minimum benchmark duration (seconds) | | `--max-duration` | `600` | Maximum benchmark duration (seconds) | | `--force-regenerate` | off | Force dataset regeneration | diff --git a/examples/04_GPTOSS120B_Example/gptoss_120b_example.yaml b/examples/04_GPTOSS120B_Example/gptoss_120b_example.yaml index 0bd3a2571..cbaeea2b3 100644 --- a/examples/04_GPTOSS120B_Example/gptoss_120b_example.yaml +++ b/examples/04_GPTOSS120B_Example/gptoss_120b_example.yaml @@ -17,8 +17,8 @@ datasets: settings: runtime: - min_duration_ms: 300 - max_duration_ms: 6000 + max_duration_ms: 6000 # 6 s cap on the performance phase (short smoke run) + # Sample count: dataset issued once (the default) — replaces the old duration-derived count, which rounded up to one dataset pass. scheduler_random_seed: 42 dataloader_random_seed: 42 diff --git a/examples/04_GPTOSS120B_Example/run.py b/examples/04_GPTOSS120B_Example/run.py index 79b2e7cc3..9e8f43001 100644 --- a/examples/04_GPTOSS120B_Example/run.py +++ b/examples/04_GPTOSS120B_Example/run.py @@ -111,7 +111,6 @@ def run_benchmark_session( rt_settings = RuntimeSettings( metric_target=metrics.Throughput(6), reported_metrics=[], - min_duration_ms=args.min_duration * 1000, max_duration_ms=args.max_duration * 1000, n_samples_from_dataset=0, n_samples_to_issue=0, @@ -265,12 +264,6 @@ def main(): ) # Benchmark configuration arguments - parser.add_argument( - "--min-duration", - type=int, - default=10, - help="Minimum duration in seconds (default: 10)", - ) parser.add_argument( "--max-duration", type=int, diff --git a/examples/04_GPTOSS120B_Example/sglang_gptoss_120b_example.yaml b/examples/04_GPTOSS120B_Example/sglang_gptoss_120b_example.yaml index 5a2d2050e..650a95648 100644 --- a/examples/04_GPTOSS120B_Example/sglang_gptoss_120b_example.yaml +++ b/examples/04_GPTOSS120B_Example/sglang_gptoss_120b_example.yaml @@ -1,7 +1,6 @@ name: "gpt-oss-120b-benchmark" version: "1.0" type: "online" -timeout: 60 model_params: name: "openai/gpt-oss-120b" @@ -39,8 +38,8 @@ datasets: num_repeats: 5 settings: runtime: - min_duration_ms: 3000 - max_duration_ms: 60000 + max_duration_ms: 60000 # 1 minute cap on the performance phase + # Sample count: perf dataset issued once (the default) — replaces the old duration-derived count, which rounded up to one dataset pass. scheduler_random_seed: 42 dataloader_random_seed: 42 diff --git a/examples/04_GPTOSS120B_Example/vllm_gptoss_120b_example.yaml b/examples/04_GPTOSS120B_Example/vllm_gptoss_120b_example.yaml index 2780e0b49..e4a19063c 100644 --- a/examples/04_GPTOSS120B_Example/vllm_gptoss_120b_example.yaml +++ b/examples/04_GPTOSS120B_Example/vllm_gptoss_120b_example.yaml @@ -1,7 +1,6 @@ name: "gpt-oss-120b-benchmark" version: "1.0" type: "online" -timeout: 60 model_params: name: "openai/gpt-oss-120b" @@ -42,8 +41,8 @@ datasets: settings: runtime: - min_duration_ms: 3000 - max_duration_ms: 60000 + max_duration_ms: 60000 # 1 minute cap on the performance phase + # Sample count: perf dataset issued once (the default) — replaces the old duration-derived count, which rounded up to one dataset pass. scheduler_random_seed: 42 dataloader_random_seed: 42 diff --git a/examples/04_GPTOSS120B_Example/vllm_gptoss_120b_per_dataset_osl_example.yaml b/examples/04_GPTOSS120B_Example/vllm_gptoss_120b_per_dataset_osl_example.yaml index 2a944a00b..351537fcd 100644 --- a/examples/04_GPTOSS120B_Example/vllm_gptoss_120b_per_dataset_osl_example.yaml +++ b/examples/04_GPTOSS120B_Example/vllm_gptoss_120b_per_dataset_osl_example.yaml @@ -16,7 +16,6 @@ name: "gpt-oss-120b-per-dataset-osl" version: "1.0" type: "online" -timeout: 9000 model_params: name: "openai/gpt-oss-120b" @@ -64,8 +63,7 @@ datasets: settings: runtime: - min_duration_ms: 30000 - max_duration_ms: 14400000 # 4h whole-session deadline (perf + all accuracy phases) + max_duration_ms: 14400000 # 4h cap on the performance phase only scheduler_random_seed: 42 dataloader_random_seed: 42 n_samples_to_issue: 2000 # PERF phase only; accuracy phases issue their own sample counts @@ -74,9 +72,11 @@ settings: type: "concurrency" target_concurrency: 1024 + timeouts: + worker_initialization_timeout_s: 300.0 + client: num_workers: 16 - worker_initialization_timeout: 300.0 log_level: "WARN" worker_gc_mode: "disabled" diff --git a/examples/05_Llama_Examples/README.md b/examples/05_Llama_Examples/README.md index f94566541..5bb795c5d 100644 --- a/examples/05_Llama_Examples/README.md +++ b/examples/05_Llama_Examples/README.md @@ -44,13 +44,13 @@ docker run --runtime nvidia --gpus all \ ### Offline mode ```bash -uv run inference-endpoint benchmark from-config -c offline_llama3_8b_cnn.yaml --timeout 600 +uv run inference-endpoint benchmark from-config -c offline_llama3_8b_cnn.yaml ``` ### Online mode ```bash -uv run inference-endpoint benchmark from-config -c online_llama3_8b_cnn.yaml --timeout 600 +uv run inference-endpoint benchmark from-config -c online_llama3_8b_cnn.yaml ``` These configs run in performance-only mode by default. To also evaluate summarization quality, add `--mode both` and install the accuracy dependencies listed in the [Llama-2-70b accuracy setup](#accuracy-evaluation-setup-optional) section below. @@ -104,5 +104,5 @@ docker run --runtime nvidia --gpus all \ ### Online mode ```bash -uv run inference-endpoint benchmark from-config -c online_llama2_70b_orca.yaml --timeout 600 +uv run inference-endpoint benchmark from-config -c online_llama2_70b_orca.yaml ``` diff --git a/examples/05_Llama_Examples/offline_llama3_8b_cnn.yaml b/examples/05_Llama_Examples/offline_llama3_8b_cnn.yaml index 57e105c76..c08e17f21 100644 --- a/examples/05_Llama_Examples/offline_llama3_8b_cnn.yaml +++ b/examples/05_Llama_Examples/offline_llama3_8b_cnn.yaml @@ -28,8 +28,7 @@ datasets: settings: runtime: - min_duration_ms: 60000 # 1 minute - max_duration_ms: 360000 # 6 minutes (Arbitrary here, and doesn't have counterpart in legacy loadgen) + max_duration_ms: 360000 # 6 minute cap on the performance phase (Arbitrary here, and doesn't have counterpart in legacy loadgen) scheduler_random_seed: 137 # For Poisson/distribution sampling dataloader_random_seed: 111 # For dataset shuffling (Will be updated after rng seeds are finalized for submission) n_samples_to_issue: 13368 # Number of samples to issue (for offline, this should match the dataset samples) diff --git a/examples/05_Llama_Examples/online_llama2_70b_orca.yaml b/examples/05_Llama_Examples/online_llama2_70b_orca.yaml index 5a7f6ce53..3c227111d 100644 --- a/examples/05_Llama_Examples/online_llama2_70b_orca.yaml +++ b/examples/05_Llama_Examples/online_llama2_70b_orca.yaml @@ -22,8 +22,8 @@ datasets: settings: runtime: - min_duration_ms: 60000 # 1 minute - max_duration_ms: 600000 # 10 minutes + max_duration_ms: 600000 # 10 minute cap on the performance phase + # Sample count: dataset issued once (the default) — replaces the old duration-derived count, which rounded up to one dataset pass. scheduler_random_seed: 42 # For Poisson/distribution sampling dataloader_random_seed: 42 # For dataset shuffling diff --git a/examples/05_Llama_Examples/online_llama3_8b_cnn.yaml b/examples/05_Llama_Examples/online_llama3_8b_cnn.yaml index 66861f2f5..82de8f7bc 100644 --- a/examples/05_Llama_Examples/online_llama3_8b_cnn.yaml +++ b/examples/05_Llama_Examples/online_llama3_8b_cnn.yaml @@ -28,8 +28,7 @@ datasets: settings: runtime: - min_duration_ms: 600000 # 10 minutes - max_duration_ms: 3600000 # 60 minutes (Arbitrary here, and doesn't have counterpart in legacy loadgen) + max_duration_ms: 3600000 # 60 minute cap on the performance phase (Arbitrary here, and doesn't have counterpart in legacy loadgen) scheduler_random_seed: 137 # For Poisson/distribution sampling dataloader_random_seed: 111 # For dataset shuffling (Will be updated after rng seeds are finalized for submission) n_samples_to_issue: 13368 diff --git a/examples/08_Qwen3-VL-235B-A22B_Example/interactive_qwen3_vl_235b_a22b_shopify_8k.yaml b/examples/08_Qwen3-VL-235B-A22B_Example/interactive_qwen3_vl_235b_a22b_shopify_8k.yaml index d28162dc0..bd8cb0854 100644 --- a/examples/08_Qwen3-VL-235B-A22B_Example/interactive_qwen3_vl_235b_a22b_shopify_8k.yaml +++ b/examples/08_Qwen3-VL-235B-A22B_Example/interactive_qwen3_vl_235b_a22b_shopify_8k.yaml @@ -3,7 +3,6 @@ name: "interactive-qwen3-vl-235b-a22b-shopify-8k-benchmark" version: "1.0" type: "online" -timeout: 1800 # 30 minutes for quick interactive runs model_params: name: "Qwen/Qwen3-VL-235B-A22B-Instruct" @@ -23,7 +22,7 @@ datasets: settings: runtime: - min_duration_ms: 600000 # 10 minute + # Sample count: dataset issued once (the default) — replaces the old duration-derived count, which rounded up to one dataset pass. scheduler_random_seed: 42 dataloader_random_seed: 42 @@ -38,7 +37,10 @@ settings: recv_buffer_size: 16777216 send_buffer_size: 16777216 max_connections: 1000 - worker_initialization_timeout: 120 + + timeouts: + # Increase for slow worker startup (spawn, imports). Default 60s may be too short. + worker_initialization_timeout_s: 120 warmup: enabled: true # Enable warmup phase before performance run diff --git a/examples/08_Qwen3-VL-235B-A22B_Example/offline_qwen3_vl_235b_a22b_shopify.yaml b/examples/08_Qwen3-VL-235B-A22B_Example/offline_qwen3_vl_235b_a22b_shopify.yaml index 2a1bd203f..5b2660af9 100644 --- a/examples/08_Qwen3-VL-235B-A22B_Example/offline_qwen3_vl_235b_a22b_shopify.yaml +++ b/examples/08_Qwen3-VL-235B-A22B_Example/offline_qwen3_vl_235b_a22b_shopify.yaml @@ -3,7 +3,6 @@ name: "offline-qwen3-vl-235b-a22b-shopify-benchmark" version: "1.0" type: "offline" -timeout: 14400 # Perf + acc run takes over 3 hours, consider limit n_samples_to_issue for perf run or remove accuracy dataset to skip accuracy run model_params: name: "Qwen/Qwen3-VL-235B-A22B-Instruct" @@ -22,7 +21,7 @@ datasets: settings: runtime: - min_duration_ms: 600000 # 10 minutes, this is override when n_samples_to_issue is set + # Sample count: dataset issued once (the default) — replaces the old duration-derived count, which rounded up to one dataset pass. scheduler_random_seed: 42 # For Poisson/distribution sampling dataloader_random_seed: 42 # For dataset shuffling @@ -36,12 +35,11 @@ settings: recv_buffer_size: 16777216 send_buffer_size: 16777216 max_connections: 1000 - # Increase timeout for slow worker startup (spawn, imports). Default 40s may be too short. - worker_initialization_timeout: 120 - drain: - warmup_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) - performance_timeout_s: null # Performance drain timeout in seconds (None = wait indefinitely) - accuracy_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely) + timeouts: + # Increase for slow worker startup (spawn, imports). Default 60s may be too short. + worker_initialization_timeout_s: 120 + performance_drain_timeout_s: null # Performance drain timeout in seconds (null = wait indefinitely) + accuracy_drain_timeout_s: null # Accuracy drain timeout in seconds (null = wait indefinitely) warmup: enabled: true # Enable warmup phase before performance run n_requests: 1600 # Warmup request count (None = full dataset once) diff --git a/examples/08_Qwen3-VL-235B-A22B_Example/server_qwen3_vl_235b_a22b_shopify.yaml b/examples/08_Qwen3-VL-235B-A22B_Example/server_qwen3_vl_235b_a22b_shopify.yaml index e95d142d5..9b9110ae9 100644 --- a/examples/08_Qwen3-VL-235B-A22B_Example/server_qwen3_vl_235b_a22b_shopify.yaml +++ b/examples/08_Qwen3-VL-235B-A22B_Example/server_qwen3_vl_235b_a22b_shopify.yaml @@ -2,7 +2,6 @@ name: "online-qwen3-vl-235b-a22b-shopify-benchmark" version: "1.0" type: "online" -timeout: 14400 model_params: name: "Qwen/Qwen3-VL-235B-A22B-Instruct" @@ -22,7 +21,7 @@ datasets: settings: runtime: - min_duration_ms: 600000 # 10 minutes, this is override when n_samples_to_issue is set + # Sample count: dataset issued once (the default) — replaces the old duration-derived count, which rounded up to one dataset pass. scheduler_random_seed: 42 dataloader_random_seed: 42 @@ -37,8 +36,10 @@ settings: recv_buffer_size: 16777216 send_buffer_size: 16777216 max_connections: 1000 - # Increase timeout for slow worker startup (spawn, imports). Default 40s may be too short. - worker_initialization_timeout: 120 + + timeouts: + # Increase for slow worker startup (spawn, imports). Default 60s may be too short. + worker_initialization_timeout_s: 120 warmup: enabled: true # Enable warmup phase before performance run n_requests: 1600 # Warmup request count (None = full dataset once) diff --git a/examples/09_Wan22_VideoGen_Example/offline_wan22_submission.yaml b/examples/09_Wan22_VideoGen_Example/offline_wan22_submission.yaml index af50258b8..fd7ed1afa 100644 --- a/examples/09_Wan22_VideoGen_Example/offline_wan22_submission.yaml +++ b/examples/09_Wan22_VideoGen_Example/offline_wan22_submission.yaml @@ -50,9 +50,9 @@ audit: settings: runtime: - # NOTE: runs are count-driven (n_samples_to_issue / audit.samples). min_duration_ms is - # NOT enforced as a duration floor by the current stop logic (counts take priority); - # MLCommons' 10-min minimum / AND-semantics is future work. Only max_duration_ms caps. + # NOTE: runs are count-driven (n_samples_to_issue / audit.samples); there is no duration + # floor — MLCommons' 10-min minimum / AND-semantics is future work. max_duration_ms only + # caps the performance phase. max_duration_ms: 14400000 # 4-hour ceiling scheduler_random_seed: 42 dataloader_random_seed: 42 diff --git a/examples/09_Wan22_VideoGen_Example/single_stream_wan22_submission.yaml b/examples/09_Wan22_VideoGen_Example/single_stream_wan22_submission.yaml index bec6f8720..de4c534e5 100644 --- a/examples/09_Wan22_VideoGen_Example/single_stream_wan22_submission.yaml +++ b/examples/09_Wan22_VideoGen_Example/single_stream_wan22_submission.yaml @@ -50,9 +50,9 @@ audit: settings: runtime: - # NOTE: runs are count-driven (n_samples_to_issue / audit counts). min_duration_ms is - # NOT enforced as a duration floor by the current stop logic (counts take priority); - # MLCommons' 10-min minimum / AND-semantics is future work. Only max_duration_ms caps. + # NOTE: runs are count-driven (n_samples_to_issue / audit counts); there is no duration + # floor — MLCommons' 10-min minimum / AND-semantics is future work. max_duration_ms only + # caps the performance phase. max_duration_ms: 7200000 # 2-hour ceiling scheduler_random_seed: 42 dataloader_random_seed: 42 diff --git a/examples/10_Agentic_Inference/kimi_agentic_benchmark.yaml b/examples/10_Agentic_Inference/kimi_agentic_benchmark.yaml index d7e64c6d1..d16d69c6e 100644 --- a/examples/10_Agentic_Inference/kimi_agentic_benchmark.yaml +++ b/examples/10_Agentic_Inference/kimi_agentic_benchmark.yaml @@ -34,9 +34,6 @@ datasets: # workers: 8 # parallel agent workers; defaults to target_concurrency if unset. settings: - runtime: - min_duration_ms: 0 - load_pattern: type: agentic_inference target_concurrency: 8 # Submission-specific concurrency. diff --git a/examples/10_Agentic_Inference/qwen_agentic_benchmark.yaml b/examples/10_Agentic_Inference/qwen_agentic_benchmark.yaml index 562742c24..d183e4bfa 100644 --- a/examples/10_Agentic_Inference/qwen_agentic_benchmark.yaml +++ b/examples/10_Agentic_Inference/qwen_agentic_benchmark.yaml @@ -39,8 +39,7 @@ datasets: settings: runtime: - min_duration_ms: 0 - max_duration_ms: 36000000 + max_duration_ms: 36000000 # 10-hour cap on the performance phase load_pattern: type: agentic_inference diff --git a/examples/11_Edge_Agentic_Example/online_edge_full_run.yaml b/examples/11_Edge_Agentic_Example/online_edge_full_run.yaml index 60256d6af..b91ab2ac4 100644 --- a/examples/11_Edge_Agentic_Example/online_edge_full_run.yaml +++ b/examples/11_Edge_Agentic_Example/online_edge_full_run.yaml @@ -14,8 +14,7 @@ # Phases run perf -> accuracy (framework order). Both are deterministic # (temperature 0, seed 42) against a reasoning-off server, so order does not # affect results. Total wall-clock ~5.5 h on a single-stream edge box -# (e.g. NVIDIA Jetson AGX Thor, Qwen3.6-27B Q4_K_M, reasoning off); `timeout` -# below is sized to 6 h. +# (e.g. NVIDIA Jetson AGX Thor, Qwen3.6-27B Q4_K_M, reasoning off). # # Requires: pip install -e ".[bfcl]" (the BFCL accuracy dataset pulls bfcl-eval) # @@ -33,7 +32,6 @@ name: "edge-agentic-full-run" version: "1.0" type: "online" -timeout: 21600 # 6 h: ~2.5 h perf + ~3 h accuracy, with headroom. model_params: name: "Qwen3.6-27B-Q4_K_M" # set to your served model name. @@ -91,7 +89,6 @@ datasets: settings: runtime: - min_duration_ms: 0 # Safety cap (4 h) so the performance phase stays bounded even if decode is # slower than expected; one pass should finish in ~2.5 h on an edge box. max_duration_ms: 14400000 diff --git a/scripts/bench_drain_tokenize.py b/scripts/bench_drain_tokenize.py new file mode 100644 index 000000000..5a0ca00d0 --- /dev/null +++ b/scripts/bench_drain_tokenize.py @@ -0,0 +1,301 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Apples-to-apples benchmark of OUTPUT-tokenization strategies for the +metrics-aggregator drain (OSL / TPOT). + +Why this exists: at the end of a run the aggregator tokenizes every sample's +output to derive OSL/TPOT. The live impl fires one asyncio task per sample, +each awaiting ``loop.run_in_executor(thread_pool, len(tok.tokenize(text)))`` +(see ``metrics_aggregator/metrics_table.py::AsyncTokenTrigger.fire`` + +``token_metrics.py::TokenizePool.token_count_async``). This script reproduces +that exact pattern standalone and pits it against a single batched +``tokenizer(texts)`` call (the batched strategy from the prior ISL ablation) +so the cost of the current design — and the win from replacing it — is measured +on identical inputs. Measured (Qwen2.5-0.5B, 48-core, 12 workers): encode_batch +is ~4.6x the current per-sample async pattern on short outputs, ~2.0x on the +realistic right-skewed OSL distribution (mean ~3.8k tok) — and, more +importantly, removes the per-sample asyncio-task backlog (1 task/sample) that +drives the drain timeout. The single batched Rust call beats thread-sharding +(the HF fast tokenizer already parallelises a batch internally). + +Strategies (all plain ``tokenize``, no chat template — matches the OSL/TPOT +text path taken when the output has no tool_calls): + + current_async EXACT live drain pattern: per-sample loop.create_task -> + TokenizePool.token_count_async -> run_in_executor, gathered. + sync_loop Serial ``len(tok.tokenize(t))`` — isolates raw tokenize cost + from asyncio/thread-pool overhead. + batch One ``tokenizer(texts)`` Rust call over all texts. + thread_batch Shard texts across ``--workers`` threads, each batch-tokenizes + its shard (GIL released inside the Rust call). + +Usage: + uv run python scripts/bench_drain_tokenize.py \ + --model Qwen/Qwen2.5-0.5B-Instruct --n-samples 20000 --runs 3 +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import random +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from typing import Any + +from inference_endpoint.async_utils.services.metrics_aggregator.token_metrics import ( + TokenizePool, +) +from transformers import AutoTokenizer + +_WORDS = ( + "the quick brown fox jumps over the lazy dog inference benchmark " + "tokenization latency throughput performance model weights attention " + "transformer layer norm softmax gradient embedding sequence decode " +).split() + + +# Measured OSL token-length distribution (max_new_tokens=20000 cap; heavily +# right-skewed: median 2153, mean 3824). Piecewise-linear inverse-CDF from the +# measured percentiles so generated lengths match the real drain workload. +_OSL_PCTL: tuple[tuple[float, int], ...] = ( + (0, 177), + (1, 303), + (5, 463), + (10, 578), + (25, 951), + (50, 2153), + (75, 4977), + (80, 6001), + (90, 9564), + (95, 13510), + (97, 16422), + (99, 20000), + (100, 20000), +) + + +def _sample_osl(rng: random.Random) -> int: + p = rng.random() * 100.0 + for (p0, v0), (p1, v1) in zip(_OSL_PCTL, _OSL_PCTL[1:], strict=False): + if p <= p1: + frac = (p - p0) / (p1 - p0) if p1 > p0 else 0.0 + return int(v0 + frac * (v1 - v0)) + return _OSL_PCTL[-1][1] + + +def _make_outputs( + n: int, profile: str, min_words: int, max_words: int, seed: int = 42 +) -> list[str]: + """Synthetic model-output texts (plain text, the OSL/TPOT common case). + + profile='mlperf' draws word counts from the measured OSL distribution + (token≈word for these common words); 'uniform' uses [min_words, max_words]. + """ + rng = random.Random(seed) + if profile == "mlperf": + lengths = [_sample_osl(rng) for _ in range(n)] + else: + lengths = [rng.randint(min_words, max_words) for _ in range(n)] + return [" ".join(rng.choices(_WORDS, k=length)) for length in lengths] + + +def _result(name: str, secs: float, n: int, total_tokens: int) -> dict[str, Any]: + return { + "strategy": name, + "wall_s": round(secs, 4), + "samples_per_s": round(n / secs) if secs else 0, + "tokens_per_s": round(total_tokens / secs) if secs else 0, + } + + +def bench_sync_loop(texts: list[str], tok: Any) -> tuple[float, int]: + t0 = time.perf_counter() + total = 0 + for t in texts: + total += len(tok.tokenize(t)) + return time.perf_counter() - t0, total + + +def bench_batch(texts: list[str], tok: Any) -> tuple[float, int]: + t0 = time.perf_counter() + enc = tok(texts, add_special_tokens=False, return_attention_mask=False) + total = sum(len(ids) for ids in enc["input_ids"]) + return time.perf_counter() - t0, total + + +def bench_encode_batch(texts: list[str], tok: Any) -> tuple[float, int]: + """Raw Rust ``encode_batch`` on the backend tokenizer — skips the + BatchEncoding/padding wrapper that ``tokenizer(...)`` builds. We only need + counts, so this is the leanest count-only path.""" + backend = tok.backend_tokenizer + # encode_batch_fast (tokenizers>=0.20) skips offset computation; fall back + # to encode_batch where unavailable. + fn = getattr(backend, "encode_batch_fast", None) or backend.encode_batch + t0 = time.perf_counter() + encs = fn(texts, add_special_tokens=False) + total = sum(len(e.ids) for e in encs) + return time.perf_counter() - t0, total + + +def bench_batch_chunked( + texts: list[str], tok: Any, chunk: int = 50_000 +) -> tuple[float, int]: + """Chunked batches — bounds peak memory for very large drains while still + feeding the Rust parallel path large slices.""" + t0 = time.perf_counter() + total = 0 + for i in range(0, len(texts), chunk): + enc = tok( + texts[i : i + chunk], + add_special_tokens=False, + return_attention_mask=False, + ) + total += sum(len(ids) for ids in enc["input_ids"]) + return time.perf_counter() - t0, total + + +def bench_thread_batch( + texts: list[str], tokenizer_name: str, workers: int +) -> tuple[float, int]: + # Each worker loads its own tokenizer (thread-local, like TokenizePool) and + # batch-tokenizes a contiguous shard. + shards: list[list[str]] = [texts[i::workers] for i in range(workers)] + tls = threading.local() + + def _work_tls(shard: list[str]) -> int: + tok = getattr(tls, "tok", None) + if tok is None: + tok = AutoTokenizer.from_pretrained(tokenizer_name) + tls.tok = tok + if not shard: + return 0 + enc = tok(shard, add_special_tokens=False, return_attention_mask=False) + return sum(len(ids) for ids in enc["input_ids"]) + + with ThreadPoolExecutor(max_workers=workers) as ex: + # Warm tokenizers on every thread before timing. + list(ex.map(lambda _: _work_tls([]), range(workers))) + t0 = time.perf_counter() + total = sum(ex.map(_work_tls, shards)) + return time.perf_counter() - t0, total + + +async def bench_current_async( + texts: list[str], pool: TokenizePool +) -> tuple[float, int]: + """EXACT live drain pattern: one asyncio task per sample, each awaiting + pool.token_count_async (-> loop.run_in_executor), then gathered.""" + loop = asyncio.get_running_loop() + t0 = time.perf_counter() + tasks = [loop.create_task(pool.token_count_async(t, loop)) for t in texts] + counts = await asyncio.gather(*tasks) + return time.perf_counter() - t0, sum(counts) + + +def _run_current_async(texts: list[str], pool: TokenizePool) -> tuple[float, int]: + try: + import uvloop # the aggregator runs on uvloop; match it. + + runner = uvloop.run + except ImportError: + runner = asyncio.run + return runner(bench_current_async(texts, pool)) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--model", default="Qwen/Qwen2.5-0.5B-Instruct") + ap.add_argument("--n-samples", type=int, default=20000) + ap.add_argument("--runs", type=int, default=3) + ap.add_argument( + "--workers", + type=int, + default=max(2, (os.cpu_count() or 16) // 4), + help="TokenizePool / thread_batch worker count (aggregator default).", + ) + ap.add_argument("--osl-profile", choices=("mlperf", "uniform"), default="mlperf") + ap.add_argument("--min-words", type=int, default=20) + ap.add_argument("--max-words", type=int, default=200) + ap.add_argument("--output", default="") + args = ap.parse_args() + + print(f"Loading tokenizer: {args.model}") + AutoTokenizer.from_pretrained(args.model) # warm cache before timing + tok = AutoTokenizer.from_pretrained(args.model) + + print( + f"Generating {args.n_samples} synthetic outputs (profile={args.osl_profile})..." + ) + texts = _make_outputs( + args.n_samples, args.osl_profile, args.min_words, args.max_words + ) + avg_words = sum(t.count(" ") + 1 for t in texts) / len(texts) + print( + f"profile={args.osl_profile} | avg {avg_words:.0f} words/output " + f"| workers={args.workers}\n" + ) + + pool = TokenizePool(args.model, n_workers=args.workers) + results: list[dict[str, Any]] = [] + try: + strategies = [ + ("current_async", lambda: _run_current_async(texts, pool)), + ("sync_loop", lambda: bench_sync_loop(texts, tok)), + ("batch", lambda: bench_batch(texts, tok)), + ("batch_chunked", lambda: bench_batch_chunked(texts, tok)), + ("encode_batch", lambda: bench_encode_batch(texts, tok)), + ( + "thread_batch", + lambda: bench_thread_batch(texts, args.model, args.workers), + ), + ] + for name, fn in strategies: + best_secs = float("inf") + total_tokens = 0 + for _ in range(args.runs): + secs, total_tokens = fn() + best_secs = min(best_secs, secs) + r = _result(name, best_secs, args.n_samples, total_tokens) + results.append(r) + print( + f"{name:<16} {r['wall_s']:>9.4f}s " + f"{r['samples_per_s']:>12,} samples/s " + f"{r['tokens_per_s']:>14,} tok/s" + ) + finally: + pool.close() + + base = next(r for r in results if r["strategy"] == "current_async") + print("\nspeedup vs current_async (best wall):") + for r in results: + if r["strategy"] != "current_async" and r["samples_per_s"]: + print( + f" {r['strategy']:<16} {r['samples_per_s'] / base['samples_per_s']:>6.1f}x" + ) + + if args.output: + with open(args.output, "w") as f: + json.dump({"args": vars(args), "results": results}, f, indent=2) + print(f"\nwrote {args.output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/regenerate_templates.py b/scripts/regenerate_templates.py index 5d72668fe..634fcb671 100644 --- a/scripts/regenerate_templates.py +++ b/scripts/regenerate_templates.py @@ -371,8 +371,6 @@ def _build_minimal(test_type: TestType, overrides: dict) -> dict: "datasets": [PERF_DATASET], "settings": { "runtime": { - "min_duration_ms": 600000, - "max_duration_ms": 0, "n_samples_to_issue": None, }, }, diff --git a/src/inference_endpoint/async_utils/services/launcher.py b/src/inference_endpoint/async_utils/services/launcher.py index f1e1dac15..8cce48666 100644 --- a/src/inference_endpoint/async_utils/services/launcher.py +++ b/src/inference_endpoint/async_utils/services/launcher.py @@ -69,6 +69,7 @@ class ServiceLauncher: def __init__(self, zmq_context: ManagedZMQContext) -> None: self._zmq_ctx = zmq_context self._procs: list[subprocess.Popen] = [] + self._modules: list[str] = [] @property def procs(self) -> list[subprocess.Popen]: @@ -118,6 +119,7 @@ async def launch( logger.info("Launching service: %s (id=%d)", svc.module, i) proc = subprocess.Popen(cmd) self._procs.append(proc) + self._modules.append(svc.module) await receiver.wait(timeout=timeout) logger.info("All %d services ready", len(services)) @@ -145,6 +147,18 @@ async def launch( # re-raise the exception. raise + def terminate(self, module: str) -> None: + """SIGTERM managed subprocesses whose module exactly matches ``module``. + + Targeted so the whole-run watchdog can abort the metrics aggregator + (whose SIGTERM handler writes an INTERRUPTED final snapshot) without + killing the event logger, which flushes its buffer on the session's + ENDED event and would lose buffered records on SIGTERM. + """ + for launched_module, proc in zip(self._modules, self._procs, strict=True): + if launched_module == module and proc.poll() is None: + proc.terminate() + def terminate_all(self, timeout: float = 5.0) -> None: """Terminate all managed subprocesses: SIGTERM then escalate to SIGKILL. diff --git a/src/inference_endpoint/async_utils/services/metrics_aggregator/publisher.py b/src/inference_endpoint/async_utils/services/metrics_aggregator/publisher.py index 578e47198..df4cd673c 100644 --- a/src/inference_endpoint/async_utils/services/metrics_aggregator/publisher.py +++ b/src/inference_endpoint/async_utils/services/metrics_aggregator/publisher.py @@ -88,10 +88,15 @@ def __init__( self._final_snapshot_path = final_snapshot_path self._tick_task: asyncio.Task | None = None self._closed = False - # publish_final is idempotent: the SIGTERM handler in - # __main__.py and the aggregator's ENDED-driven path can both - # call it; the second call must not re-publish or re-write. + # publish_final is idempotent AND serialized: the SIGTERM handler in + # __main__.py and the aggregator's ENDED-driven path can both call + # it. The lock makes a raced second caller block until the in-flight + # finalize (including the atomic file write) completes before + # early-returning, so the SIGTERM path's shutdown_event.set() can + # never let main() return while the write is still in flight + # (which would abandon a .tmp and leave no final_snapshot.json). self._finalized = False + self._final_lock = asyncio.Lock() # ------------------------------------------------------------------ # Live tick task @@ -189,13 +194,28 @@ async def publish_final( (which would let a conflate-mode TUI see the live tick instead of the terminal state as the last message). - Idempotent: only the first call writes/publishes; subsequent - calls early-return. The SIGTERM handler relies on this to - race safely with the ENDED-driven path. + Idempotent and serialized: only the first call writes/publishes; + a concurrent second call blocks until the first finishes, then + early-returns. The SIGTERM handler relies on this to race safely + with the ENDED-driven path — its ``shutdown_event.set()`` cannot + run before an in-flight finalize write has completed. """ - if self._finalized: - return - self._finalized = True + async with self._final_lock: + if self._finalized: + return + self._finalized = True + await self._publish_final_locked( + registry, n_pending_tasks=n_pending_tasks, interrupted=interrupted + ) + + async def _publish_final_locked( + self, + registry: MetricsRegistry, + *, + n_pending_tasks: int, + interrupted: bool, + ) -> None: + """Finalize body; runs exactly once, under ``_final_lock``.""" if self._tick_task is not None: self._tick_task.cancel() try: diff --git a/src/inference_endpoint/commands/audit.py b/src/inference_endpoint/commands/audit.py index 1b8eddca1..9df903e26 100644 --- a/src/inference_endpoint/commands/audit.py +++ b/src/inference_endpoint/commands/audit.py @@ -129,6 +129,13 @@ def run_audit(config: BenchmarkConfig, base_report_dir: Path) -> AuditResult: report = bench.report if report is None: raise ExecutionError(f"Audit phase '{spec.label}' produced no report") + # A timed-out phase produced an INTERRUPTED report at best; certifying + # a compliance result from it is never valid. + if bench.run_timed_out: + raise ExecutionError( + f"Audit phase '{spec.label}' hit the run timeout " + "(settings.timeouts.run_timeout_s); report marked INTERRUPTED" + ) # A SIGINT/SIGTERM during a (long) audit phase is turned into a graceful # stop, so the phase returns with an "interrupted" report. Propagate it # as KeyboardInterrupt so the CLI exits 130 (interrupted), not as a diff --git a/src/inference_endpoint/commands/benchmark/cli.py b/src/inference_endpoint/commands/benchmark/cli.py index 0edb4da79..68aaa0f1e 100644 --- a/src/inference_endpoint/commands/benchmark/cli.py +++ b/src/inference_endpoint/commands/benchmark/cli.py @@ -174,7 +174,15 @@ def from_config( except (yaml.YAMLError, ValidationError, ValueError, FileNotFoundError) as e: raise InputValidationError(f"Config error: {e}") from e if timeout is not None: - resolved = resolved.with_updates(timeout=timeout) + resolved = resolved.with_updates( + settings=resolved.settings.model_copy( + update={ + "timeouts": resolved.settings.timeouts.with_updates( + run_timeout_s=timeout + ) + } + ) + ) if report_dir is not None: resolved = resolved.with_updates(report_dir=report_dir) test_mode = mode or ( diff --git a/src/inference_endpoint/commands/benchmark/execute.py b/src/inference_endpoint/commands/benchmark/execute.py index 6cc4213d4..239159584 100644 --- a/src/inference_endpoint/commands/benchmark/execute.py +++ b/src/inference_endpoint/commands/benchmark/execute.py @@ -34,6 +34,7 @@ import shutil import signal import tempfile +import time import uuid from collections.abc import Callable from dataclasses import dataclass, field @@ -162,6 +163,7 @@ class BenchmarkResult: # settings.profiling.engine is set; None otherwise. Rendered into # report.txt and a sibling profiling.json by finalize_benchmark. profiling: dict[str, Any] | None = None + run_timed_out: bool = False @dataclass @@ -533,9 +535,7 @@ def setup_benchmark( f"Mode: {test_mode}, Target QPS: {config.settings.load_pattern.target_qps}, Responses: {collect_responses}" ) if rt_settings is not None: - logger.info( - f"Min Duration: {rt_settings.min_duration_ms / 1000:.1f}s, Expected samples: {total_samples}" - ) + logger.info(f"Expected samples: {total_samples}") else: logger.info(f"Accuracy-only mode, Expected samples: {total_samples}") for ec in eval_configs: @@ -569,7 +569,7 @@ def _build_phases( ) -> list[PhaseConfig]: """Build the phase list from BenchmarkContext.""" phases: list[PhaseConfig] = [] - drain_cfg = ctx.config.settings.drain + timeouts = ctx.config.settings.timeouts if ctx.dataloader is not None and ctx.rt_settings is not None: perf_dataset = next( @@ -609,7 +609,7 @@ def _build_phases( warmup_dataset, PhaseType.WARMUP, drain_after=warmup_cfg.drain, - drain_timeout=drain_cfg.warmup_timeout_s, + drain_timeout=timeouts.warmup_drain_timeout_s, ) ) @@ -620,7 +620,7 @@ def _build_phases( ctx.dataloader, PhaseType.PERFORMANCE, strategy=perf_strategy, - drain_timeout=drain_cfg.performance_timeout_s, + drain_timeout=timeouts.performance_drain_timeout_s, routing_headers=routing_headers, ) ) @@ -675,7 +675,7 @@ def _build_phases( acc_settings, acc_ds, PhaseType.ACCURACY, - drain_timeout=drain_cfg.accuracy_timeout_s, + drain_timeout=timeouts.accuracy_drain_timeout_s, ) ) @@ -727,6 +727,7 @@ async def _create_issuer( api_type: APIType = config.endpoint_config.api_type # client.api_type is propagated from endpoint_config.api_type by # BenchmarkConfig._propagate_client_api_type — no override needed here. + timeouts = config.settings.timeouts client_overrides: dict = { "endpoint_urls": [ urljoin(e.rstrip("/") + "/", api_type.default_route()) @@ -735,6 +736,12 @@ async def _create_issuer( "api_key": config.endpoint_config.api_key, "event_logs_dir": ctx.report_dir, "cpu_affinity": ctx.affinity_plan, + # Worker lifecycle deadlines live in settings.timeouts; the + # HTTPClientConfig fields are excluded runtime carriers populated + # only here. + "worker_initialization_timeout_s": timeouts.worker_initialization_timeout_s, + "worker_graceful_shutdown_wait_s": timeouts.worker_graceful_shutdown_wait_s, + "worker_force_kill_timeout_s": timeouts.worker_force_kill_timeout_s, } if ctx.accuracy_only: # Single-stream (num_workers=1, max_connections=1) is baked into @@ -805,6 +812,8 @@ def _on_sample_complete(result: QueryResult) -> None: async def _run_benchmark_async( ctx: BenchmarkContext, loop: asyncio.AbstractEventLoop, + *, + deadline: float | None = None, ) -> BenchmarkResult: """Run async benchmark session.""" config = ctx.config @@ -846,6 +855,45 @@ async def _run_benchmark_async( # idempotent, so the clean-path shutdown below is a harmless second call. http_client: HTTPEndpointClient | None = None + # Whole-run watchdog. Armed before the pipeline starts so setup stalls + # (service launch, endpoint connect) are bounded too, and kept armed + # through the metrics drain so run_timeout_s can SIGTERM a stuck + # aggregator drain. Cancelled in the outermost finally. + run_timed_out = False + # The session is created later inside the pipeline scope; bind it through + # a mutable holder so the callback never touches a possibly-unbound local + # (a NameError inside a loop callback is swallowed by the loop's exception + # handler, which would leave the watchdog inert). + session_ref: list[BenchmarkSession] = [] + run_timeout_s = config.settings.timeouts.run_timeout_s + + def _on_run_timeout() -> None: + nonlocal run_timed_out + run_timed_out = True + logger.error( + "Run timeout (%.1fs) reached; aborting run — report will be " + "marked INTERRUPTED.", + run_timeout_s, + ) + # Stop the session first: it short-circuits _drain_inflight and + # run()'s finally publishes ENDED promptly, so the aggregator still + # records the buffered tokenizer-drain samples. Then SIGTERM the + # aggregator: its handler writes the INTERRUPTED final snapshot + # (publish_final is first-wins, so INTERRUPTED stays authoritative; + # even if a still-draining aggregator finalizes as COMPLETE first, + # run_benchmark raises on run_timed_out, so a timed-out run always + # fails loudly). Targeted (not all services): the event logger + # flushes on ENDED, which session.stop() still delivers. + if session_ref: + session_ref[0].stop() + pipe.terminate_metrics_aggregator() + + run_watchdog = ( + loop.call_later(max(0.0, deadline - time.monotonic()), _on_run_timeout) + if deadline is not None + else None + ) + try: tmpfs_dir.mkdir(parents=True, exist_ok=True) event_log_dir.mkdir(parents=True, exist_ok=True) @@ -882,6 +930,7 @@ async def _run_benchmark_async( on_sample_complete=on_sample_complete, session_id=session_id, ) + session_ref.append(session) phases = _build_phases(ctx, perf_strategy=agentic_inference_strategy) max_duration_ms = ( @@ -921,18 +970,48 @@ def _on_phase_start(phase: PhaseConfig) -> None: loop.add_signal_handler(signal.SIGINT, session.stop) try: - result = await session.run(phases, on_phase_start=_on_phase_start) - session_completed_normally = True + if run_timed_out: + # Deadline elapsed during setup — never start issuing + # load after it. Run the already-stopped session so + # STARTED/ENDED still flow: the event logger exits only + # on ENDED, and the drain below waits for it. Zero + # samples issue; the INTERRUPTED artifacts still get + # written. + session.stop() + result = await session.run(phases) + else: + result = await session.run( + phases, on_phase_start=_on_phase_start + ) + session_completed_normally = True except Exception as e: - raise ExecutionError(f"Benchmark execution failed: {e}") from e + if run_timed_out: + # The watchdog already aborted the run; a teardown race + # can surface here as a generic exception. Fall through + # with an empty session result so finalize still writes + # the INTERRUPTED report artifacts — run_benchmark + # raises the timeout ExecutionError after finalization. + logger.exception( + "Session error after run timeout fired " + "(continuing to finalize)" + ) + result = SessionResult( + session_id=session_id, + phase_results=[], + start_time_ns=0, + end_time_ns=0, + ) + else: + raise ExecutionError(f"Benchmark execution failed: {e}") from e finally: _timeout_done = True perf_timeout.cancel() loop.remove_signal_handler(signal.SIGINT) # Fire /stop_profile for URLs whose /start_profile succeeded. # Unifies the clean phase-end path and the abort path — both - # reach this block. - profiler.stop(session_completed_normally) + # reach this block. A watchdog abort counts as an abort even + # when session.run returned normally after session.stop(). + profiler.stop(session_completed_normally and not run_timed_out) # Graceful drain runs on both the clean-finish and session- # failure paths (BenchmarkSession.run publishes ENDED in its own # finally, so a failed run still has a terminal snapshot worth @@ -985,6 +1064,9 @@ def _on_phase_start(phase: PhaseConfig) -> None: "Failed to salvage tmpfs: %s — tmpfs retained at %s", e, tmpfs_dir ) raise + finally: + if run_watchdog is not None: + run_watchdog.cancel() return BenchmarkResult( session=result, @@ -992,13 +1074,25 @@ def _on_phase_start(phase: PhaseConfig) -> None: report=report, tmpfs_dir=tmpfs_dir, profiling=profiler.payload(), + run_timed_out=run_timed_out, ) -def run_benchmark_async(ctx: BenchmarkContext) -> BenchmarkResult: - """Run async benchmark. Sync entry point — drives the event loop.""" +def run_benchmark_async( + ctx: BenchmarkContext, *, deadline: float | None = None +) -> BenchmarkResult: + """Run async benchmark. Sync entry point — drives the event loop. + + When ``deadline`` is None and ``settings.timeouts.run_timeout_s`` is set, + computes its own deadline at entry, so each audit phase gets a full + per-phase budget. + """ + if deadline is None: + run_timeout_s = ctx.config.settings.timeouts.run_timeout_s + if run_timeout_s is not None: + deadline = time.monotonic() + run_timeout_s loop = LoopManager().default_loop - return loop.run_until_complete(_run_benchmark_async(ctx, loop)) + return loop.run_until_complete(_run_benchmark_async(ctx, loop, deadline=deadline)) def _write_scoring_artifacts( @@ -1116,6 +1210,11 @@ def finalize_benchmark(ctx: BenchmarkContext, bench: BenchmarkResult) -> None: result = bench.session collector = bench.collector report = bench.report + if report is not None and bench.run_timed_out and report.complete: + # Split-brain guard: the aggregator may have finalized COMPLETE before + # the watchdog's SIGTERM landed. A timed-out run must never publish + # complete:true artifacts, so force the flag honest before writing. + report = msgspec.structs.replace(report, complete=False) # Write scoring artifacts + copy event log from tmpfs to disk (scorers read # sample_idx_map.json + events.jsonl from here). @@ -1129,7 +1228,16 @@ def finalize_benchmark(ctx: BenchmarkContext, bench: BenchmarkResult) -> None: # then the exception propagates as before. accuracy_scores: list[dict[str, Any]] = [] try: - accuracy_scores = score_accuracy(ctx, result) + if bench.run_timed_out: + # Phases may never have started (scorer init KeyErrors on missing + # sample maps) and partial phases would yield misleading subset + # scores; the scoring artifacts above are still on disk for + # inspection. + logger.warning( + "Run timeout fired — skipping accuracy scoring on partial data" + ) + else: + accuracy_scores = score_accuracy(ctx, result) finally: # Attach the per-dataset accuracy list so result_summary.json, the # console summary, and report.txt all carry it (stays [] on a scoring @@ -1173,17 +1281,39 @@ def run_benchmark( a config with an ``audit:`` block, point ``run_audit`` at ``/audit``). The compliance audit is dispatched by the caller (``cli._run``), not here, so this module does not depend on ``commands.audit``. + + The whole-run watchdog deadline is taken at entry, so setup (tokenizer/ + dataset load) counts against ``run_timeout_s``; a hung *synchronous* setup + step itself stays unbounded — the deadline is only checked once setup + returns, and enforced by the event-loop timer thereafter. """ logger.debug( "BenchmarkConfig (%s):\n%s", type(config).__name__, config.model_dump_json(indent=2, exclude_none=True), ) + # Deadline for the whole-run watchdog is taken at entry so setup + # (tokenizer/dataset load) counts against run_timeout_s too. + deadline: float | None = None + run_timeout_s = config.settings.timeouts.run_timeout_s + if run_timeout_s is not None: + deadline = time.monotonic() + run_timeout_s ctx = setup_benchmark(config, test_mode) + if deadline is not None and time.monotonic() >= deadline: + # Setup alone consumed the budget: fail before any services start. + raise ExecutionError( + f"Run timeout ({run_timeout_s}s) reached during setup; " + "no services were started" + ) bench: BenchmarkResult | None = None try: - bench = run_benchmark_async(ctx) + bench = run_benchmark_async(ctx, deadline=deadline) finalize_benchmark(ctx, bench) + if bench.run_timed_out: + raise ExecutionError( + f"Run timeout ({run_timeout_s}s) reached; run aborted and " + "report marked INTERRUPTED" + ) except KeyboardInterrupt: # Salvage results (finally), then propagate to main.py -> exit 130. logger.warning("Benchmark interrupted by user") diff --git a/src/inference_endpoint/commands/benchmark/pipeline.py b/src/inference_endpoint/commands/benchmark/pipeline.py index 7778ac416..bd056ac48 100644 --- a/src/inference_endpoint/commands/benchmark/pipeline.py +++ b/src/inference_endpoint/commands/benchmark/pipeline.py @@ -100,7 +100,7 @@ def _build_aggregator_args( metrics_output_dir: Path, enable_streaming: bool, tokenizer_name: str | None, - drain_timeout_s: float, + drain_timeout_s: float | None, tokenizer_workers: int, early_stopping: bool, ) -> list[str]: @@ -121,7 +121,11 @@ def _build_aggregator_args( args.append("--early-stopping") if tokenizer_name is not None: args.extend(["--tokenizer", tokenizer_name]) - args.extend(["--drain-timeout", str(drain_timeout_s)]) + # Aggregator argv contract keeps 0 = unlimited (hand-launch default); + # the schema uses None = unlimited, so convert at the argv boundary. + args.extend( + ["--drain-timeout", "0" if drain_timeout_s is None else str(drain_timeout_s)] + ) args.extend(["--tokenizer-workers", str(tokenizer_workers)]) return args @@ -280,7 +284,7 @@ async def start(self) -> None: stack.callback(self._close_subscriber) self._launcher = ServiceLauncher(zmq_ctx) - drain = self._config.settings.drain + timeouts = self._config.settings.timeouts aggregator_args = _build_aggregator_args( socket_dir=zmq_ctx.socket_dir, pub_socket_name=pub_socket_name, @@ -288,8 +292,8 @@ async def start(self) -> None: metrics_output_dir=self._metrics_output_dir, enable_streaming=self._enable_streaming, tokenizer_name=self._tokenizer_name, - drain_timeout_s=drain.metrics_drain_timeout_s, - tokenizer_workers=drain.metrics_tokenizer_workers, + drain_timeout_s=timeouts.metrics_drain_timeout_s, + tokenizer_workers=self._config.settings.metrics_tokenizer_workers, early_stopping=self._config.settings.early_stopping.enabled, ) event_logger_args = _build_event_logger_args( @@ -302,7 +306,7 @@ async def start(self) -> None: ServiceConfig(module=_AGGREGATOR_MODULE, args=aggregator_args), ServiceConfig(module=_EVENT_LOGGER_MODULE, args=event_logger_args), ], - timeout=self._config.settings.service_ready_timeout_s, + timeout=timeouts.service_ready_timeout_s, ) except BaseException as e: if self._launcher is not None: # launch may have spawned children @@ -360,6 +364,17 @@ async def drain_and_build_report(self) -> Report | None: ) return report + def terminate_metrics_aggregator(self) -> None: + """SIGTERM the metrics aggregator; safe no-op before launch. + + Run-watchdog abort path: targeted so the aggregator's SIGTERM handler + writes the INTERRUPTED final snapshot while the event logger stays + alive to flush its buffer on the session's ENDED event. + """ + if self._launcher is None: + return + self._launcher.terminate(_AGGREGATOR_MODULE) + def _kill_services(self) -> None: """Best-effort service termination owned by the pipeline ExitStack. diff --git a/src/inference_endpoint/config/audit.py b/src/inference_endpoint/config/audit.py new file mode 100644 index 000000000..0af28d38a --- /dev/null +++ b/src/inference_endpoint/config/audit.py @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compliance audit configuration. + +Split criterion: one module per config domain; the audit test registry and its +per-test config models live here. ``config/schema.py`` re-exports the public +surface. +""" + +from __future__ import annotations + +from enum import Enum +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + + +class AuditTestId(str, Enum): + """Registered compliance audit test identifiers.""" + + # Output-caching audit — MLPerf TEST04 (duplicate-query caching detection). + OUTPUT_CACHING_TEST = "output_caching_test" + + +class OutputCachingTestConfig(BaseModel): + """Configuration for the output-caching audit (MLPerf TEST04). + + The output-caching test runs two back-to-back phases — a reference run of + distinct samples and an audit run that repeats one fixed sample — then + checks that the audit QPS does not exceed the reference QPS by more than + ``threshold``. A large speedup indicates the SUT is caching responses. + + samples: reference-phase query count (required — an explicit count keeps + the per-phase completion check meaningful; a duration-driven phase has + no independent target to validate completion against) + audit_samples: audit-phase query count (None → equals samples) + sample_index: which dataset row is repeated (MLCommons performance_issue_same_index) + threshold: tolerance shared by both pass checks — each phase must complete + ≥ requested * (1 - threshold), and audit_qps must stay < ref_qps * (1 + threshold) + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + test: Literal[AuditTestId.OUTPUT_CACHING_TEST] + only: bool = Field( + False, + description="Run only the audit — skip the main benchmark (upstream-style standalone TEST04)", + ) + samples: int = Field(..., ge=1, description="Reference phase query count") + audit_samples: int | None = Field( + None, ge=1, description="Audit phase query count (default: equals samples)" + ) + sample_index: int = Field( + 0, ge=0, description="Dataset row index repeated in the audit phase" + ) + threshold: float = Field( + 0.10, + gt=0, + lt=1, + description=( + "Tolerance for both checks: each phase must complete " + "≥ requested * (1 - threshold), and audit_qps must stay " + "< ref_qps * (1 + threshold)" + ), + ) + + +# Single member today; becomes +# Annotated[OutputCachingTestConfig | ..., Field(discriminator="test")] +# when additional audit tests are added. +AuditConfig = OutputCachingTestConfig diff --git a/src/inference_endpoint/config/datasets.py b/src/inference_endpoint/config/datasets.py new file mode 100644 index 000000000..06fcd4d0a --- /dev/null +++ b/src/inference_endpoint/config/datasets.py @@ -0,0 +1,284 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Dataset configuration models. + +Split criterion: one module per config domain; the dataset models and their +generation-config-override merge helpers live here (the override keys' only +consumer is ``Dataset``, so they stay together). ``config/schema.py`` +re-exports the public surface. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Annotated, Any, Self + +import cyclopts +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from .enums import DatasetType, EvalMethod, ScorerMethod +from .model_params import ModelParams + + +def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]: + """Recursively merge ``override`` into ``base`` and return the result. + + For overlapping keys whose values are both dicts, recurse; otherwise the + override value wins. Mutates a *copy* — callers can safely pass model_dump() + output. Used by ``Dataset.effective_generation_config`` so a sparse nested + override (e.g. ``{osl_distribution: {max: 512}}``) preserves siblings. + """ + out = dict(base) + for k, v in override.items(): + if isinstance(v, dict) and isinstance(out.get(k), dict): + out[k] = _deep_merge(out[k], v) + else: + out[k] = v + return out + + +# ModelParams fields that drive the single global tokenizer / MetricsAggregator +# (launched once from top-level model_params), so a per-dataset override would +# desync ISL/OSL/TTFT/TPOT accounting without changing what is measured. Rejected +# as generation_config_override keys — they are per-run/identity, not per-dataset. +_METRICS_DECOUPLED_OVERRIDE_KEYS = frozenset({"name", "streaming", "tokenizer_name"}) + + +class AgenticInferenceConfig(BaseModel): + """Agentic inference conversation configuration. + + Configuration for benchmarking conversational AI workloads with turn sequencing. + Enables testing agentic inference conversations where each turn depends on previous responses. + Presence of this block in the dataset config enables agentic inference mode. + + Attributes: + turn_timeout_s: Deadline between issuing a turn and receiving its + response. A timeout aborts that turn and all remaining client + turns of the same conversation because subsequent turns depend + on the timed-out response. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + turn_timeout_s: float = Field( + default=86400.0, + gt=0, + description=( + "Per-turn timeout in seconds. A timeout aborts that turn and all " + "remaining turns in the same conversation." + ), + ) + enable_salt: bool = Field( + False, + description=( + "Add deterministic salt markers before and after the system prompt " + "to prevent KV cache reuse across trajectories in agentic inference setting." + ), + ) + inject_tool_delay: bool = Field( + False, + description=( + "Pause for a predefined duration between turns. Duration is defined " + "in dataset." + ), + ) + routing_headers: tuple[str, ...] = Field( + default=("X-Session-ID",), + description=( + "HTTP header names populated with the conversation ID on every " + "agentic request." + ), + ) + num_trajectories_to_issue: int | None = Field( + default=None, + gt=0, + description=( + "Number of conversation trajectories to start. Defaults to one pass " + "over the dataset; values above the dataset size repeat trajectories " + "with unique logical conversation ids." + ), + ) + stop_issuing_on_first_user_complete: bool = Field( + False, + description=( + "When performance tracking stops because the first concurrency slot " + "has no next trajectory left to assign, also stop issuing future " + "turns. If false, replay continues outside the performance window " + "for accuracy/log coverage." + ), + ) + + +class AccuracyConfig(BaseModel): + """Accuracy configuration. + + eval_method: Scorer to use (see ScorerMethod enum for options). + ground_truth: Column in the dataset containing ground truth. Defaults to "ground_truth". + extractor: Post-processor to extract answers from model output + (abcd_extractor, boxed_math_extractor, identity_extractor, python_code_extractor). + Optional for scorers that declare REQUIRES_EXTRACTOR = False (e.g. vbench). + num_repeats: Number of times to repeat the dataset for evaluation. Defaults to 1. + extras: Free-form keyword args forwarded to the scorer's ``__init__`` — + used for scorer-specific knobs that don't warrant a top-level field + (e.g. ``vbench_project_path``, ``subprocess_timeout_s`` for VBench). + + Example: + accuracy_config: + eval_method: "pass_at_1" + ground_truth: "answer" + extractor: "boxed_math_extractor" + num_repeats: 5 + extras: + vbench_project_path: "/path/to/accuracy" + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + eval_method: ScorerMethod | None = Field(None, description="Scorer method") + ground_truth: str | None = Field(None, description="Ground truth column name") + extractor: str | None = Field( + None, + description="Answer extractor (abcd_extractor, boxed_math_extractor, identity_extractor, python_code_extractor)", + ) + num_repeats: int = Field( + 1, ge=1, description="Repeat dataset N times for evaluation" + ) + extras: dict[str, Any] | None = Field( + None, + description="Free-form scorer kwargs (e.g. vbench_project_path, subprocess_timeout_s)", + ) + + +class Dataset(BaseModel): + """Dataset configuration. + + Name and type have smart defaults: name is auto-derived from path, + type defaults to PERFORMANCE. + + Accepts CLI strings via BeforeValidator on BenchmarkConfig.datasets: + ``[perf|acc:][,key=value...]`` + """ + + model_config = ConfigDict(extra="forbid", frozen=True, str_strip_whitespace=True) + + name: str = Field("", description="Dataset name (auto-derived from path if empty)") + type: DatasetType = Field( + DatasetType.PERFORMANCE, description="Dataset purpose: performance or accuracy" + ) + path: Annotated[ + str | None, cyclopts.Parameter(alias="--dataset", help="Dataset file path") + ] = None + format: str | None = Field(None, description="Dataset format (auto-detected)") + samples: int | None = Field(None, gt=0, description="Number of samples to use") + eval_method: EvalMethod | None = Field( + None, description="Accuracy evaluation method" + ) + parser: dict[str, str] | None = Field( + None, description="Column remapping: {prompt: , system: }" + ) + generate_params: dict[str, Any] | None = Field( + None, description="Dataset-specific parameters passed to the generate() method" + ) + accuracy_config: AccuracyConfig | None = Field( + None, description="Accuracy evaluation settings" + ) + agentic_inference: AgenticInferenceConfig | None = Field( + None, description="Agentic inference conversation configuration" + ) + # Per-dataset generation config is a first-class capability: different + # accuracy datasets legitimately want different generation settings (e.g. + # per-dataset max OSL or top_p, as seen in DS-V4), and dataset-scoping also + # enables per-dataset dynamic OSL distributions. Only generation knobs are + # overridable — per-run/identity fields (`_METRICS_DECOUPLED_OVERRIDE_KEYS`: + # name / streaming / tokenizer_name) drive the single global tokenizer and + # MetricsAggregator, so overriding them per-dataset would desync ISL/OSL/ + # TTFT/TPOT accounting; they are rejected at validation. + # + # TODO(post-mortem): split ModelParams into a per-run ModelIdentity and a + # GenerationConfig, so the override surface is exactly the generation fields + # and identity fields cannot be named here at all. Field/method names use + # "generation_config" to keep that migration mechanical. + # + # Nested dicts (`osl_distribution`, `chat_template_kwargs`) are deep-merged + # so sparse overrides preserve sibling defaults. + generation_config_override: dict[str, Any] | None = Field( + None, + description=( + "Per-dataset overrides for the top-level model_params (sparse — " + "only the fields you want to override). Merged on top of " + "BenchmarkConfig.model_params at dataset-load time. Useful for " + "MLPerf-style runs where accuracy and performance use different " + "output budgets in the same fleet, e.g. " + "generation_config_override: {max_new_tokens: 32768, " + "temperature: 0.0}. NOTE: per-run/identity keys (`name`, " + "`streaming`, `tokenizer_name`) are rejected here — set them on " + "top-level model_params." + ), + ) + + @model_validator(mode="after") + def _auto_derive_name(self) -> Self: + """Derive name from path stem if not explicitly provided.""" + if not self.name and self.path: + object.__setattr__(self, "name", Path(self.path).stem) + return self + + @model_validator(mode="after") + def _validate_generation_config_override(self) -> Self: + """Fail fast on unknown keys and on per-run/identity keys the single + global tokenizer / MetricsAggregator would ignore. Override *values* + are validated at merge time (see ``effective_generation_config``) + because cross-field validation needs the base ``ModelParams`` from + ``BenchmarkConfig``. + """ + if self.generation_config_override: + keys = set(self.generation_config_override) + valid = set(ModelParams.model_fields) + bad = sorted(keys - valid) + if bad: + raise ValueError( + f"Dataset '{self.name}': unknown keys in " + f"generation_config_override: {bad}. " + f"Valid keys: {sorted(valid)}" + ) + decoupled = sorted(keys & _METRICS_DECOUPLED_OVERRIDE_KEYS) + if decoupled: + raise ValueError( + f"Dataset '{self.name}': generation_config_override keys " + f"{decoupled} are not honored per-dataset — the single " + "global tokenizer / metrics aggregator is launched from " + "top-level model_params, so a per-dataset value would " + "desync ISL/OSL/TTFT/TPOT accounting. Set them on " + "top-level model_params instead." + ) + return self + + def effective_generation_config(self, base: ModelParams) -> ModelParams: + """Return base merged with this dataset's generation-config overrides. + + Nested dicts are deep-merged so a sparse nested override preserves + sibling defaults (e.g. ``{osl_distribution: {max: 512}}`` keeps the + base ``type/mean/std/min``). The merged dict is re-validated through + ``ModelParams.model_validate`` so type-invalid scalar overrides (e.g. + ``temperature: 'hot'``) are rejected. Note that this only catches + scalar invalidity — a sparse nested override whose merged result + passes default-validation will not raise (callers that need stricter + nested validation should set ``base`` to an explicit instance). + """ + if not self.generation_config_override: + return base + merged = _deep_merge(base.model_dump(), self.generation_config_override) + return ModelParams.model_validate(merged) diff --git a/src/inference_endpoint/config/enums.py b/src/inference_endpoint/config/enums.py new file mode 100644 index 000000000..e48c70e85 --- /dev/null +++ b/src/inference_endpoint/config/enums.py @@ -0,0 +1,131 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Configuration enums. + +Split criterion: one module per config domain; enums shared across the config +models live here so every sibling module can import them without cycles. +``config/schema.py`` re-exports the public surface. +""" + +from __future__ import annotations + +from enum import Enum + + +class LoadPatternType(str, Enum): + """Load pattern types.""" + + MAX_THROUGHPUT = "max_throughput" # Offline: all queries at t=0 + POISSON = "poisson" # Online: fixed QPS with Poisson distribution + CONCURRENCY = "concurrency" # Online: fixed concurrent requests + AGENTIC_INFERENCE = ( + "agentic_inference" # Agentic inference conversations with turn sequencing + ) + BURST = "burst" # Burst pattern (TODO) + STEP = "step" # Step pattern (TODO) + + +class OSLDistributionType(str, Enum): + """Output Sequence Length distribution types.""" + + ORIGINAL = "original" # Use original distribution from dataset (default) + FIXED = "fixed" # Fixed length for all outputs + UNIFORM = "uniform" # Uniform distribution between min and max + NORMAL = "normal" # Normal/Gaussian distribution + + +class DatasetType(str, Enum): + """Dataset purpose type.""" + + PERFORMANCE = "performance" + ACCURACY = "accuracy" + + +class EvalMethod(str, Enum): + """Evaluation methods for accuracy testing.""" + + EXACT_MATCH = "exact_match" + CONTAINS = "contains" + JUDGE = "judge" + + +class ScorerMethod(str, Enum): + """Registered scorer methods for accuracy evaluation.""" + + PASS_AT_1 = "pass_at_1" + STRING_MATCH = "string_match" + ROUGE = "rouge" + CODE_BENCH = "code_bench_scorer" + SHOPIFY_CATEGORY_F1 = "shopify_category_f1" + AGENTIC_INFERENCE_INLINE = "agentic_inference_inline" + VBENCH = "vbench" + BFCL_V4 = "bfcl_v4" + LEGACY_MLPERF_DEEPSEEK_R1 = "legacy_mlperf_deepseek_r1" + SWE_BENCH = "swe_bench_scorer" + + +class TestMode(str, Enum): + """Test mode controlling performance issuance and response collection. + + - PERF: Run performance and ordinary configured scoring without in-process + collection; skip scorers that own an external evaluation run + - ACC: Skip performance and collect responses for configured scoring + - BOTH: Run performance and configured scoring with response collection + """ + + PERF = "perf" + ACC = "acc" + BOTH = "both" + + +class StreamingMode(str, Enum): + """Streaming mode for response handling. + + - AUTO: Automatically enable for online mode, disable for offline mode + - ON: Force streaming enabled (for TTFT metrics) + - OFF: Force streaming disabled + """ + + AUTO = "auto" + ON = "on" + OFF = "off" + + +class TestType(str, Enum): + """Test type for both config classification and execution mode. + + - OFFLINE: Max throughput benchmark (all queries at t=0) + - ONLINE: Sustained QPS benchmark (Poisson or concurrency-based) + - EVAL: Accuracy evaluation + - SUBMISSION: Official submission (may include both perf and accuracy) + """ + + OFFLINE = "offline" + ONLINE = "online" + EVAL = "eval" + SUBMISSION = "submission" + + +class ProfilerEngine(str, Enum): + """Inference engine whose profiling protocol the client should drive. + + Selects the HTTP path layout used to derive start/stop URLs from + ``endpoint_config.endpoints``. Each value corresponds to one server-side + profiling protocol; add a new variant + ``_PROFILE_PATHS`` row to support + another engine. + """ + + VLLM = "vllm" diff --git a/src/inference_endpoint/config/model_params.py b/src/inference_endpoint/config/model_params.py new file mode 100644 index 000000000..a823039d7 --- /dev/null +++ b/src/inference_endpoint/config/model_params.py @@ -0,0 +1,165 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Model generation parameters and submission reference. + +Split criterion: one module per config domain; the model/generation-parameter +models and the submission reference live here. ``config/schema.py`` re-exports +the public surface. +""" + +from __future__ import annotations + +from typing import Annotated, Any, Self + +import cyclopts +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from .enums import OSLDistributionType, StreamingMode +from .ruleset_base import BenchmarkSuiteRuleset + + +def _non_default_completion_controls(mp: ModelParams) -> list[str]: + """Completion-only ModelParams controls set to a non-default value. + + ``min_new_tokens``/``skip_special_tokens`` are only honored by the + ``openai_completions`` adapter; ``BenchmarkConfig`` rejects them for other + ``api_type``s. Shared by the top-level and per-dataset-override checks so + both config surfaces validate identically. + """ + checks = { + "min_new_tokens": mp.min_new_tokens != 1, + "skip_special_tokens": not mp.skip_special_tokens, + } + return [name for name, non_default in checks.items() if non_default] + + +class OSLDistribution(BaseModel): + """Output Sequence Length distribution configuration. + + Distribution types: + - ORIGINAL: Use the natural distribution from the dataset (default) + - FIXED: All outputs have the same length (uses mean value) + - UNIFORM: Uniformly distributed between min and max + - NORMAL: Normal/Gaussian distribution with mean and std + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + type: OSLDistributionType = Field( + OSLDistributionType.ORIGINAL, description="Distribution type" + ) + mean: int | None = Field(None, description="Mean length (FIXED/NORMAL)") + std: int | None = Field(None, description="Std deviation (NORMAL)") + min: Annotated[ + int, + cyclopts.Parameter(alias="--min-output-tokens", help="Minimum output length"), + ] = 1 + max: int = Field(2048, description="Maximum output length") + + +class ModelParams(BaseModel): + """Model generation parameters.""" + + model_config = ConfigDict(extra="forbid", frozen=True, str_strip_whitespace=True) + + name: Annotated[ + str, + cyclopts.Parameter(alias="--model", help="Model name", required=True), + ] = "" + temperature: float | None = Field(None, description="Sampling temperature") + seed: Annotated[ + int | None, + cyclopts.Parameter( + alias="--seed", help="Random seed for reproducible sampling" + ), + ] = Field(None, description="Random seed for reproducible sampling") + top_k: int | None = Field(None, description="Top-K sampling") + top_p: float | None = Field(None, description="Top-P (nucleus) sampling") + repetition_penalty: float | None = Field(None, description="Repetition penalty") + presence_penalty: float | None = Field(None, description="Presence penalty") + frequency_penalty: float | None = Field(None, description="Frequency penalty") + chat_template_kwargs: dict[str, Any] | None = Field( + None, + description="Per-request chat-template kwargs forwarded to compatible servers.", + ) + max_new_tokens: Annotated[ + int, cyclopts.Parameter(alias="--max-output-tokens", help="Max output tokens") + ] = 1024 + min_new_tokens: int = Field( + 1, + ge=0, + description="Minimum output tokens for OpenAI text-completions servers", + ) + skip_special_tokens: bool = Field( + True, + description=( + "Whether OpenAI text-completions servers omit special tokens from decoded output" + ), + ) + osl_distribution: OSLDistribution | None = Field( + None, description="Output sequence length distribution" + ) + streaming: Annotated[ + StreamingMode, + cyclopts.Parameter(alias="--streaming", help="Streaming mode: auto/on/off"), + ] = StreamingMode.AUTO + tokenizer_name: Annotated[ + str | None, + cyclopts.Parameter( + alias="--tokenizer", + help="HF repo ID or local path for the tokenizer. Overrides model name for client-side token metrics (ISL/OSL/TPOT).", + ), + ] = None + + @model_validator(mode="after") + def _validate_generation_lengths(self) -> Self: + if self.min_new_tokens > self.max_new_tokens: + raise ValueError( + "min_new_tokens must be less than or equal to max_new_tokens" + ) + return self + + +class SubmissionReference(BaseModel): + """Reference configuration for official benchmark submissions. + + Links a submission to a specific model and ruleset (competition rules). + The ruleset defines constraints like min duration, sample counts, and + performance targets that must be met for a valid submission. + + Example: + submission_ref: + model: "llama-2-70b" + ruleset: "mlperf-inference-v5.1" + """ + + model_config = ConfigDict(extra="forbid", frozen=True, str_strip_whitespace=True) + + model: str # Model identifier (e.g., "llama-2-70b") + ruleset: str # Ruleset name/version (e.g., "mlperf-inference-v5.1") + + def get_ruleset_instance(self) -> BenchmarkSuiteRuleset: + """Get the actual ruleset instance from registry. + + Returns: + BenchmarkSuiteRuleset instance + + Raises: + KeyError: If ruleset not found in registry + """ + from .ruleset_registry import get_ruleset + + return get_ruleset(self.ruleset) diff --git a/src/inference_endpoint/config/rulesets/mlcommons/rules.py b/src/inference_endpoint/config/rulesets/mlcommons/rules.py index 2625d514c..1a12a40f7 100644 --- a/src/inference_endpoint/config/rulesets/mlcommons/rules.py +++ b/src/inference_endpoint/config/rulesets/mlcommons/rules.py @@ -27,7 +27,6 @@ from .... import metrics from ...ruleset_base import BenchmarkSuiteRuleset from ...runtime_settings import RuntimeSettings -from ...schema import SystemDefaults from ...user_config import UserConfig from . import models @@ -214,7 +213,7 @@ def apply_user_config( return _RuntimeSettings( metric_target=metric_target if metric_target is not None - else SystemDefaults.DEFAULT_METRIC, + else metrics.Throughput(0.0), reported_metrics=reported_metrics, min_duration_ms=min_duration_ms, max_duration_ms=max_duration_ms, diff --git a/src/inference_endpoint/config/runtime_settings.py b/src/inference_endpoint/config/runtime_settings.py index 6573ef82a..3bd6e15bc 100644 --- a/src/inference_endpoint/config/runtime_settings.py +++ b/src/inference_endpoint/config/runtime_settings.py @@ -94,9 +94,6 @@ class RuntimeSettings: reported_metrics: list[metrics.Metric] """List of metrics to collect and report""" - min_duration_ms: int - """Minimum benchmark duration in milliseconds""" - max_duration_ms: int | None """Maximum benchmark duration in milliseconds (timeout). None means no wall-clock limit.""" @@ -118,6 +115,11 @@ class RuntimeSettings: load_pattern: LoadPattern | None """Load pattern configuration""" + min_duration_ms: int | None = field(default=None, kw_only=True) + """Minimum performance-phase duration in ms (None/0 = no duration target: + issue the dataset once). Only rulesets set this; the config surface has no + duration-derived sample count.""" + sample_order: SampleOrderSpec = field(default_factory=SampleOrderSpec, kw_only=True) """Sample-ordering strategy (default: without-replacement).""" @@ -188,10 +190,8 @@ def _from_config_default( kwargs = { "metric_target": metrics.Throughput(effective_qps), "reported_metrics": [metrics.Throughput(effective_qps)], - "min_duration_ms": runtime_cfg.min_duration_ms, - "max_duration_ms": None - if runtime_cfg.max_duration_ms == 0 - else runtime_cfg.max_duration_ms, + "min_duration_ms": None, + "max_duration_ms": runtime_cfg.max_duration_ms, "n_samples_from_dataset": dataloader_num_samples, "n_samples_to_issue": runtime_cfg.n_samples_to_issue, # From config (CLI --num-samples or YAML) "min_sample_count": 1, @@ -213,7 +213,7 @@ def total_samples_to_issue( Priority: 1. If `n_samples_to_issue` is set, return it (explicit override) - 2. If min_duration_ms=0, return all dataset samples (new CLI default) + 2. If no duration target is set, return all dataset samples 3. Otherwise, calculate from metric target * duration Args: @@ -251,11 +251,12 @@ def total_samples_to_issue( ) return self.n_samples_from_dataset - # If min_duration is 0, use all dataset samples (new CLI default behavior) - if self.min_duration_ms == 0: + # No duration target (None from config, 0 from programmatic callers): + # issue the dataset once. + if not self.min_duration_ms: result = max(self.min_sample_count, self.n_samples_from_dataset) logger.debug( - f"Sample count: {result} (using all dataset samples, duration=0)" + f"Sample count: {result} (using all dataset samples, no duration target)" ) return result diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index e9a69e5d2..c809e0f8b 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -15,18 +15,23 @@ """Configuration schema — single source of truth for YAML and CLI. -All Pydantic models here define both the YAML config structure and the CLI interface. -cyclopts auto-generates CLI flags from fields. Use cyclopts.Parameter(alias=...) -on Annotated fields to declare shorthand aliases alongside dotted paths. +All Pydantic models define both the YAML config structure and the CLI +interface. cyclopts auto-generates CLI flags from fields. Use +cyclopts.Parameter(alias=...) on Annotated fields to declare shorthand +aliases alongside dotted paths. + +Split criterion: one module per config domain (enums / audit / model_params / +datasets / settings / timeouts); this module owns only the root aggregate +(``BenchmarkConfig`` and its cross-field validation) plus the explicit +re-export hub, so every existing ``config.schema`` import site keeps working. """ from __future__ import annotations import logging from collections import Counter -from enum import Enum from pathlib import Path -from typing import Annotated, Any, ClassVar, Literal, Self, Union +from typing import Annotated, Any, Literal, Self, Union from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit import cyclopts @@ -36,972 +41,85 @@ ConfigDict, Discriminator, Field, - SerializerFunctionWrapHandler, Tag, TypeAdapter, field_validator, - model_serializer, model_validator, ) -from .. import metrics from ..core.types import APIType -from ..endpoint_client.config import HTTPClientConfig from ..exceptions import CLIError from ..utils import WithUpdatesMixin -from .ruleset_base import BenchmarkSuiteRuleset +from .audit import AuditConfig, AuditTestId, OutputCachingTestConfig +from .datasets import AccuracyConfig, AgenticInferenceConfig, Dataset +from .enums import ( + DatasetType, + EvalMethod, + LoadPatternType, + OSLDistributionType, + ProfilerEngine, + ScorerMethod, + StreamingMode, + TestMode, + TestType, +) +from .model_params import ( + ModelParams, + OSLDistribution, + SubmissionReference, + _non_default_completion_controls, +) +from .settings import ( + EarlyStoppingConfig, + LoadPattern, + OfflineSettings, + OnlineSettings, + ProfilingConfig, + RuntimeConfig, + Settings, + WarmupConfig, +) +from .timeouts import Timeouts from .utils import parse_dataset_string, resolve_env_vars -logger = logging.getLogger(__name__) - - -class SystemDefaults(BaseModel): - DEFAULT_TIMEOUT: ClassVar[float] = 300.0 - DEFAULT_METRIC: ClassVar[metrics.Metric] = metrics.Throughput(0.0) - - -def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]: - """Recursively merge ``override`` into ``base`` and return the result. - - For overlapping keys whose values are both dicts, recurse; otherwise the - override value wins. Mutates a *copy* — callers can safely pass model_dump() - output. Used by ``Dataset.effective_generation_config`` so a sparse nested - override (e.g. ``{osl_distribution: {max: 512}}``) preserves siblings. - """ - out = dict(base) - for k, v in override.items(): - if isinstance(v, dict) and isinstance(out.get(k), dict): - out[k] = _deep_merge(out[k], v) - else: - out[k] = v - return out - - -# ModelParams fields that drive the single global tokenizer / MetricsAggregator -# (launched once from top-level model_params), so a per-dataset override would -# desync ISL/OSL/TTFT/TPOT accounting without changing what is measured. Rejected -# as generation_config_override keys — they are per-run/identity, not per-dataset. -_METRICS_DECOUPLED_OVERRIDE_KEYS = frozenset({"name", "streaming", "tokenizer_name"}) - - -def _non_default_completion_controls(mp: ModelParams) -> list[str]: - """Completion-only ModelParams controls set to a non-default value. - - ``min_new_tokens``/``skip_special_tokens`` are only honored by the - ``openai_completions`` adapter; ``BenchmarkConfig`` rejects them for other - ``api_type``s. Shared by the top-level and per-dataset-override checks so - both config surfaces validate identically. - """ - checks = { - "min_new_tokens": mp.min_new_tokens != 1, - "skip_special_tokens": not mp.skip_special_tokens, - } - return [name for name, non_default in checks.items() if non_default] - - -class LoadPatternType(str, Enum): - """Load pattern types.""" - - MAX_THROUGHPUT = "max_throughput" # Offline: all queries at t=0 - POISSON = "poisson" # Online: fixed QPS with Poisson distribution - CONCURRENCY = "concurrency" # Online: fixed concurrent requests - AGENTIC_INFERENCE = ( - "agentic_inference" # Agentic inference conversations with turn sequencing - ) - BURST = "burst" # Burst pattern (TODO) - STEP = "step" # Step pattern (TODO) - - -class OSLDistributionType(str, Enum): - """Output Sequence Length distribution types.""" - - ORIGINAL = "original" # Use original distribution from dataset (default) - FIXED = "fixed" # Fixed length for all outputs - UNIFORM = "uniform" # Uniform distribution between min and max - NORMAL = "normal" # Normal/Gaussian distribution - - -class DatasetType(str, Enum): - """Dataset purpose type.""" - - PERFORMANCE = "performance" - ACCURACY = "accuracy" - - -class EvalMethod(str, Enum): - """Evaluation methods for accuracy testing.""" - - EXACT_MATCH = "exact_match" - CONTAINS = "contains" - JUDGE = "judge" - - -class ScorerMethod(str, Enum): - """Registered scorer methods for accuracy evaluation.""" - - PASS_AT_1 = "pass_at_1" - STRING_MATCH = "string_match" - ROUGE = "rouge" - CODE_BENCH = "code_bench_scorer" - SHOPIFY_CATEGORY_F1 = "shopify_category_f1" - AGENTIC_INFERENCE_INLINE = "agentic_inference_inline" - VBENCH = "vbench" - BFCL_V4 = "bfcl_v4" - LEGACY_MLPERF_DEEPSEEK_R1 = "legacy_mlperf_deepseek_r1" - SWE_BENCH = "swe_bench_scorer" - - -class AuditTestId(str, Enum): - """Registered compliance audit test identifiers.""" - - # Output-caching audit — MLPerf TEST04 (duplicate-query caching detection). - OUTPUT_CACHING_TEST = "output_caching_test" - - -class OutputCachingTestConfig(BaseModel): - """Configuration for the output-caching audit (MLPerf TEST04). - - The output-caching test runs two back-to-back phases — a reference run of - distinct samples and an audit run that repeats one fixed sample — then - checks that the audit QPS does not exceed the reference QPS by more than - ``threshold``. A large speedup indicates the SUT is caching responses. - - samples: reference-phase query count (required — an explicit count keeps - the per-phase completion check meaningful; a duration-driven phase has - no independent target to validate completion against) - audit_samples: audit-phase query count (None → equals samples) - sample_index: which dataset row is repeated (MLCommons performance_issue_same_index) - threshold: tolerance shared by both pass checks — each phase must complete - ≥ requested * (1 - threshold), and audit_qps must stay < ref_qps * (1 + threshold) - """ - - model_config = ConfigDict(frozen=True, extra="forbid") - - test: Literal[AuditTestId.OUTPUT_CACHING_TEST] - only: bool = Field( - False, - description="Run only the audit — skip the main benchmark (upstream-style standalone TEST04)", - ) - samples: int = Field(..., ge=1, description="Reference phase query count") - audit_samples: int | None = Field( - None, ge=1, description="Audit phase query count (default: equals samples)" - ) - sample_index: int = Field( - 0, ge=0, description="Dataset row index repeated in the audit phase" - ) - threshold: float = Field( - 0.10, - gt=0, - lt=1, - description=( - "Tolerance for both checks: each phase must complete " - "≥ requested * (1 - threshold), and audit_qps must stay " - "< ref_qps * (1 + threshold)" - ), - ) - - -# Single member today; becomes -# Annotated[OutputCachingTestConfig | ..., Field(discriminator="test")] -# when additional audit tests are added. -AuditConfig = OutputCachingTestConfig - - -class TestMode(str, Enum): - """Test mode controlling performance issuance and response collection. - - - PERF: Run performance and ordinary configured scoring without in-process - collection; skip scorers that own an external evaluation run - - ACC: Skip performance and collect responses for configured scoring - - BOTH: Run performance and configured scoring with response collection - """ - - PERF = "perf" - ACC = "acc" - BOTH = "both" - - -class StreamingMode(str, Enum): - """Streaming mode for response handling. - - - AUTO: Automatically enable for online mode, disable for offline mode - - ON: Force streaming enabled (for TTFT metrics) - - OFF: Force streaming disabled - """ - - AUTO = "auto" - ON = "on" - OFF = "off" - - -class TestType(str, Enum): - """Test type for both config classification and execution mode. - - - OFFLINE: Max throughput benchmark (all queries at t=0) - - ONLINE: Sustained QPS benchmark (Poisson or concurrency-based) - - EVAL: Accuracy evaluation - - SUBMISSION: Official submission (may include both perf and accuracy) - """ - - OFFLINE = "offline" - ONLINE = "online" - EVAL = "eval" - SUBMISSION = "submission" - - -# Mapping from template type strings to TestType enums -# Single source of truth for template type conversion -TEMPLATE_TYPE_MAP = { - "offline": TestType.OFFLINE, - "online": TestType.ONLINE, - "eval": TestType.EVAL, - "submission": TestType.SUBMISSION, -} - - -class OSLDistribution(BaseModel): - """Output Sequence Length distribution configuration. - - Distribution types: - - ORIGINAL: Use the natural distribution from the dataset (default) - - FIXED: All outputs have the same length (uses mean value) - - UNIFORM: Uniformly distributed between min and max - - NORMAL: Normal/Gaussian distribution with mean and std - """ - - model_config = ConfigDict(extra="forbid", frozen=True) - - type: OSLDistributionType = Field( - OSLDistributionType.ORIGINAL, description="Distribution type" - ) - mean: int | None = Field(None, description="Mean length (FIXED/NORMAL)") - std: int | None = Field(None, description="Std deviation (NORMAL)") - min: Annotated[ - int, - cyclopts.Parameter(alias="--min-output-tokens", help="Minimum output length"), - ] = 1 - max: int = Field(2048, description="Maximum output length") - - -class ModelParams(BaseModel): - """Model generation parameters.""" - - model_config = ConfigDict(extra="forbid", frozen=True, str_strip_whitespace=True) - - name: Annotated[ - str, - cyclopts.Parameter(alias="--model", help="Model name", required=True), - ] = "" - temperature: float | None = Field(None, description="Sampling temperature") - seed: Annotated[ - int | None, - cyclopts.Parameter( - alias="--seed", help="Random seed for reproducible sampling" - ), - ] = Field(None, description="Random seed for reproducible sampling") - top_k: int | None = Field(None, description="Top-K sampling") - top_p: float | None = Field(None, description="Top-P (nucleus) sampling") - repetition_penalty: float | None = Field(None, description="Repetition penalty") - presence_penalty: float | None = Field(None, description="Presence penalty") - frequency_penalty: float | None = Field(None, description="Frequency penalty") - chat_template_kwargs: dict[str, Any] | None = Field( - None, - description="Per-request chat-template kwargs forwarded to compatible servers.", - ) - max_new_tokens: Annotated[ - int, cyclopts.Parameter(alias="--max-output-tokens", help="Max output tokens") - ] = 1024 - min_new_tokens: int = Field( - 1, - ge=0, - description="Minimum output tokens for OpenAI text-completions servers", - ) - skip_special_tokens: bool = Field( - True, - description=( - "Whether OpenAI text-completions servers omit special tokens from decoded output" - ), - ) - osl_distribution: OSLDistribution | None = Field( - None, description="Output sequence length distribution" - ) - streaming: Annotated[ - StreamingMode, - cyclopts.Parameter(alias="--streaming", help="Streaming mode: auto/on/off"), - ] = StreamingMode.AUTO - tokenizer_name: Annotated[ - str | None, - cyclopts.Parameter( - alias="--tokenizer", - help="HF repo ID or local path for the tokenizer. Overrides model name for client-side token metrics (ISL/OSL/TPOT).", - ), - ] = None - - @model_validator(mode="after") - def _validate_generation_lengths(self) -> Self: - if self.min_new_tokens > self.max_new_tokens: - raise ValueError( - "min_new_tokens must be less than or equal to max_new_tokens" - ) - return self - - -class SubmissionReference(BaseModel): - """Reference configuration for official benchmark submissions. - - Links a submission to a specific model and ruleset (competition rules). - The ruleset defines constraints like min duration, sample counts, and - performance targets that must be met for a valid submission. - - Example: - submission_ref: - model: "llama-2-70b" - ruleset: "mlperf-inference-v5.1" - """ - - model_config = ConfigDict(extra="forbid", frozen=True, str_strip_whitespace=True) - - model: str # Model identifier (e.g., "llama-2-70b") - ruleset: str # Ruleset name/version (e.g., "mlperf-inference-v5.1") - - def get_ruleset_instance(self) -> BenchmarkSuiteRuleset: - """Get the actual ruleset instance from registry. - - Returns: - BenchmarkSuiteRuleset instance - - Raises: - KeyError: If ruleset not found in registry - """ - from .ruleset_registry import get_ruleset - - return get_ruleset(self.ruleset) - - -class AgenticInferenceConfig(BaseModel): - """Agentic inference conversation configuration. - - Configuration for benchmarking conversational AI workloads with turn sequencing. - Enables testing agentic inference conversations where each turn depends on previous responses. - Presence of this block in the dataset config enables agentic inference mode. - - Attributes: - turn_timeout_s: Deadline between issuing a turn and receiving its - response. A timeout aborts that turn and all remaining client - turns of the same conversation because subsequent turns depend - on the timed-out response. - """ - - model_config = ConfigDict(extra="forbid", frozen=True) - - turn_timeout_s: float = Field( - default=86400.0, - gt=0, - description=( - "Per-turn timeout in seconds. A timeout aborts that turn and all " - "remaining turns in the same conversation." - ), - ) - enable_salt: bool = Field( - False, - description=( - "Add deterministic salt markers before and after the system prompt " - "to prevent KV cache reuse across trajectories in agentic inference setting." - ), - ) - inject_tool_delay: bool = Field( - False, - description=( - "Pause for a predefined duration between turns. Duration is defined " - "in dataset." - ), - ) - routing_headers: tuple[str, ...] = Field( - default=("X-Session-ID",), - description=( - "HTTP header names populated with the conversation ID on every " - "agentic request." - ), - ) - num_trajectories_to_issue: int | None = Field( - default=None, - gt=0, - description=( - "Number of conversation trajectories to start. Defaults to one pass " - "over the dataset; values above the dataset size repeat trajectories " - "with unique logical conversation ids." - ), - ) - stop_issuing_on_first_user_complete: bool = Field( - False, - description=( - "When performance tracking stops because the first concurrency slot " - "has no next trajectory left to assign, also stop issuing future " - "turns. If false, replay continues outside the performance window " - "for accuracy/log coverage." - ), - ) - - -class Dataset(BaseModel): - """Dataset configuration. - - Name and type have smart defaults: name is auto-derived from path, - type defaults to PERFORMANCE. - - Accepts CLI strings via BeforeValidator on BenchmarkConfig.datasets: - ``[perf|acc:][,key=value...]`` - """ - - model_config = ConfigDict(extra="forbid", frozen=True, str_strip_whitespace=True) - - name: str = Field("", description="Dataset name (auto-derived from path if empty)") - type: DatasetType = Field( - DatasetType.PERFORMANCE, description="Dataset purpose: performance or accuracy" - ) - path: Annotated[ - str | None, cyclopts.Parameter(alias="--dataset", help="Dataset file path") - ] = None - format: str | None = Field(None, description="Dataset format (auto-detected)") - samples: int | None = Field(None, gt=0, description="Number of samples to use") - eval_method: EvalMethod | None = Field( - None, description="Accuracy evaluation method" - ) - parser: dict[str, str] | None = Field( - None, description="Column remapping: {prompt: , system: }" - ) - generate_params: dict[str, Any] | None = Field( - None, description="Dataset-specific parameters passed to the generate() method" - ) - accuracy_config: AccuracyConfig | None = Field( - None, description="Accuracy evaluation settings" - ) - agentic_inference: AgenticInferenceConfig | None = Field( - None, description="Agentic inference conversation configuration" - ) - # Per-dataset generation config is a first-class capability: different - # accuracy datasets legitimately want different generation settings (e.g. - # per-dataset max OSL or top_p, as seen in DS-V4), and dataset-scoping also - # enables per-dataset dynamic OSL distributions. Only generation knobs are - # overridable — per-run/identity fields (`_METRICS_DECOUPLED_OVERRIDE_KEYS`: - # name / streaming / tokenizer_name) drive the single global tokenizer and - # MetricsAggregator, so overriding them per-dataset would desync ISL/OSL/ - # TTFT/TPOT accounting; they are rejected at validation. - # - # TODO(post-mortem): split ModelParams into a per-run ModelIdentity and a - # GenerationConfig, so the override surface is exactly the generation fields - # and identity fields cannot be named here at all. Field/method names use - # "generation_config" to keep that migration mechanical. - # - # Nested dicts (`osl_distribution`, `chat_template_kwargs`) are deep-merged - # so sparse overrides preserve sibling defaults. - generation_config_override: dict[str, Any] | None = Field( - None, - description=( - "Per-dataset overrides for the top-level model_params (sparse — " - "only the fields you want to override). Merged on top of " - "BenchmarkConfig.model_params at dataset-load time. Useful for " - "MLPerf-style runs where accuracy and performance use different " - "output budgets in the same fleet, e.g. " - "generation_config_override: {max_new_tokens: 32768, " - "temperature: 0.0}. NOTE: per-run/identity keys (`name`, " - "`streaming`, `tokenizer_name`) are rejected here — set them on " - "top-level model_params." - ), - ) - - @model_validator(mode="after") - def _auto_derive_name(self) -> Self: - """Derive name from path stem if not explicitly provided.""" - if not self.name and self.path: - object.__setattr__(self, "name", Path(self.path).stem) - return self - - @model_validator(mode="after") - def _validate_generation_config_override(self) -> Self: - """Fail fast on unknown keys and on per-run/identity keys the single - global tokenizer / MetricsAggregator would ignore. Override *values* - are validated at merge time (see ``effective_generation_config``) - because cross-field validation needs the base ``ModelParams`` from - ``BenchmarkConfig``. - """ - if self.generation_config_override: - keys = set(self.generation_config_override) - valid = set(ModelParams.model_fields) - bad = sorted(keys - valid) - if bad: - raise ValueError( - f"Dataset '{self.name}': unknown keys in " - f"generation_config_override: {bad}. " - f"Valid keys: {sorted(valid)}" - ) - decoupled = sorted(keys & _METRICS_DECOUPLED_OVERRIDE_KEYS) - if decoupled: - raise ValueError( - f"Dataset '{self.name}': generation_config_override keys " - f"{decoupled} are not honored per-dataset — the single " - "global tokenizer / metrics aggregator is launched from " - "top-level model_params, so a per-dataset value would " - "desync ISL/OSL/TTFT/TPOT accounting. Set them on " - "top-level model_params instead." - ) - return self - - def effective_generation_config(self, base: ModelParams) -> ModelParams: - """Return base merged with this dataset's generation-config overrides. - - Nested dicts are deep-merged so a sparse nested override preserves - sibling defaults (e.g. ``{osl_distribution: {max: 512}}`` keeps the - base ``type/mean/std/min``). The merged dict is re-validated through - ``ModelParams.model_validate`` so type-invalid scalar overrides (e.g. - ``temperature: 'hot'``) are rejected. Note that this only catches - scalar invalidity — a sparse nested override whose merged result - passes default-validation will not raise (callers that need stricter - nested validation should set ``base`` to an explicit instance). - """ - if not self.generation_config_override: - return base - merged = _deep_merge(base.model_dump(), self.generation_config_override) - return ModelParams.model_validate(merged) - - -class AccuracyConfig(BaseModel): - """Accuracy configuration. - - eval_method: Scorer to use (see ScorerMethod enum for options). - ground_truth: Column in the dataset containing ground truth. Defaults to "ground_truth". - extractor: Post-processor to extract answers from model output - (abcd_extractor, boxed_math_extractor, identity_extractor, python_code_extractor). - Optional for scorers that declare REQUIRES_EXTRACTOR = False (e.g. vbench). - num_repeats: Number of times to repeat the dataset for evaluation. Defaults to 1. - extras: Free-form keyword args forwarded to the scorer's ``__init__`` — - used for scorer-specific knobs that don't warrant a top-level field - (e.g. ``vbench_project_path``, ``subprocess_timeout_s`` for VBench). - - Example: - accuracy_config: - eval_method: "pass_at_1" - ground_truth: "answer" - extractor: "boxed_math_extractor" - num_repeats: 5 - extras: - vbench_project_path: "/path/to/accuracy" - """ - - model_config = ConfigDict(extra="forbid", frozen=True) - - eval_method: ScorerMethod | None = Field(None, description="Scorer method") - ground_truth: str | None = Field(None, description="Ground truth column name") - extractor: str | None = Field( - None, - description="Answer extractor (abcd_extractor, boxed_math_extractor, identity_extractor, python_code_extractor)", - ) - num_repeats: int = Field( - 1, ge=1, description="Repeat dataset N times for evaluation" - ) - extras: dict[str, Any] | None = Field( - None, - description="Free-form scorer kwargs (e.g. vbench_project_path, subprocess_timeout_s)", - ) - - -class RuntimeConfig(BaseModel): - """Runtime configuration. - - Sample count priority (in RuntimeSettings.total_samples_to_issue()): - 1. n_samples_to_issue (if specified) — explicit override - 2. Calculated from QPS * duration — duration-based (default: 600000ms) - 3. All dataset samples — fallback when duration is 0 - """ - - model_config = ConfigDict(extra="forbid", frozen=True) - - min_duration_ms: Annotated[ - int, - cyclopts.Parameter( - alias="--duration", help="Min duration (ms, or with suffix: 600s, 10m)" - ), - ] = Field(600000, ge=0) - max_duration_ms: int = Field( - 0, - ge=0, - description="Maximum test duration in ms (0 for no limit)", - ) - - @field_validator("min_duration_ms", "max_duration_ms", mode="before") - @classmethod - def _parse_duration_suffix(cls, v: object) -> object: - """Accept duration with unit suffix: 600s, 10m, 600000ms, or plain int (ms).""" - if isinstance(v, str): - v = v.strip() - if v.endswith("ms"): - return int(v[:-2]) - if v.endswith("m"): - return int(float(v[:-1]) * 60_000) - if v.endswith("s"): - return int(float(v[:-1]) * 1000) - return v - - n_samples_to_issue: Annotated[ - int | None, - cyclopts.Parameter(alias="--num-samples", help="Sample count override"), - ] = Field(None, gt=0) - scheduler_random_seed: int = Field(42, description="Scheduler RNG seed") - dataloader_random_seed: int = Field(42, description="Dataloader RNG seed") - - @model_validator(mode="after") - def _validate_durations(self) -> Self: - if self.max_duration_ms != 0 and self.max_duration_ms < self.min_duration_ms: - raise ValueError( - f"max_duration_ms ({self.max_duration_ms}) must be >= " - f"min_duration_ms ({self.min_duration_ms})" - ) - return self - - -@cyclopts.Parameter(name="*") -class LoadPattern(BaseModel): - """Load pattern configuration. - - Different patterns use target_qps differently: - - max_throughput: target_qps used for calculating total queries (offline, optional with default) - - poisson: target_qps sets scheduler rate (online, required - validated) - - concurrency: issue at fixed target_concurrency (online, required - validated) - """ - - model_config = ConfigDict(extra="forbid", frozen=True) - - type: Annotated[ - LoadPatternType, - cyclopts.Parameter(name="--load-pattern", help="Load pattern type"), - ] = LoadPatternType.MAX_THROUGHPUT - target_qps: Annotated[ - float | None, cyclopts.Parameter(alias="--target-qps", help="Target QPS") - ] = Field(None, gt=0) - target_concurrency: Annotated[ - int | None, - cyclopts.Parameter(alias="--concurrency", help="Concurrent requests"), - ] = Field(None, gt=0) - - # TODO(vir): remove once the formal tail-cutting mechanism lands. - use_legacy_loadgen_qps_metrics: Annotated[ - bool, - cyclopts.Parameter( - negative="--no-use-legacy-loadgen-qps-metrics", - help=( - "Only applies to the poisson load pattern. Report QPS/TPS using " - "the legacy MLPerf LoadGen Server 'completed' definition — (completed-1)/T " - "and tokens/T, T = first issued request to completion of the " - "last-issued request (see mlcommons/inference loadgen/results.cc). " - "--no-... uses endpoints-native completed/duration. Ignored for " - "non-poisson patterns." - ), - ), - ] = True - - @model_serializer(mode="wrap") - def _serialize(self, handler: SerializerFunctionWrapHandler) -> dict[str, Any]: - # use_legacy_loadgen_qps_metrics only applies to poisson; drop it from - # the serialized form (and thus YAML templates) for other patterns. - data = handler(self) - if self.type != LoadPatternType.POISSON: - data.pop("use_legacy_loadgen_qps_metrics", None) - return data - - @model_validator(mode="after") - def _validate_completeness(self) -> Self: - if self.type == LoadPatternType.POISSON and ( - self.target_qps is None or self.target_qps <= 0 - ): - raise ValueError("Poisson requires --target-qps (e.g., --target-qps 100)") - if self.type == LoadPatternType.CONCURRENCY and ( - not self.target_concurrency or self.target_concurrency <= 0 - ): - raise ValueError( - "Concurrency requires --concurrency (e.g., --concurrency 10)" - ) - if self.type == LoadPatternType.AGENTIC_INFERENCE and ( - not self.target_concurrency or self.target_concurrency <= 0 - ): - raise ValueError( - "Agentic inference requires --concurrency (e.g., --concurrency 96)" - ) - return self - - def __str__(self) -> str: - """Human-readable "type (param=value)" form for logging, e.g. - ``concurrency (target_concurrency=7)`` / ``poisson (target_qps=10.0)``. - Patterns without a driving parameter render as just the type name. - """ - if self.type in ( - LoadPatternType.CONCURRENCY, - LoadPatternType.AGENTIC_INFERENCE, - ): - return f"{self.type.value} (target_concurrency={self.target_concurrency})" - if self.type == LoadPatternType.POISSON: - return f"{self.type.value} (target_qps={self.target_qps})" - return self.type.value - - -@cyclopts.Parameter(name="*") -class WarmupConfig(BaseModel): - """Warmup phase configuration. Runs before the performance phase; results are not recorded.""" - - model_config = ConfigDict(extra="forbid", frozen=True) - - enabled: Annotated[ - bool, - cyclopts.Parameter( - alias="--warmup", help="Enable warmup phase before performance run" - ), - ] = Field(False, description="Enable warmup phase before performance run") - n_requests: Annotated[ - int | None, - cyclopts.Parameter( - alias="--warmup-requests", - help="Warmup request count (None = full dataset once)", - ), - ] = Field(None, gt=0, description="Warmup request count (None = full dataset once)") - salt: Annotated[ - bool, - cyclopts.Parameter( - alias="--warmup-salt", - help="Prepend a unique random hex salt to each warmup prompt", - ), - ] = Field( - True, description="Prepend a unique random hex salt to each warmup prompt" - ) - drain: Annotated[ - bool, - cyclopts.Parameter( - alias="--warmup-drain", - help="Drain in-flight warmup requests before starting the performance phase", - ), - ] = Field( - False, - description="Drain in-flight warmup requests before starting the performance phase", - ) - warmup_random_seed: Annotated[ - int, - cyclopts.Parameter( - alias="--warmup-seed", - help="RNG seed for warmup scheduling and sample ordering", - ), - ] = Field(42, description="RNG seed for warmup scheduling and sample ordering") - - -class DrainConfig(BaseModel): - """Per-phase in-flight response drain timeout configuration.""" - - model_config = ConfigDict(extra="forbid", frozen=True) - - warmup_timeout_s: Annotated[ - float | None, - cyclopts.Parameter( - alias="--warmup-drain-timeout", - help="Warmup drain timeout in seconds (None = wait indefinitely)", - ), - ] = Field( - 240.0, - gt=0, - description="Warmup drain timeout in seconds (None = wait indefinitely)", - ) - performance_timeout_s: Annotated[ - float | None, - cyclopts.Parameter( - alias="--performance-drain-timeout", - help="Performance drain timeout in seconds (None = wait indefinitely)", - ), - ] = Field( - 240.0, - gt=0, - description="Performance drain timeout in seconds (None = wait indefinitely)", - ) - accuracy_timeout_s: Annotated[ - float | None, - cyclopts.Parameter( - alias="--accuracy-drain-timeout", - help="Accuracy drain timeout in seconds (None = wait indefinitely)", - ), - ] = Field( - None, - gt=0, - description="Accuracy drain timeout in seconds (None = wait indefinitely)", - ) - metrics_drain_timeout_s: Annotated[ - float, - cyclopts.Parameter( - alias="--metrics-drain-timeout", - help=( - "Wall-clock budget (seconds) for the metrics aggregator to finish " - "tokenizing buffered samples after the run ends. Set to 0 to wait " - "indefinitely. Increase for very large datasets where the end-of-run " - "tokenize batch is big." - ), - ), - ] = Field( - 0.0, - ge=0, - description=( - "Wall-clock budget (seconds) to finish tokenizing buffered samples " - "after ENDED (default: 0 = unlimited). An incomplete drain is " - "surfaced via n_pending_tasks > 0, never silently dropped." - ), - ) - metrics_tokenizer_workers: Annotated[ - int, - cyclopts.Parameter( - alias="--metrics-tokenizer-workers", - help=( - "In-process tokenizer threads for live (mid-run) ISL/OSL/TPOT in " - "the metrics aggregator. 0 defers all tokenization to the " - "end-of-run drain, which always uses the auto-sized sharded pool." - ), - ), - ] = Field( - 4, - ge=0, - description=( - "In-process tokenizer threads for live (mid-run) ISL/OSL/TPOT " - "(default: 4; 0 = defer everything to the end-of-run drain)." - ), - ) - - -class ProfilerEngine(str, Enum): - """Inference engine whose profiling protocol the client should drive. - - Selects the HTTP path layout used to derive start/stop URLs from - ``endpoint_config.endpoints``. Each value corresponds to one server-side - profiling protocol; add a new variant + ``_PROFILE_PATHS`` row to support - another engine. - """ - - VLLM = "vllm" - - -@cyclopts.Parameter(name="*") -class ProfilingConfig(BaseModel): - """Client-side trigger for the server's profiler. - - When ``engine`` is set, fires POST ```` at performance-phase - begin and POST ```` at performance-phase end. URLs are derived - using the engine-specific protocol from ``urls`` when set, otherwise - from ``endpoint_config.endpoints``. - Server must be launched with profiling enabled (e.g. vLLM's - ``--profiler-config.profiler=cuda|torch``); the schedule - (``delay_iterations``, ``max_iterations``) is set there, not here. - """ - - model_config = ConfigDict(extra="forbid", frozen=True) - - engine: Annotated[ - ProfilerEngine | None, - cyclopts.Parameter( - alias="--profile", - help="Profile the named inference engine around the performance phase", - ), - ] = Field( - None, - description="Profile the named inference engine around the performance phase", - ) - urls: Annotated[ - list[str] | None, - cyclopts.Parameter( - alias="--profile-urls", - help="Override URL(s) for profiler triggers; " - "defaults to endpoint_config.endpoints", - negative="", - ), - ] = Field( - None, - description="URL(s) the profiler start/stop triggers are derived from. " - "When None, derived from endpoint_config.endpoints instead. Use when " - "the profiler admin endpoint differs from the inference endpoint.", - ) - - @field_validator("urls", mode="after") - @classmethod - def _validate_url_scheme(cls, v: list[str] | None) -> list[str] | None: - if v is None: - return v - for url in v: - if not url.startswith(("http://", "https://")): - raise ValueError( - f"Profiling endpoint URL must include scheme " - f"(http:// or https://), got: {url!r}" - ) - return v - - -class EarlyStoppingConfig(BaseModel): - """MLPerf-style early-stopping percentile estimates (on by default). - - Adds conservative, confidence-backed estimates of the tail percentiles to the - TTFT / TPOT / latency metrics in ``result_summary.json``. Computed once at run - COMPLETE from data the aggregator already keeps (hot path untouched), and the - output field is additive — so it is on by default; ``enabled: false`` is the - single opt-out (e.g. for consumers that strictly validate the summary schema). - Percentile targets, confidence (0.99), and tolerance (0.0) are LoadGen-parity - constants in ``metrics/early_stopping.py``, not knobs. Estimate-only: no - target-latency pass/fail and no dynamic mid-run halt. See ``docs/early_stopping.md``. - """ - - model_config = ConfigDict(extra="forbid", frozen=True) - - enabled: Annotated[ - bool, - cyclopts.Parameter( - alias="--early-stopping", # --no-early-stopping is the meaningful opt-out - help="Report MLPerf early-stopping percentile estimates for TTFT/TPOT/latency", - ), - ] = Field(True, description="Early-stopping percentile estimates (default on)") - - -@cyclopts.Parameter(name="*") -class Settings(BaseModel): - """Test settings.""" - - model_config = ConfigDict(extra="forbid", frozen=True) - - runtime: RuntimeConfig = Field(default_factory=RuntimeConfig) - load_pattern: LoadPattern = Field(default_factory=LoadPattern) - client: HTTPClientConfig = Field(default_factory=HTTPClientConfig) - drain: DrainConfig = Field( - default_factory=DrainConfig, - description="Per-phase in-flight response drain timeout configuration", - ) - warmup: WarmupConfig = Field(default_factory=WarmupConfig) - profiling: ProfilingConfig = Field(default_factory=ProfilingConfig) - early_stopping: EarlyStoppingConfig = Field( - default_factory=EarlyStoppingConfig, - description="MLPerf early-stopping percentile estimates (on by default; enabled: false opts out)", - ) - service_ready_timeout_s: Annotated[ - float, - cyclopts.Parameter( - alias="--service-ready-timeout", - help="Seconds to wait for metrics/event-logger services to start", - ), - ] = Field( - default=30.0, - ge=0, - description="Seconds to wait for metrics-aggregator/event-logger services to become ready.", - ) - - -class OfflineSettings(Settings): - """Offline mode default settings.""" - - load_pattern: Annotated[LoadPattern, cyclopts.Parameter(show=False)] = Field( - default_factory=lambda: LoadPattern(type=LoadPatternType.MAX_THROUGHPUT) - ) +__all__ = [ + "APIType", + "AccuracyConfig", + "AgenticInferenceConfig", + "AuditConfig", + "AuditTestId", + "BenchmarkConfig", + "Dataset", + "DatasetType", + "EarlyStoppingConfig", + "EndpointConfig", + "EvalMethod", + "LoadPattern", + "LoadPatternType", + "ModelParams", + "OSLDistribution", + "OSLDistributionType", + "OfflineBenchmarkConfig", + "OfflineSettings", + "OnlineBenchmarkConfig", + "OnlineSettings", + "OutputCachingTestConfig", + "ProfilerEngine", + "ProfilingConfig", + "RuntimeConfig", + "ScorerMethod", + "Settings", + "StreamingMode", + "SubmissionReference", + "TestMode", + "TestType", + "Timeouts", + "WarmupConfig", +] +logger = logging.getLogger(__name__) -class OnlineSettings(Settings): - """Online mode default settings.""" - pass class EndpointConfig(BaseModel): @@ -1078,10 +196,6 @@ class BenchmarkConfig(WithUpdatesMixin, BaseModel): Path | None, cyclopts.Parameter(alias="--report-dir", help="Report output directory"), ] = None - timeout: Annotated[ - float | None, - cyclopts.Parameter(alias="--timeout", help="Global timeout in seconds"), - ] = None # verbose is handled by cyclopts meta app (-v flag), not here verbose: Annotated[bool, cyclopts.Parameter(show=False)] = Field( False, description="Enable verbose logging" @@ -1120,7 +234,6 @@ def _resolve_and_validate(self) -> Self: Validation: - Workers must be -1 (auto) or >= 1 - - max_duration_ms >= min_duration_ms >= 0 - No duplicate dataset (name, type) pairs - Load pattern must match test type """ diff --git a/src/inference_endpoint/config/settings.py b/src/inference_endpoint/config/settings.py new file mode 100644 index 000000000..f90893859 --- /dev/null +++ b/src/inference_endpoint/config/settings.py @@ -0,0 +1,343 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test settings models (the ``settings:`` block). + +Split criterion: one module per config domain; the runtime/load-pattern/ +warmup/profiling settings and the ``Settings`` aggregate live here. +``config/schema.py`` re-exports the public surface. +""" + +from __future__ import annotations + +from typing import Annotated, Any, Self + +import cyclopts +from pydantic import ( + BaseModel, + ConfigDict, + Field, + SerializerFunctionWrapHandler, + field_validator, + model_serializer, + model_validator, +) + +from ..endpoint_client.config import HTTPClientConfig +from .enums import LoadPatternType, ProfilerEngine +from .timeouts import Timeouts + + +class RuntimeConfig(BaseModel): + """Runtime configuration. + + Sample count priority (in RuntimeSettings.total_samples_to_issue()): + 1. n_samples_to_issue (if specified) — explicit override + 2. All dataset samples — issue the dataset once + + ``max_duration_ms`` is a workload duration (part of the benchmark + definition), not a give-up deadline — those live in ``settings.timeouts``. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + max_duration_ms: int | None = Field( + None, + gt=0, + description="Maximum test duration in ms (None for no limit)", + ) + + @field_validator("max_duration_ms", mode="before") + @classmethod + def _parse_duration_suffix(cls, v: object) -> object: + """Accept duration with unit suffix: 600s, 10m, 600000ms, or plain int (ms).""" + if isinstance(v, str): + v = v.strip() + if v.endswith("ms"): + return int(v[:-2]) + if v.endswith("m"): + return int(float(v[:-1]) * 60_000) + if v.endswith("s"): + return int(float(v[:-1]) * 1000) + return v + + n_samples_to_issue: Annotated[ + int | None, + cyclopts.Parameter(alias="--num-samples", help="Sample count override"), + ] = Field(None, gt=0) + scheduler_random_seed: int = Field(42, description="Scheduler RNG seed") + dataloader_random_seed: int = Field(42, description="Dataloader RNG seed") + + +@cyclopts.Parameter(name="*") +class LoadPattern(BaseModel): + """Load pattern configuration. + + Different patterns use target_qps differently: + - max_throughput: target_qps used for calculating total queries (offline, optional with default) + - poisson: target_qps sets scheduler rate (online, required - validated) + - concurrency: issue at fixed target_concurrency (online, required - validated) + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + type: Annotated[ + LoadPatternType, + cyclopts.Parameter(name="--load-pattern", help="Load pattern type"), + ] = LoadPatternType.MAX_THROUGHPUT + target_qps: Annotated[ + float | None, cyclopts.Parameter(alias="--target-qps", help="Target QPS") + ] = Field(None, gt=0) + target_concurrency: Annotated[ + int | None, + cyclopts.Parameter(alias="--concurrency", help="Concurrent requests"), + ] = Field(None, gt=0) + + # TODO(vir): remove once the formal tail-cutting mechanism lands. + use_legacy_loadgen_qps_metrics: Annotated[ + bool, + cyclopts.Parameter( + negative="--no-use-legacy-loadgen-qps-metrics", + help=( + "Only applies to the poisson load pattern. Report QPS/TPS using " + "the legacy MLPerf LoadGen Server 'completed' definition — (completed-1)/T " + "and tokens/T, T = first issued request to completion of the " + "last-issued request (see mlcommons/inference loadgen/results.cc). " + "--no-... uses endpoints-native completed/duration. Ignored for " + "non-poisson patterns." + ), + ), + ] = True + + @model_serializer(mode="wrap") + def _serialize(self, handler: SerializerFunctionWrapHandler) -> dict[str, Any]: + # use_legacy_loadgen_qps_metrics only applies to poisson; drop it from + # the serialized form (and thus YAML templates) for other patterns. + data = handler(self) + if self.type != LoadPatternType.POISSON: + data.pop("use_legacy_loadgen_qps_metrics", None) + return data + + @model_validator(mode="after") + def _validate_completeness(self) -> Self: + if self.type == LoadPatternType.POISSON and ( + self.target_qps is None or self.target_qps <= 0 + ): + raise ValueError("Poisson requires --target-qps (e.g., --target-qps 100)") + if self.type == LoadPatternType.CONCURRENCY and ( + not self.target_concurrency or self.target_concurrency <= 0 + ): + raise ValueError( + "Concurrency requires --concurrency (e.g., --concurrency 10)" + ) + if self.type == LoadPatternType.AGENTIC_INFERENCE and ( + not self.target_concurrency or self.target_concurrency <= 0 + ): + raise ValueError( + "Agentic inference requires --concurrency (e.g., --concurrency 96)" + ) + return self + + def __str__(self) -> str: + """Human-readable "type (param=value)" form for logging, e.g. + ``concurrency (target_concurrency=7)`` / ``poisson (target_qps=10.0)``. + Patterns without a driving parameter render as just the type name. + """ + if self.type in ( + LoadPatternType.CONCURRENCY, + LoadPatternType.AGENTIC_INFERENCE, + ): + return f"{self.type.value} (target_concurrency={self.target_concurrency})" + if self.type == LoadPatternType.POISSON: + return f"{self.type.value} (target_qps={self.target_qps})" + return self.type.value + + +@cyclopts.Parameter(name="*") +class WarmupConfig(BaseModel): + """Warmup phase configuration. Runs before the performance phase; results are not recorded.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + enabled: Annotated[ + bool, + cyclopts.Parameter( + alias="--warmup", help="Enable warmup phase before performance run" + ), + ] = Field(False, description="Enable warmup phase before performance run") + n_requests: Annotated[ + int | None, + cyclopts.Parameter( + alias="--warmup-requests", + help="Warmup request count (None = full dataset once)", + ), + ] = Field(None, gt=0, description="Warmup request count (None = full dataset once)") + salt: Annotated[ + bool, + cyclopts.Parameter( + alias="--warmup-salt", + help="Prepend a unique random hex salt to each warmup prompt", + ), + ] = Field( + True, description="Prepend a unique random hex salt to each warmup prompt" + ) + drain: Annotated[ + bool, + cyclopts.Parameter( + alias="--warmup-drain", + help="Drain in-flight warmup requests before starting the performance phase", + ), + ] = Field( + False, + description="Drain in-flight warmup requests before starting the performance phase", + ) + warmup_random_seed: Annotated[ + int, + cyclopts.Parameter( + alias="--warmup-seed", + help="RNG seed for warmup scheduling and sample ordering", + ), + ] = Field(42, description="RNG seed for warmup scheduling and sample ordering") + + +@cyclopts.Parameter(name="*") +class ProfilingConfig(BaseModel): + """Client-side trigger for the server's profiler. + + When ``engine`` is set, fires POST ```` at performance-phase + begin and POST ```` at performance-phase end. URLs are derived + using the engine-specific protocol from ``urls`` when set, otherwise + from ``endpoint_config.endpoints``. + Server must be launched with profiling enabled (e.g. vLLM's + ``--profiler-config.profiler=cuda|torch``); the schedule + (``delay_iterations``, ``max_iterations``) is set there, not here. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + engine: Annotated[ + ProfilerEngine | None, + cyclopts.Parameter( + alias="--profile", + help="Profile the named inference engine around the performance phase", + ), + ] = Field( + None, + description="Profile the named inference engine around the performance phase", + ) + urls: Annotated[ + list[str] | None, + cyclopts.Parameter( + alias="--profile-urls", + help="Override URL(s) for profiler triggers; " + "defaults to endpoint_config.endpoints", + negative="", + ), + ] = Field( + None, + description="URL(s) the profiler start/stop triggers are derived from. " + "When None, derived from endpoint_config.endpoints instead. Use when " + "the profiler admin endpoint differs from the inference endpoint.", + ) + + @field_validator("urls", mode="after") + @classmethod + def _validate_url_scheme(cls, v: list[str] | None) -> list[str] | None: + if v is None: + return v + for url in v: + if not url.startswith(("http://", "https://")): + raise ValueError( + f"Profiling endpoint URL must include scheme " + f"(http:// or https://), got: {url!r}" + ) + return v + + +class EarlyStoppingConfig(BaseModel): + """MLPerf-style early-stopping percentile estimates (on by default). + + Adds conservative, confidence-backed estimates of the tail percentiles to the + TTFT / TPOT / latency metrics in ``result_summary.json``. Computed once at run + COMPLETE from data the aggregator already keeps (hot path untouched), and the + output field is additive — so it is on by default; ``enabled: false`` is the + single opt-out (e.g. for consumers that strictly validate the summary schema). + Percentile targets, confidence (0.99), and tolerance (0.0) are LoadGen-parity + constants in ``metrics/early_stopping.py``, not knobs. Estimate-only: no + target-latency pass/fail and no dynamic mid-run halt. See ``docs/early_stopping.md``. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + enabled: Annotated[ + bool, + cyclopts.Parameter( + alias="--early-stopping", # --no-early-stopping is the meaningful opt-out + help="Report MLPerf early-stopping percentile estimates for TTFT/TPOT/latency", + ), + ] = Field(True, description="Early-stopping percentile estimates (default on)") + + +@cyclopts.Parameter(name="*") +class Settings(BaseModel): + """Test settings.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + runtime: RuntimeConfig = Field(default_factory=RuntimeConfig) + load_pattern: LoadPattern = Field(default_factory=LoadPattern) + client: HTTPClientConfig = Field(default_factory=HTTPClientConfig) + timeouts: Timeouts = Field( + default_factory=Timeouts, + description="All global waits and deadlines (see config/timeouts.py)", + ) + warmup: WarmupConfig = Field(default_factory=WarmupConfig) + profiling: ProfilingConfig = Field(default_factory=ProfilingConfig) + early_stopping: EarlyStoppingConfig = Field( + default_factory=EarlyStoppingConfig, + description="MLPerf early-stopping percentile estimates (on by default; enabled: false opts out)", + ) + metrics_tokenizer_workers: Annotated[ + int, + cyclopts.Parameter( + alias="--metrics-tokenizer-workers", + help=( + "In-process tokenizer threads for live (mid-run) ISL/OSL/TPOT in " + "the metrics aggregator. 0 defers all tokenization to the " + "end-of-run drain, which always uses the auto-sized sharded pool." + ), + ), + ] = Field( + 4, + ge=0, + description=( + "In-process tokenizer threads for live (mid-run) ISL/OSL/TPOT " + "(default: 4; 0 = defer everything to the end-of-run drain)." + ), + ) + + +class OfflineSettings(Settings): + """Offline mode default settings.""" + + load_pattern: Annotated[LoadPattern, cyclopts.Parameter(show=False)] = Field( + default_factory=lambda: LoadPattern(type=LoadPatternType.MAX_THROUGHPUT) + ) + + +class OnlineSettings(Settings): + """Online mode default settings.""" + + pass diff --git a/src/inference_endpoint/config/templates/concurrency_template.yaml b/src/inference_endpoint/config/templates/concurrency_template.yaml index b0d9df61e..e16ed4f51 100644 --- a/src/inference_endpoint/config/templates/concurrency_template.yaml +++ b/src/inference_endpoint/config/templates/concurrency_template.yaml @@ -10,8 +10,6 @@ datasets: # Dataset configs prompt: text_input settings: runtime: - min_duration_ms: 600000 # Min duration (ms, or with suffix: 600s, 10m) - max_duration_ms: 0 # Maximum test duration in ms (0 for no limit) n_samples_to_issue: null # Sample count override load_pattern: type: concurrency # Load pattern type | options: max_throughput, poisson, concurrency, agentic_inference, burst, step diff --git a/src/inference_endpoint/config/templates/concurrency_template_full.yaml b/src/inference_endpoint/config/templates/concurrency_template_full.yaml index 217a762bc..d2c3efafe 100644 --- a/src/inference_endpoint/config/templates/concurrency_template_full.yaml +++ b/src/inference_endpoint/config/templates/concurrency_template_full.yaml @@ -51,8 +51,7 @@ datasets: # Dataset configs generation_config_override: null # Per-dataset overrides for the top-level model_params (sparse — only the fields you want to override). Merged on top of BenchmarkConfig.model_params at dataset-load time. Useful for MLPerf-style runs where accuracy and performance use different output budgets in the same fleet, e.g. generation_config_override: {max_new_tokens: 32768, temperature: 0.0}. NOTE: per-run/identity keys (`name`, `streaming`, `tokenizer_name`) are rejected here — set them on top-level model_params. settings: runtime: - min_duration_ms: 600000 # Min duration (ms, or with suffix: 600s, 10m) - max_duration_ms: 0 # Maximum test duration in ms (0 for no limit) + max_duration_ms: null # Maximum test duration in ms (None for no limit) n_samples_to_issue: null # Sample count override scheduler_random_seed: 42 # Scheduler RNG seed dataloader_random_seed: 42 # Dataloader RNG seed @@ -77,19 +76,20 @@ settings: linger: -1 # ZMQ linger on close (-1=block until sent) immediate: 1 # ZMQ IMMEDIATE (1=only enqueue on ready) stream_all_chunks: false # Stream all chunks to main thread (caution: perf overhead) - worker_initialization_timeout: 60.0 # Worker init timeout (seconds) - worker_graceful_shutdown_wait: 0.5 # Post-run graceful shutdown wait (seconds) - worker_force_kill_timeout: 0.5 # Force kill timeout after graceful wait (seconds) insecure: false # Skip TLS certificate verification max_idle_time: 4.0 # Discard connections idle longer than this (seconds) min_required_connections: -1 # Min connections to initialize (-1=auto, 0=disabled) worker_gc_mode: relaxed # Worker GC strategy | options: disabled, relaxed, system - drain: # Per-phase in-flight response drain timeout configuration - warmup_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) - performance_timeout_s: 240.0 # Performance drain timeout in seconds (None = wait indefinitely) - accuracy_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely) - metrics_drain_timeout_s: 0.0 # Wall-clock budget (seconds) to finish tokenizing buffered samples after ENDED (default: 0 = unlimited). An incomplete drain is surfaced via n_pending_tasks > 0, never silently dropped. - metrics_tokenizer_workers: 4 # In-process tokenizer threads for live (mid-run) ISL/OSL/TPOT (default: 4; 0 = defer everything to the end-of-run drain). + timeouts: # All global waits and deadlines (see config/timeouts.py) + run_timeout_s: null # Whole-run watchdog in seconds (None = off). Covers every phase including drains; firing aborts the run, marks the report INTERRUPTED, and exits non-zero. Never derives per-stage deadlines. + service_ready_timeout_s: 30.0 # Seconds to wait for metrics-aggregator/event-logger services to become ready. + warmup_drain_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) + performance_drain_timeout_s: null # Performance drain timeout in seconds (None = wait indefinitely) + accuracy_drain_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely; accuracy is unbounded by default because every sample must complete) + metrics_drain_timeout_s: null # Wall-clock budget (seconds) to finish tokenizing buffered samples after ENDED (None = wait indefinitely). An incomplete drain is surfaced via n_pending_tasks > 0, never silently dropped. + worker_initialization_timeout_s: 60.0 # Endpoint-client worker init timeout (seconds) + worker_graceful_shutdown_wait_s: 0.5 # Endpoint-client post-run graceful shutdown wait (seconds) + worker_force_kill_timeout_s: 0.5 # Endpoint-client force kill timeout after graceful wait (seconds) warmup: enabled: false # Enable warmup phase before performance run n_requests: null # Warmup request count (None = full dataset once) @@ -101,14 +101,13 @@ settings: urls: null # URL(s) the profiler start/stop triggers are derived from. When None, derived from endpoint_config.endpoints instead. Use when the profiler admin endpoint differs from the inference endpoint. early_stopping: # MLPerf early-stopping percentile estimates (on by default; enabled: false opts out) enabled: true # Early-stopping percentile estimates (default on) - service_ready_timeout_s: 30.0 # Seconds to wait for metrics-aggregator/event-logger services to become ready. + metrics_tokenizer_workers: 4 # In-process tokenizer threads for live (mid-run) ISL/OSL/TPOT (default: 4; 0 = defer everything to the end-of-run drain). endpoint_config: endpoints: # Endpoint URL(s). Must include scheme, e.g. 'http://host:port'. - http://localhost:8000 api_key: null # API key api_type: openai # API type: openai, sglang, or videogen | options: openai, openai_completions, sglang, videogen report_dir: null # Report output directory -timeout: null # Global timeout in seconds verbose: false # Enable verbose logging enable_cpu_affinity: true # NUMA-aware CPU pinning audit: null # Compliance audit config (YAML only). When set, runs the audit after the main benchmark. diff --git a/src/inference_endpoint/config/templates/offline_template.yaml b/src/inference_endpoint/config/templates/offline_template.yaml index 3305aa1ed..e4d2de8a6 100644 --- a/src/inference_endpoint/config/templates/offline_template.yaml +++ b/src/inference_endpoint/config/templates/offline_template.yaml @@ -10,8 +10,6 @@ datasets: # Dataset configs prompt: text_input settings: runtime: - min_duration_ms: 600000 # Min duration (ms, or with suffix: 600s, 10m) - max_duration_ms: 0 # Maximum test duration in ms (0 for no limit) n_samples_to_issue: null # Sample count override endpoint_config: endpoints: # Endpoint URL(s). Must include scheme, e.g. 'http://host:port'. diff --git a/src/inference_endpoint/config/templates/offline_template_full.yaml b/src/inference_endpoint/config/templates/offline_template_full.yaml index 587735956..7250f2ca5 100644 --- a/src/inference_endpoint/config/templates/offline_template_full.yaml +++ b/src/inference_endpoint/config/templates/offline_template_full.yaml @@ -51,8 +51,7 @@ datasets: # Dataset configs generation_config_override: null # Per-dataset overrides for the top-level model_params (sparse — only the fields you want to override). Merged on top of BenchmarkConfig.model_params at dataset-load time. Useful for MLPerf-style runs where accuracy and performance use different output budgets in the same fleet, e.g. generation_config_override: {max_new_tokens: 32768, temperature: 0.0}. NOTE: per-run/identity keys (`name`, `streaming`, `tokenizer_name`) are rejected here — set them on top-level model_params. settings: runtime: - min_duration_ms: 600000 # Min duration (ms, or with suffix: 600s, 10m) - max_duration_ms: 0 # Maximum test duration in ms (0 for no limit) + max_duration_ms: null # Maximum test duration in ms (None for no limit) n_samples_to_issue: null # Sample count override scheduler_random_seed: 42 # Scheduler RNG seed dataloader_random_seed: 42 # Dataloader RNG seed @@ -77,19 +76,20 @@ settings: linger: -1 # ZMQ linger on close (-1=block until sent) immediate: 1 # ZMQ IMMEDIATE (1=only enqueue on ready) stream_all_chunks: false # Stream all chunks to main thread (caution: perf overhead) - worker_initialization_timeout: 60.0 # Worker init timeout (seconds) - worker_graceful_shutdown_wait: 0.5 # Post-run graceful shutdown wait (seconds) - worker_force_kill_timeout: 0.5 # Force kill timeout after graceful wait (seconds) insecure: false # Skip TLS certificate verification max_idle_time: 4.0 # Discard connections idle longer than this (seconds) min_required_connections: -1 # Min connections to initialize (-1=auto, 0=disabled) worker_gc_mode: relaxed # Worker GC strategy | options: disabled, relaxed, system - drain: # Per-phase in-flight response drain timeout configuration - warmup_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) - performance_timeout_s: 240.0 # Performance drain timeout in seconds (None = wait indefinitely) - accuracy_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely) - metrics_drain_timeout_s: 0.0 # Wall-clock budget (seconds) to finish tokenizing buffered samples after ENDED (default: 0 = unlimited). An incomplete drain is surfaced via n_pending_tasks > 0, never silently dropped. - metrics_tokenizer_workers: 4 # In-process tokenizer threads for live (mid-run) ISL/OSL/TPOT (default: 4; 0 = defer everything to the end-of-run drain). + timeouts: # All global waits and deadlines (see config/timeouts.py) + run_timeout_s: null # Whole-run watchdog in seconds (None = off). Covers every phase including drains; firing aborts the run, marks the report INTERRUPTED, and exits non-zero. Never derives per-stage deadlines. + service_ready_timeout_s: 30.0 # Seconds to wait for metrics-aggregator/event-logger services to become ready. + warmup_drain_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) + performance_drain_timeout_s: null # Performance drain timeout in seconds (None = wait indefinitely) + accuracy_drain_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely; accuracy is unbounded by default because every sample must complete) + metrics_drain_timeout_s: null # Wall-clock budget (seconds) to finish tokenizing buffered samples after ENDED (None = wait indefinitely). An incomplete drain is surfaced via n_pending_tasks > 0, never silently dropped. + worker_initialization_timeout_s: 60.0 # Endpoint-client worker init timeout (seconds) + worker_graceful_shutdown_wait_s: 0.5 # Endpoint-client post-run graceful shutdown wait (seconds) + worker_force_kill_timeout_s: 0.5 # Endpoint-client force kill timeout after graceful wait (seconds) warmup: enabled: false # Enable warmup phase before performance run n_requests: null # Warmup request count (None = full dataset once) @@ -101,14 +101,13 @@ settings: urls: null # URL(s) the profiler start/stop triggers are derived from. When None, derived from endpoint_config.endpoints instead. Use when the profiler admin endpoint differs from the inference endpoint. early_stopping: # MLPerf early-stopping percentile estimates (on by default; enabled: false opts out) enabled: true # Early-stopping percentile estimates (default on) - service_ready_timeout_s: 30.0 # Seconds to wait for metrics-aggregator/event-logger services to become ready. + metrics_tokenizer_workers: 4 # In-process tokenizer threads for live (mid-run) ISL/OSL/TPOT (default: 4; 0 = defer everything to the end-of-run drain). endpoint_config: endpoints: # Endpoint URL(s). Must include scheme, e.g. 'http://host:port'. - http://localhost:8000 api_key: null # API key api_type: openai # API type: openai, sglang, or videogen | options: openai, openai_completions, sglang, videogen report_dir: null # Report output directory -timeout: null # Global timeout in seconds verbose: false # Enable verbose logging enable_cpu_affinity: true # NUMA-aware CPU pinning audit: null # Compliance audit config (YAML only). When set, runs the audit after the main benchmark. diff --git a/src/inference_endpoint/config/templates/online_template.yaml b/src/inference_endpoint/config/templates/online_template.yaml index 501670691..65180d2e6 100644 --- a/src/inference_endpoint/config/templates/online_template.yaml +++ b/src/inference_endpoint/config/templates/online_template.yaml @@ -10,8 +10,6 @@ datasets: # Dataset configs prompt: text_input settings: runtime: - min_duration_ms: 600000 # Min duration (ms, or with suffix: 600s, 10m) - max_duration_ms: 0 # Maximum test duration in ms (0 for no limit) n_samples_to_issue: null # Sample count override load_pattern: type: poisson # Load pattern type | options: max_throughput, poisson, concurrency, agentic_inference, burst, step diff --git a/src/inference_endpoint/config/templates/online_template_full.yaml b/src/inference_endpoint/config/templates/online_template_full.yaml index 95bc8555c..8b0d18f39 100644 --- a/src/inference_endpoint/config/templates/online_template_full.yaml +++ b/src/inference_endpoint/config/templates/online_template_full.yaml @@ -51,8 +51,7 @@ datasets: # Dataset configs generation_config_override: null # Per-dataset overrides for the top-level model_params (sparse — only the fields you want to override). Merged on top of BenchmarkConfig.model_params at dataset-load time. Useful for MLPerf-style runs where accuracy and performance use different output budgets in the same fleet, e.g. generation_config_override: {max_new_tokens: 32768, temperature: 0.0}. NOTE: per-run/identity keys (`name`, `streaming`, `tokenizer_name`) are rejected here — set them on top-level model_params. settings: runtime: - min_duration_ms: 600000 # Min duration (ms, or with suffix: 600s, 10m) - max_duration_ms: 0 # Maximum test duration in ms (0 for no limit) + max_duration_ms: null # Maximum test duration in ms (None for no limit) n_samples_to_issue: null # Sample count override scheduler_random_seed: 42 # Scheduler RNG seed dataloader_random_seed: 42 # Dataloader RNG seed @@ -78,19 +77,20 @@ settings: linger: -1 # ZMQ linger on close (-1=block until sent) immediate: 1 # ZMQ IMMEDIATE (1=only enqueue on ready) stream_all_chunks: false # Stream all chunks to main thread (caution: perf overhead) - worker_initialization_timeout: 60.0 # Worker init timeout (seconds) - worker_graceful_shutdown_wait: 0.5 # Post-run graceful shutdown wait (seconds) - worker_force_kill_timeout: 0.5 # Force kill timeout after graceful wait (seconds) insecure: false # Skip TLS certificate verification max_idle_time: 4.0 # Discard connections idle longer than this (seconds) min_required_connections: -1 # Min connections to initialize (-1=auto, 0=disabled) worker_gc_mode: relaxed # Worker GC strategy | options: disabled, relaxed, system - drain: # Per-phase in-flight response drain timeout configuration - warmup_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) - performance_timeout_s: 240.0 # Performance drain timeout in seconds (None = wait indefinitely) - accuracy_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely) - metrics_drain_timeout_s: 0.0 # Wall-clock budget (seconds) to finish tokenizing buffered samples after ENDED (default: 0 = unlimited). An incomplete drain is surfaced via n_pending_tasks > 0, never silently dropped. - metrics_tokenizer_workers: 4 # In-process tokenizer threads for live (mid-run) ISL/OSL/TPOT (default: 4; 0 = defer everything to the end-of-run drain). + timeouts: # All global waits and deadlines (see config/timeouts.py) + run_timeout_s: null # Whole-run watchdog in seconds (None = off). Covers every phase including drains; firing aborts the run, marks the report INTERRUPTED, and exits non-zero. Never derives per-stage deadlines. + service_ready_timeout_s: 30.0 # Seconds to wait for metrics-aggregator/event-logger services to become ready. + warmup_drain_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) + performance_drain_timeout_s: null # Performance drain timeout in seconds (None = wait indefinitely) + accuracy_drain_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely; accuracy is unbounded by default because every sample must complete) + metrics_drain_timeout_s: null # Wall-clock budget (seconds) to finish tokenizing buffered samples after ENDED (None = wait indefinitely). An incomplete drain is surfaced via n_pending_tasks > 0, never silently dropped. + worker_initialization_timeout_s: 60.0 # Endpoint-client worker init timeout (seconds) + worker_graceful_shutdown_wait_s: 0.5 # Endpoint-client post-run graceful shutdown wait (seconds) + worker_force_kill_timeout_s: 0.5 # Endpoint-client force kill timeout after graceful wait (seconds) warmup: enabled: false # Enable warmup phase before performance run n_requests: null # Warmup request count (None = full dataset once) @@ -102,14 +102,13 @@ settings: urls: null # URL(s) the profiler start/stop triggers are derived from. When None, derived from endpoint_config.endpoints instead. Use when the profiler admin endpoint differs from the inference endpoint. early_stopping: # MLPerf early-stopping percentile estimates (on by default; enabled: false opts out) enabled: true # Early-stopping percentile estimates (default on) - service_ready_timeout_s: 30.0 # Seconds to wait for metrics-aggregator/event-logger services to become ready. + metrics_tokenizer_workers: 4 # In-process tokenizer threads for live (mid-run) ISL/OSL/TPOT (default: 4; 0 = defer everything to the end-of-run drain). endpoint_config: endpoints: # Endpoint URL(s). Must include scheme, e.g. 'http://host:port'. - http://localhost:8000 api_key: null # API key api_type: openai # API type: openai, sglang, or videogen | options: openai, openai_completions, sglang, videogen report_dir: null # Report output directory -timeout: null # Global timeout in seconds verbose: false # Enable verbose logging enable_cpu_affinity: true # NUMA-aware CPU pinning audit: null # Compliance audit config (YAML only). When set, runs the audit after the main benchmark. diff --git a/src/inference_endpoint/config/templates/submission_template.yaml b/src/inference_endpoint/config/templates/submission_template.yaml index ac3c2b11d..70b9925a7 100644 --- a/src/inference_endpoint/config/templates/submission_template.yaml +++ b/src/inference_endpoint/config/templates/submission_template.yaml @@ -46,7 +46,6 @@ datasets: settings: runtime: - min_duration_ms: 600000 # 10 minutes max_duration_ms: 1800000 # 30 minutes scheduler_random_seed: 42 # For Poisson/distribution sampling dataloader_random_seed: 42 # For dataset shuffling diff --git a/src/inference_endpoint/config/timeouts.py b/src/inference_endpoint/config/timeouts.py new file mode 100644 index 000000000..c767ad445 --- /dev/null +++ b/src/inference_endpoint/config/timeouts.py @@ -0,0 +1,145 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Global waits and deadlines (the ``settings.timeouts`` block). + +Split criterion: one module per config domain; every global time knob that +bounds how long the harness waits — startup readiness, per-phase drains, the +worker lifecycle, and the whole-run watchdog — lives here. Workload durations +(``runtime.max_duration_ms``) are part of the benchmark definition, not waits, +and stay in ``runtime``. Dataset-scoped time knobs (e.g. agentic +``turn_timeout_s``) stay in their dataset config blocks. +""" + +from __future__ import annotations + +from typing import Annotated + +import cyclopts +from pydantic import BaseModel, ConfigDict, Field + +from ..utils import WithUpdatesMixin + + +@cyclopts.Parameter(name="*") +class Timeouts(WithUpdatesMixin, BaseModel): + """All global waits and deadlines. ``None`` = wait indefinitely / off. + + Reaching an optional deadline means something is stuck; ``run_timeout_s`` + is the whole-run watchdog — when it fires the run is aborted and the + report is marked INTERRUPTED. It never derives or caps the other + deadlines. Workload durations (``runtime.max_duration_ms``) are NOT + timeouts and do not live here. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + run_timeout_s: Annotated[ + float | None, + cyclopts.Parameter( + alias="--timeout", + help=( + "Whole-run watchdog in seconds (None = off). Firing aborts the " + "run and marks the report INTERRUPTED." + ), + ), + ] = Field( + None, + gt=0, + description=( + "Whole-run watchdog in seconds (None = off). Covers every phase " + "including drains; firing aborts the run, marks the report " + "INTERRUPTED, and exits non-zero. Never derives per-stage deadlines." + ), + ) + service_ready_timeout_s: Annotated[ + float, + cyclopts.Parameter( + alias="--service-ready-timeout", + help="Seconds to wait for metrics/event-logger services to start", + ), + ] = Field( + 30.0, + ge=0, + description="Seconds to wait for metrics-aggregator/event-logger services to become ready.", + ) + warmup_drain_timeout_s: Annotated[ + float | None, + cyclopts.Parameter( + alias="--warmup-drain-timeout", + help="Warmup drain timeout in seconds (None = wait indefinitely)", + ), + ] = Field( + 240.0, + gt=0, + description="Warmup drain timeout in seconds (None = wait indefinitely)", + ) + performance_drain_timeout_s: Annotated[ + float | None, + cyclopts.Parameter( + alias="--performance-drain-timeout", + help="Performance drain timeout in seconds (None = wait indefinitely)", + ), + ] = Field( + None, + gt=0, + description="Performance drain timeout in seconds (None = wait indefinitely)", + ) + accuracy_drain_timeout_s: Annotated[ + float | None, + cyclopts.Parameter( + alias="--accuracy-drain-timeout", + help="Accuracy drain timeout in seconds (None = wait indefinitely)", + ), + ] = Field( + None, + gt=0, + description=( + "Accuracy drain timeout in seconds (None = wait indefinitely; " + "accuracy is unbounded by default because every sample must complete)" + ), + ) + metrics_drain_timeout_s: Annotated[ + float | None, + cyclopts.Parameter( + alias="--metrics-drain-timeout", + help=( + "Wall-clock budget (seconds) for the metrics aggregator to finish " + "tokenizing buffered samples after the run ends " + "(None = wait indefinitely)" + ), + ), + ] = Field( + None, + gt=0, + description=( + "Wall-clock budget (seconds) to finish tokenizing buffered samples " + "after ENDED (None = wait indefinitely). An incomplete drain is " + "surfaced via n_pending_tasks > 0, never silently dropped." + ), + ) + worker_initialization_timeout_s: float = Field( + 60.0, ge=0, description="Endpoint-client worker init timeout (seconds)" + ) + worker_graceful_shutdown_wait_s: float = Field( + 0.5, + ge=0, + description="Endpoint-client post-run graceful shutdown wait (seconds)", + ) + worker_force_kill_timeout_s: float = Field( + 0.5, + ge=0, + description="Endpoint-client force kill timeout after graceful wait (seconds)", + ) diff --git a/src/inference_endpoint/endpoint_client/config.py b/src/inference_endpoint/endpoint_client/config.py index b2839996d..6802a2851 100644 --- a/src/inference_endpoint/endpoint_client/config.py +++ b/src/inference_endpoint/endpoint_client/config.py @@ -187,15 +187,24 @@ class HTTPClientConfig(WithUpdatesMixin, BaseModel): False, description="Stream all chunks to main thread (caution: perf overhead)" ) - # Worker lifecycle timeouts - worker_initialization_timeout: float = Field( - 60.0, description="Worker init timeout (seconds)" - ) - worker_graceful_shutdown_wait: float = Field( - 0.5, description="Post-run graceful shutdown wait (seconds)" + # Worker lifecycle timeouts — runtime carriers. The authoritative user + # knobs live in settings.timeouts; setup copies them here (no CLI flag, + # never serialized). WithUpdatesMixin.with_updates reads exclude=True + # fields directly, so copies preserve the injected values. + worker_initialization_timeout_s: Annotated[ + float, cyclopts.Parameter(parse=False) + ] = Field(60.0, exclude=True, description="Worker init timeout (seconds)") + worker_graceful_shutdown_wait_s: Annotated[ + float, cyclopts.Parameter(parse=False) + ] = Field( + 0.5, exclude=True, description="Post-run graceful shutdown wait (seconds)" ) - worker_force_kill_timeout: float = Field( - 0.5, description="Force kill timeout after graceful wait (seconds)" + worker_force_kill_timeout_s: Annotated[float, cyclopts.Parameter(parse=False)] = ( + Field( + 0.5, + exclude=True, + description="Force kill timeout after graceful wait (seconds)", + ) ) # Set to True to skip certificate verification (e.g. self-signed certs). diff --git a/src/inference_endpoint/endpoint_client/worker_manager.py b/src/inference_endpoint/endpoint_client/worker_manager.py index ae0d194df..bd0304447 100644 --- a/src/inference_endpoint/endpoint_client/worker_manager.py +++ b/src/inference_endpoint/endpoint_client/worker_manager.py @@ -92,7 +92,7 @@ async def initialize(self) -> None: except TimeoutError as e: raise TimeoutError( - f"Workers failed to initialize within {self.http_config.worker_initialization_timeout}s" + f"Workers failed to initialize within {self.http_config.worker_initialization_timeout_s}s" ) from e finally: @@ -130,7 +130,7 @@ def _pin_workers(self) -> None: async def _wait_for_workers_with_liveness_check(self) -> None: """Wait for workers, checking liveness at 10% intervals.""" - timeout = self.http_config.worker_initialization_timeout + timeout = self.http_config.worker_initialization_timeout_s check_interval = timeout * 0.10 if timeout else 1.0 start = time.monotonic() @@ -165,7 +165,7 @@ async def shutdown(self) -> None: if worker.is_alive(): worker.terminate() - await asyncio.sleep(self.http_config.worker_graceful_shutdown_wait) + await asyncio.sleep(self.http_config.worker_graceful_shutdown_wait_s) # Force kill remaining for worker in self.workers: @@ -176,7 +176,7 @@ async def shutdown(self) -> None: await asyncio.gather( *( asyncio.to_thread( - worker.join, timeout=self.http_config.worker_force_kill_timeout + worker.join, timeout=self.http_config.worker_force_kill_timeout_s ) for worker in self.workers ) diff --git a/tests/integration/commands/test_accuracy_pipeline.py b/tests/integration/commands/test_accuracy_pipeline.py index 3f6c66055..ab6cac027 100644 --- a/tests/integration/commands/test_accuracy_pipeline.py +++ b/tests/integration/commands/test_accuracy_pipeline.py @@ -35,7 +35,6 @@ LoadPattern, LoadPatternType, ModelParams, - RuntimeConfig, Settings, StreamingMode, TestMode, @@ -119,7 +118,6 @@ def test_accuracy_scoring_with_echo_server( ), ], settings=Settings( - runtime=RuntimeConfig(min_duration_ms=0), load_pattern=LoadPattern(type=LoadPatternType.MAX_THROUGHPUT), client=HTTPClientConfig( num_workers=1, warmup_connections=0, max_connections=10 diff --git a/tests/integration/commands/test_benchmark_command.py b/tests/integration/commands/test_benchmark_command.py index ec9f5cb56..1aec94185 100644 --- a/tests/integration/commands/test_benchmark_command.py +++ b/tests/integration/commands/test_benchmark_command.py @@ -42,7 +42,6 @@ from inference_endpoint.endpoint_client.config import HTTPClientConfig _TEST_SETTINGS = Settings( - runtime=RuntimeConfig(min_duration_ms=0), load_pattern=LoadPattern(type=LoadPatternType.MAX_THROUGHPUT), client=HTTPClientConfig(num_workers=1, warmup_connections=0, max_connections=10), ) @@ -61,8 +60,10 @@ def _config(endpoint_url: str, dataset_path: str, **overrides) -> BenchmarkConfi def _poisson_settings(target_qps: float, duration_s: int = 2) -> Settings: + # Pin the workload length via an explicit sample count equivalent to + # target_qps * duration_s. return Settings( - runtime=RuntimeConfig(min_duration_ms=duration_s * 1000), + runtime=RuntimeConfig(n_samples_to_issue=int(target_qps * duration_s)), load_pattern=LoadPattern(type=LoadPatternType.POISSON, target_qps=target_qps), client=HTTPClientConfig( num_workers=1, warmup_connections=0, max_connections=10 @@ -121,7 +122,7 @@ def test_concurrency_benchmark( type=TestType.ONLINE, model_params=ModelParams(name="echo-server", streaming=streaming), settings=Settings( - runtime=RuntimeConfig(min_duration_ms=2000), + runtime=RuntimeConfig(n_samples_to_issue=40), load_pattern=LoadPattern( type=LoadPatternType.CONCURRENCY, target_concurrency=4 ), @@ -176,7 +177,6 @@ def test_mode_logging(self, mock_http_echo_server, ds_dataset_path, caplog): ( TestType.OFFLINE, Settings( - runtime=RuntimeConfig(min_duration_ms=0), load_pattern=LoadPattern(type=LoadPatternType.MAX_THROUGHPUT), client=HTTPClientConfig( num_workers=1, warmup_connections=0, max_connections=10 @@ -186,7 +186,6 @@ def test_mode_logging(self, mock_http_echo_server, ds_dataset_path, caplog): ( TestType.ONLINE, Settings( - runtime=RuntimeConfig(min_duration_ms=0), load_pattern=LoadPattern( type=LoadPatternType.CONCURRENCY, target_concurrency=1 ), @@ -281,7 +280,6 @@ def test_cli_run_dispatches_main_run_before_audit( model_params=ModelParams(name="echo-server", streaming=StreamingMode.OFF), datasets=[Dataset(path=ds_dataset_path, type=DatasetType.PERFORMANCE)], settings=Settings( - runtime=RuntimeConfig(min_duration_ms=0), load_pattern=LoadPattern(type=LoadPatternType.MAX_THROUGHPUT), client=HTTPClientConfig( num_workers=1, warmup_connections=0, max_connections=10 @@ -377,8 +375,8 @@ def _resolve_template(template_path: Path, server_url: str) -> dict: # The other 5 templates benefit from warm module / IPC caches and don't # need the headroom. 120 s is a generous safety margin that does not # change the production default, only this integration test. - data["settings"].setdefault("client", {}) - data["settings"]["client"]["worker_initialization_timeout"] = 120.0 + data["settings"].setdefault("timeouts", {}) + data["settings"]["timeouts"]["worker_initialization_timeout_s"] = 120.0 # Accuracy datasets can't run e2e against echo server (no scorer), so keep only performance datasets. data["datasets"] = [ diff --git a/tests/integration/commands/test_cli.py b/tests/integration/commands/test_cli.py index cec491eb1..d8c1f7a84 100644 --- a/tests/integration/commands/test_cli.py +++ b/tests/integration/commands/test_cli.py @@ -284,8 +284,6 @@ def test_offline(self, mock_http_echo_server, ds_dataset_path, tmp_path): tmp_path, "benchmark", "offline", - "--duration", - "0", "--streaming", "off", ) @@ -308,8 +306,8 @@ def test_poisson(self, mock_http_echo_server, ds_dataset_path, tmp_path): "poisson", "--target-qps", "50", - "--duration", - "2000", + "--num-samples", + "100", ) assert r["n_samples_issued"] > 0 @@ -326,7 +324,7 @@ def test_concurrency(self, mock_http_echo_server, ds_dataset_path, tmp_path): "concurrency", "--concurrency", "4", - "--duration", - "2000", + "--num-samples", + "40", ) assert r["n_samples_issued"] > 0 diff --git a/tests/integration/commands/test_run_timeout.py b/tests/integration/commands/test_run_timeout.py new file mode 100644 index 000000000..e4d811b01 --- /dev/null +++ b/tests/integration/commands/test_run_timeout.py @@ -0,0 +1,172 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Whole-run watchdog (settings.timeouts.run_timeout_s) integration tests. + +Locking invariant: a fired run watchdog must never produce a COMPLETE +report. The watchdog SIGTERMs the metrics aggregator (whose handler writes +an INTERRUPTED final snapshot) before stopping the session, and +``run_benchmark`` exits non-zero via ``ExecutionError``. +""" + +import json +from pathlib import Path + +import pytest +from inference_endpoint.commands.benchmark.execute import run_benchmark +from inference_endpoint.config.schema import ( + BenchmarkConfig, + Dataset, + DatasetType, + EndpointConfig, + LoadPattern, + LoadPatternType, + ModelParams, + RuntimeConfig, + Settings, + StreamingMode, + TestMode, + TestType, + WarmupConfig, +) +from inference_endpoint.config.timeouts import Timeouts +from inference_endpoint.endpoint_client.config import HTTPClientConfig +from inference_endpoint.exceptions import ExecutionError + +# Local character-level tokenizer: lets the metrics aggregator tokenize +# ISL/OSL without a HuggingFace Hub download (same trick as +# test_benchmark_command.py). +_CHAR_TOKENIZER_DIR = Path(__file__).resolve().parents[2] / "assets/tokenizers/char" + +_FAST_CLIENT = HTTPClientConfig(num_workers=1, warmup_connections=0, max_connections=10) + + +def _read_final_snapshot(report_dir: Path) -> dict: + snapshot_path = report_dir / "metrics" / "final_snapshot.json" + assert snapshot_path.exists(), "aggregator must still write a final snapshot" + return json.loads(snapshot_path.read_text()) + + +def _read_result_summary(report_dir: Path) -> dict: + return json.loads((report_dir / "performance" / "result_summary.json").read_text()) + + +@pytest.mark.integration +def test_run_timeout_produces_interrupted_report( + mock_http_echo_server, ds_dataset_path, tmp_path +): + """run_timeout_s firing mid-run aborts with an INTERRUPTED report.""" + config = BenchmarkConfig( + type=TestType.ONLINE, + endpoint_config=EndpointConfig(endpoints=[mock_http_echo_server.url]), + model_params=ModelParams(name="echo-server", streaming=StreamingMode.OFF), + datasets=[Dataset(path=str(ds_dataset_path), type=DatasetType.PERFORMANCE)], + report_dir=tmp_path, + settings=Settings( + load_pattern=LoadPattern(type=LoadPatternType.POISSON, target_qps=5), + client=_FAST_CLIENT, + # 600 samples at 5 QPS is a ~120 s workload, so only the watchdog + # can end the run. + runtime=RuntimeConfig(n_samples_to_issue=600), + timeouts=Timeouts(run_timeout_s=2.0), + warmup=WarmupConfig(enabled=False), + ), + ) + + with pytest.raises(ExecutionError, match="Run timeout"): + run_benchmark(config, TestMode.PERF) + + snapshot = _read_final_snapshot(tmp_path) + assert snapshot["state"] == "interrupted" + + # Locking invariant: a fired run watchdog must never yield a COMPLETE report. + summary = _read_result_summary(tmp_path) + assert summary["complete"] is False + + +@pytest.mark.integration +def test_generous_run_timeout_completes_normally( + mock_http_echo_server, ds_dataset_path, tmp_path +): + """A run_timeout_s far above the workload length never fires: the run + finishes cleanly and publishes a COMPLETE report.""" + config = BenchmarkConfig( + type=TestType.OFFLINE, + endpoint_config=EndpointConfig(endpoints=[mock_http_echo_server.url]), + model_params=ModelParams(name="echo-server", streaming=StreamingMode.OFF), + datasets=[Dataset(path=str(ds_dataset_path), type=DatasetType.PERFORMANCE)], + report_dir=tmp_path, + settings=Settings( + load_pattern=LoadPattern(type=LoadPatternType.MAX_THROUGHPUT), + client=_FAST_CLIENT, + timeouts=Timeouts(run_timeout_s=300.0), + warmup=WarmupConfig(enabled=False), + ), + ) + + run_benchmark(config, TestMode.PERF) # must not raise + + snapshot = _read_final_snapshot(tmp_path) + assert snapshot["state"] == "complete" + summary = _read_result_summary(tmp_path) + assert summary["complete"] is True + + +@pytest.mark.integration +def test_run_timeout_during_metrics_drain_interrupts(mock_http_echo_server, tmp_path): + """The watchdog stays armed through the metrics drain. + + The session itself finishes quickly, but the aggregator is left with a + deliberately huge tokenization backlog (large prompts echoed back as + outputs, metrics_tokenizer_workers=0 so nothing tokenizes mid-run, and + the metrics drain unlimited). The watchdog must fire while the + aggregator drains, SIGTERM it, and surface the run as INTERRUPTED with + a non-zero exit. + """ + # ~25 MB of prompt text; the echo server doubles it into OSL, so the + # drain has ~50M characters to tokenize — far more than run_timeout_s + # allows on any hardware. + dataset_path = tmp_path / "big_prompts.jsonl" + prompt = "lorem ipsum " * 21_000 # ~250 KB per sample + with dataset_path.open("w") as f: + for i in range(100): + f.write(json.dumps({"prompt": f"{i} {prompt}"}) + "\n") + + report_dir = tmp_path / "report" + config = BenchmarkConfig( + type=TestType.OFFLINE, + endpoint_config=EndpointConfig(endpoints=[mock_http_echo_server.url]), + model_params=ModelParams( + name=str(_CHAR_TOKENIZER_DIR), streaming=StreamingMode.OFF + ), + datasets=[Dataset(path=str(dataset_path), type=DatasetType.PERFORMANCE)], + report_dir=report_dir, + settings=Settings( + load_pattern=LoadPattern(type=LoadPatternType.MAX_THROUGHPUT), + client=_FAST_CLIENT, + # Defer every ISL/OSL tokenization to the end-of-run drain. + metrics_tokenizer_workers=0, + # metrics_drain_timeout_s stays None (unlimited): only the + # run watchdog can end the drain. + timeouts=Timeouts(run_timeout_s=2.5), + warmup=WarmupConfig(enabled=False), + ), + ) + + with pytest.raises(ExecutionError, match="Run timeout"): + run_benchmark(config, TestMode.PERF) + + snapshot = _read_final_snapshot(report_dir) + assert snapshot["state"] == "interrupted" diff --git a/tests/integration/commands/test_warmup.py b/tests/integration/commands/test_warmup.py index 4622365e5..3c26302a0 100644 --- a/tests/integration/commands/test_warmup.py +++ b/tests/integration/commands/test_warmup.py @@ -96,7 +96,7 @@ def _offline_config( model_params=ModelParams(name="test-model", streaming=StreamingMode.OFF), datasets=[ConfigDataset(path=str(dataset_path), type=DatasetType.PERFORMANCE)], settings=OfflineSettings( - runtime=RuntimeConfig(min_duration_ms=0, n_samples_to_issue=n_perf_samples), + runtime=RuntimeConfig(n_samples_to_issue=n_perf_samples), load_pattern=LoadPattern(type=LoadPatternType.MAX_THROUGHPUT), client=_MINIMAL_CLIENT, warmup=warmup, diff --git a/tests/unit/async_utils/services/test_launcher.py b/tests/unit/async_utils/services/test_launcher.py new file mode 100644 index 000000000..15f5c78fb --- /dev/null +++ b/tests/unit/async_utils/services/test_launcher.py @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import signal +import subprocess +import sys +from unittest.mock import MagicMock + +import pytest +from inference_endpoint.async_utils.services.launcher import ServiceLauncher + + +@pytest.mark.unit +def test_terminate_sigterms_only_exact_module_match(): + target = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"]) + bystander = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"]) + launcher = ServiceLauncher(MagicMock()) + launcher._procs = [target, bystander] + launcher._modules = ["svc.metrics_aggregator", "prefix.svc.metrics_aggregator"] + try: + launcher.terminate("svc.metrics_aggregator") + assert target.wait(timeout=5.0) == -signal.SIGTERM + assert bystander.poll() is None, ( + "terminate() must match the exact module name; a mere suffix match " + "must stay alive" + ) + finally: + for proc in (target, bystander): + if proc.poll() is None: + proc.kill() + proc.wait(timeout=5.0) + + +@pytest.mark.unit +def test_terminate_ignores_already_exited_proc(): + dead = subprocess.Popen([sys.executable, "-c", "pass"]) + dead.wait(timeout=5.0) + launcher = ServiceLauncher(MagicMock()) + launcher._procs = [dead] + launcher._modules = ["svc.metrics_aggregator"] + + launcher.terminate("svc.metrics_aggregator") + + assert dead.returncode == 0 diff --git a/tests/unit/async_utils/transport/test_protocol.py b/tests/unit/async_utils/transport/test_protocol.py new file mode 100644 index 000000000..16dc8da6a --- /dev/null +++ b/tests/unit/async_utils/transport/test_protocol.py @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import asyncio +from collections import deque + +import pytest +from inference_endpoint.async_utils.transport.protocol import MessageSubscriber + + +class _IntCodec: + def encode(self, item: int) -> tuple[bytes, bytes]: + return b"test____", str(item).encode() + + def decode(self, payload: bytes) -> int: + return int(payload) + + def on_decode_error(self, payload: bytes, exc: Exception) -> int | None: + return None + + +class _QueueSubscriber(MessageSubscriber[int]): + def __init__( + self, + loop: asyncio.AbstractEventLoop, + payloads: list[bytes], + *, + max_read_batch_size: int, + ) -> None: + super().__init__(_IntCodec(), "test://subscriber", loop) + self._payloads = deque(payloads) + self._max_read_batch_size = max_read_batch_size + self.batches: list[list[int]] = [] + self.received: list[int] = [] + self.done = asyncio.Event() + self.expected = len(payloads) + self.release = asyncio.Event() + self.block_processing = False + self.active = 0 + self.max_active = 0 + + def receive(self) -> bytes | None: + if not self._payloads: + raise StopIteration + return self._payloads.popleft() + + async def process(self, items: list[int]) -> None: + self.active += 1 + self.max_active = max(self.max_active, self.active) + if self.block_processing: + await self.release.wait() + self.batches.append(items) + self.received.extend(items) + self.active -= 1 + if len(self.received) >= self.expected: + self.done.set() + + +@pytest.mark.unit +@pytest.mark.asyncio +async def test_subscriber_caps_each_read_and_reschedules_without_new_edge(): + subscriber = _QueueSubscriber( + asyncio.get_running_loop(), + [str(i).encode() for i in range(5)], + max_read_batch_size=2, + ) + + subscriber._on_readable() + await asyncio.wait_for(subscriber.done.wait(), timeout=1) + + assert subscriber.received == [0, 1, 2, 3, 4] + assert subscriber.batches == [[0, 1], [2, 3], [4]] + + +@pytest.mark.unit +@pytest.mark.asyncio +async def test_subscriber_processes_batches_single_flight_in_fifo_order(): + subscriber = _QueueSubscriber( + asyncio.get_running_loop(), [b"1"], max_read_batch_size=4 + ) + subscriber.block_processing = True + subscriber.expected = 2 + + subscriber._on_readable() + subscriber._payloads.append(b"2") + subscriber._on_readable() + await asyncio.sleep(0) + subscriber.release.set() + await asyncio.wait_for(subscriber.done.wait(), timeout=1) + + assert subscriber.received == [1, 2] + assert subscriber.max_active == 1 + + +@pytest.mark.unit +def test_none_payloads_count_toward_read_budget_and_close_cancels_resume(): + subscriber = _QueueSubscriber( + asyncio.new_event_loop(), + [None, None, b"3"], # type: ignore[list-item] + max_read_batch_size=2, + ) + try: + subscriber._on_readable() + + assert list(subscriber._payloads) == [b"3"] + assert subscriber._read_continuation is not None + + subscriber.close() + assert subscriber._read_continuation is None + finally: + subscriber.loop.close() diff --git a/tests/unit/commands/test_benchmark.py b/tests/unit/commands/test_benchmark.py index 1b1302af2..52b998989 100644 --- a/tests/unit/commands/test_benchmark.py +++ b/tests/unit/commands/test_benchmark.py @@ -48,6 +48,7 @@ finalize_benchmark, setup_benchmark, ) +from inference_endpoint.commands.benchmark.pipeline import _build_aggregator_args from inference_endpoint.commands.benchmark.profiling import ( ProfileController, _derive_profile_urls, @@ -60,7 +61,6 @@ AgenticInferenceConfig, BenchmarkConfig, DatasetType, - DrainConfig, LoadPattern, LoadPatternType, OfflineSettings, @@ -82,8 +82,9 @@ from inference_endpoint.config.schema import ( OnlineBenchmarkConfig as OnlineConfig, ) +from inference_endpoint.config.timeouts import Timeouts from inference_endpoint.config.utils import cli_error_formatter as _error_formatter -from inference_endpoint.core.types import QueryResult +from inference_endpoint.core.types import APIType, QueryResult from inference_endpoint.dataset_manager.dataset import Dataset from inference_endpoint.dataset_manager.predefined.swe_bench import SWEBench from inference_endpoint.endpoint_client.config import HTTPClientConfig @@ -266,15 +267,13 @@ def test_mode_defaults(self, cls, extra_kwargs, expected_type, expected_streamin config = cls(**_OFFLINE_KWARGS, **extra_kwargs) assert config.type == expected_type assert config.model_params.streaming == expected_streaming - assert config.settings.runtime.min_duration_ms == 600000 + assert config.settings.runtime.n_samples_to_issue is None @pytest.mark.unit def test_num_samples_override(self): config = OfflineConfig( **_OFFLINE_KWARGS, - settings=OfflineSettings( - runtime=RuntimeConfig(min_duration_ms=0, n_samples_to_issue=100) - ), + settings=OfflineSettings(runtime=RuntimeConfig(n_samples_to_issue=100)), ) assert config.settings.runtime.n_samples_to_issue == 100 @@ -332,30 +331,6 @@ def test_concurrency_injection_into_swe_bench_extras( assert acc_ds.accuracy_config.extras.get("workers") == expected_workers -class TestDurationSuffix: - """Test duration suffix parsing (600s, 10m, 600000ms, plain int).""" - - @pytest.mark.unit - @pytest.mark.parametrize( - "value, expected_ms", - [ - ("600s", 600000), - ("10m", 600000), - ("600000ms", 600000), - ("600000", 600000), - (600000, 600000), - ("0.5m", 30000), - ("1.5s", 1500), - ], - ) - def test_duration_suffix(self, value, expected_ms): - config = OfflineConfig( - **_OFFLINE_KWARGS, - settings=OfflineSettings(runtime=RuntimeConfig(min_duration_ms=value)), - ) - assert config.settings.runtime.min_duration_ms == expected_ms - - class TestDatasetParsing: """Test dataset string coercion through BenchmarkConfig construction.""" @@ -527,7 +502,7 @@ def test_from_config_handler(self, mock_run, tmp_path): config_file.write_text(yaml_content) from_config(config=config_file, timeout=42.0, mode=TestMode.BOTH) called_config, called_mode = mock_run.call_args[0] - assert called_config.timeout == 42.0 + assert called_config.settings.timeouts.run_timeout_s == 42.0 assert called_mode == TestMode.BOTH @pytest.mark.unit @@ -1033,69 +1008,6 @@ def test_warmup_default_in_settings(self): assert warmup.n_requests is None -class TestDrainConfig: - """Tests for DrainConfig schema model.""" - - @pytest.mark.unit - def test_defaults(self): - cfg = DrainConfig() - assert cfg.warmup_timeout_s == 240.0 - assert cfg.performance_timeout_s == 240.0 - assert cfg.accuracy_timeout_s is None - assert cfg.metrics_drain_timeout_s == 0.0 - - @pytest.mark.unit - @pytest.mark.parametrize( - "field", - ["warmup_timeout_s", "performance_timeout_s", "accuracy_timeout_s"], - ) - @pytest.mark.parametrize("value", [0, -1.0]) - def test_timeout_must_be_positive_or_none(self, field, value): - with pytest.raises(ValidationError): - DrainConfig(**{field: value}) - - @pytest.mark.unit - def test_metrics_drain_timeout_zero_is_valid(self): - cfg = DrainConfig(metrics_drain_timeout_s=0) - assert cfg.metrics_drain_timeout_s == 0.0 - - @pytest.mark.unit - def test_metrics_drain_timeout_negative_rejected(self): - with pytest.raises(ValidationError): - DrainConfig(metrics_drain_timeout_s=-1.0) - - @pytest.mark.unit - def test_extra_fields_rejected(self): - with pytest.raises(ValidationError): - DrainConfig(unknown_field=1) - - @pytest.mark.unit - def test_yaml_roundtrip(self, tmp_path): - yaml_content = """ -type: "offline" -model_params: - name: "test-model" -endpoint_config: - endpoints: ["http://test:8000"] -datasets: - - path: "test.jsonl" -settings: - drain: - warmup_timeout_s: 12.5 - performance_timeout_s: 30.0 - accuracy_timeout_s: null - metrics_drain_timeout_s: 300.0 -""" - config_file = tmp_path / "drain.yaml" - config_file.write_text(yaml_content) - config = BenchmarkConfig.from_yaml_file(config_file) - drain = config.settings.drain - assert drain.warmup_timeout_s == 12.5 - assert drain.performance_timeout_s == 30.0 - assert drain.accuracy_timeout_s is None - assert drain.metrics_drain_timeout_s == 300.0 - - class TestAggregatorArgs: """Tests that metrics aggregator subprocess args are correctly forwarded.""" @@ -1129,7 +1041,7 @@ def _make_ctx(self, config, tmp_path): @pytest.mark.asyncio @pytest.mark.parametrize( "timeout_s, expected_flag", - [(120.0, "120.0"), (0.0, "0.0"), (60.0, "60.0")], + [(120.0, "120.0"), (None, "0"), (60.0, "60.0")], ) async def test_drain_timeout_forwarded_to_aggregator_args( self, tmp_path, timeout_s, expected_flag @@ -1137,7 +1049,7 @@ async def test_drain_timeout_forwarded_to_aggregator_args( config = OfflineConfig( **_OFFLINE_KWARGS, settings=OfflineSettings( - drain=DrainConfig(metrics_drain_timeout_s=timeout_s) + timeouts=Timeouts(metrics_drain_timeout_s=timeout_s) ), ) ctx = self._make_ctx(config, tmp_path) @@ -1182,12 +1094,29 @@ async def _capture_launch(service_configs, *, timeout): idx = args.index("--drain-timeout") assert args[idx + 1] == expected_flag + @pytest.mark.unit + def test_none_drain_timeout_builds_unlimited_argv(self): + """None (= unlimited) must cross the argv boundary as "0", never "None".""" + args = _build_aggregator_args( + socket_dir="/tmp/sockets", + pub_socket_name="pub", + metrics_socket_name="metrics", + metrics_output_dir=Path("/tmp/metrics"), + enable_streaming=False, + tokenizer_name=None, + drain_timeout_s=None, + tokenizer_workers=2, + early_stopping=False, + ) + idx = args.index("--drain-timeout") + assert args[idx + 1] == "0" + @pytest.mark.unit @pytest.mark.asyncio async def test_tokenizer_and_workers_forwarded_from_schema(self, tmp_path): """The benchmark forwards --tokenizer and --tokenizer-workers; the workers value comes from the schema default - (drain.metrics_tokenizer_workers), the single source of truth.""" + (settings.metrics_tokenizer_workers), the single source of truth.""" config = OfflineConfig(**_OFFLINE_KWARGS, settings=OfflineSettings()) ctx = self._make_ctx(config, tmp_path) ctx.tokenizer_name = "gpt2" @@ -1231,7 +1160,7 @@ async def _capture_launch(service_configs, *, timeout): idx = args.index("--tokenizer") assert args[idx + 1] == "gpt2" idx = args.index("--tokenizer-workers") - expected = str(config.settings.drain.metrics_tokenizer_workers) + expected = str(config.settings.metrics_tokenizer_workers) assert args[idx + 1] == expected @pytest.mark.unit @@ -1889,10 +1818,10 @@ def test_configured_drain_timeouts_propagate_to_phases( config = OfflineConfig( **_OFFLINE_KWARGS, settings=OfflineSettings( - drain=DrainConfig( - warmup_timeout_s=7.0, - performance_timeout_s=15.0, - accuracy_timeout_s=45.0, + timeouts=Timeouts( + warmup_drain_timeout_s=7.0, + performance_drain_timeout_s=15.0, + accuracy_drain_timeout_s=45.0, ), warmup=WarmupConfig(enabled=True, drain=True), ), @@ -2611,6 +2540,37 @@ def test_accuracy_only_normalizes_client_and_target_concurrency( assert ctx.config.settings.client.max_connections == 1 assert ctx.config.settings.load_pattern.target_concurrency == 1 + @pytest.mark.unit + def test_accuracy_only_setup_validates_with_non_default_api_type( + self, tmp_path, _base_patches, _simple_dataset, _rt_settings + ): + """A non-default endpoint api_type (propagated into the client via + _propagate_client_api_type's with_updates) must survive the ACC-mode + client normalization re-validation without errors.""" + config = OnlineConfig( + endpoint_config={"endpoints": ["http://x"], "api_type": "sglang"}, + model_params={"name": "test-model"}, + settings=OnlineSettings( + load_pattern=LoadPattern( + type=LoadPatternType.CONCURRENCY, target_concurrency=10 + ), + client=HTTPClientConfig( + num_workers=4, warmup_connections=0, max_connections=8 + ), + ), + report_dir=str(tmp_path), + ) + ctx = self._setup( + config, + TestMode.ACC, + (_simple_dataset, [], []), + _rt_settings, + ) + + assert ctx.config.settings.client.num_workers == 1 + assert ctx.config.endpoint_config.api_type == APIType.SGLANG + assert ctx.config.settings.client.api_type == APIType.SGLANG + @pytest.mark.unit def test_perf_run_leaves_target_concurrency_untouched( self, tmp_path, _base_patches, _simple_dataset, _rt_settings diff --git a/tests/unit/compliance/test_output_caching.py b/tests/unit/compliance/test_output_caching.py index ec2bc2af4..505bc6864 100644 --- a/tests/unit/compliance/test_output_caching.py +++ b/tests/unit/compliance/test_output_caching.py @@ -479,6 +479,7 @@ def test_refuses_result_on_incomplete_phase(self, tmp_path, monkeypatch): incomplete = MagicMock() incomplete.complete = False bench = MagicMock() + bench.run_timed_out = False bench.report = incomplete self._patch_phase(monkeypatch, num_samples=100, bench=bench) @@ -495,12 +496,30 @@ def test_interrupted_phase_raises_keyboard_interrupt(self, tmp_path, monkeypatch interrupted.state = "interrupted" interrupted.complete = False bench = MagicMock() + bench.run_timed_out = False bench.report = interrupted self._patch_phase(monkeypatch, num_samples=100, bench=bench) with pytest.raises(KeyboardInterrupt): run_audit(config, tmp_path) + @pytest.mark.unit + def test_run_timeout_raises_execution_error(self, tmp_path, monkeypatch): + """A whole-run watchdog (settings.timeouts.run_timeout_s) firing during + an audit phase must surface as ExecutionError naming the timeout, not + as the Ctrl-C KeyboardInterrupt path.""" + config = self._audit_config() + interrupted = MagicMock() + interrupted.state = "interrupted" + interrupted.complete = False + bench = MagicMock() + bench.run_timed_out = True + bench.report = interrupted + self._patch_phase(monkeypatch, num_samples=100, bench=bench) + + with pytest.raises(ExecutionError, match="run_timeout_s"): + run_audit(config, tmp_path) + @pytest.mark.unit def test_keyboard_interrupt_propagates(self, tmp_path, monkeypatch): """SIGINT during a phase surfaces as KeyboardInterrupt (exit 130), not a @@ -534,6 +553,7 @@ def test_strips_accuracy_datasets_from_phase_config(self, tmp_path, monkeypatch) config.datasets = [perf_ds, acc_ds] bench = MagicMock() + bench.run_timed_out = False bench.report = None # abort after the first phase's with_updates call self._patch_phase(monkeypatch, num_samples=100, bench=bench) @@ -566,6 +586,7 @@ def test_verify_zero_qps_raises_execution_error_not_bare_valueerror( report.qps = 0.0 report.n_samples_completed = 0 bench = MagicMock() + bench.run_timed_out = False bench.report = report self._patch_phase(monkeypatch, num_samples=100, bench=bench) @@ -589,6 +610,7 @@ def test_tmpfs_dir_removed_after_phase(self, tmp_path, monkeypatch): report.qps = 1.0 report.n_samples_completed = 4 bench = MagicMock() + bench.run_timed_out = False bench.report = report tmpfs_dir = tmp_path / "tmpfs" tmpfs_dir.mkdir() @@ -648,6 +670,7 @@ def test_acc_mode_phase_keeps_accuracy_datasets(self, tmp_path, monkeypatch): "inference_endpoint.commands.audit.setup_benchmark", setup_spy ) bench = MagicMock() + bench.run_timed_out = False bench.report = None # abort right after setup, before finalize matters bench.tmpfs_dir = Path("/nonexistent-tmpfs-path-for-tests") monkeypatch.setattr( diff --git a/tests/unit/config/test_schema.py b/tests/unit/config/test_schema.py index 1e4b977aa..949670e59 100644 --- a/tests/unit/config/test_schema.py +++ b/tests/unit/config/test_schema.py @@ -353,32 +353,19 @@ def test_online_max_throughput_rejected(self): ) @pytest.mark.unit - def test_negative_min_duration_rejected(self): - with pytest.raises(ValueError, match="greater than or equal to 0"): + def test_max_duration_zero_rejected(self): + with pytest.raises(ValueError, match="greater than 0"): BenchmarkConfig( type=TestType.OFFLINE, model_params={"name": "M"}, endpoint_config={"endpoints": ["http://x"]}, datasets=[{"path": "D"}], - settings={"runtime": {"min_duration_ms": -1}}, - ) - - @pytest.mark.unit - def test_max_lt_min_duration_rejected(self): - with pytest.raises(ValueError, match="max_duration_ms"): - BenchmarkConfig( - type=TestType.OFFLINE, - model_params={"name": "M"}, - endpoint_config={"endpoints": ["http://x"]}, - datasets=[{"path": "D"}], - settings={ - "runtime": {"min_duration_ms": 5000, "max_duration_ms": 1000} - }, + settings={"runtime": {"max_duration_ms": 0}}, ) @pytest.mark.unit def test_max_duration_below_zero_rejected(self): - with pytest.raises(ValueError, match="greater than or equal to 0"): + with pytest.raises(ValueError, match="greater than 0"): BenchmarkConfig( type=TestType.OFFLINE, model_params={"name": "M"}, @@ -525,7 +512,7 @@ def test_redact_secret_fields_scrubs_url_credentials(self): assert redacted["description"] == value["description"] @pytest.mark.unit - def test_max_duration_zero_converts_to_none_in_runtime_settings(self): + def test_max_duration_defaults_to_none_in_runtime_settings(self): from inference_endpoint.config.runtime_settings import RuntimeSettings config = BenchmarkConfig( @@ -533,7 +520,6 @@ def test_max_duration_zero_converts_to_none_in_runtime_settings(self): model_params={"name": "M"}, endpoint_config={"endpoints": ["http://x"]}, datasets=[{"path": "D"}], - settings={"runtime": {"max_duration_ms": 0}}, ) rt = RuntimeSettings.from_config(config, dataloader_num_samples=100) assert rt.max_duration_ms is None @@ -900,7 +886,7 @@ class TestAgenticInferenceTotalSamples: """Tests for total_samples_to_issue() with agentic_inference load pattern.""" @pytest.mark.unit - def test_agentic_inference_uses_dataset_size_ignoring_duration(self): + def test_agentic_inference_uses_dataset_size(self): config = BenchmarkConfig( type=TestType.ONLINE, model_params={"name": "M"}, @@ -908,7 +894,6 @@ def test_agentic_inference_uses_dataset_size_ignoring_duration(self): datasets=[{"path": "D", "agentic_inference": {}}], settings={ "load_pattern": {"type": "agentic_inference", "target_concurrency": 4}, - "runtime": {"min_duration_ms": 600000}, }, ) rt = RuntimeSettings.from_config(config, dataloader_num_samples=4316) diff --git a/tests/unit/config/test_timeouts.py b/tests/unit/config/test_timeouts.py new file mode 100644 index 000000000..0c61b6d6a --- /dev/null +++ b/tests/unit/config/test_timeouts.py @@ -0,0 +1,294 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the consolidated ``settings.timeouts`` block (Timeouts model), +the reworked ``runtime.max_duration_ms`` knob, and the hard removal of the +pre-consolidation config surface (``settings.drain``, top-level ``timeout``, +``settings.service_ready_timeout_s``, ``runtime.min_duration_ms``, and the +``settings.client.worker_*`` knobs).""" + +import random + +import pytest +import yaml +from inference_endpoint.config.runtime_settings import RuntimeSettings +from inference_endpoint.config.schema import ( + BenchmarkConfig, + LoadPattern, + LoadPatternType, + RuntimeConfig, + TestType, +) +from inference_endpoint.config.timeouts import Timeouts +from inference_endpoint.metrics.metric import Throughput +from pydantic import ValidationError + +_MINIMAL_KWARGS = { + "type": TestType.OFFLINE, + "model_params": {"name": "M"}, + "endpoint_config": {"endpoints": ["http://x"]}, + "datasets": [{"path": "D"}], +} + + +class TestTimeoutsDefaults: + @pytest.mark.unit + def test_defaults(self): + cfg = Timeouts() + assert cfg.run_timeout_s is None + assert cfg.service_ready_timeout_s == 30.0 + assert cfg.warmup_drain_timeout_s == 240.0 + assert cfg.performance_drain_timeout_s is None + assert cfg.accuracy_drain_timeout_s is None + assert cfg.metrics_drain_timeout_s is None + assert cfg.worker_initialization_timeout_s == 60.0 + assert cfg.worker_graceful_shutdown_wait_s == 0.5 + assert cfg.worker_force_kill_timeout_s == 0.5 + + @pytest.mark.unit + def test_mounted_on_settings_by_default(self): + config = BenchmarkConfig(**_MINIMAL_KWARGS) + assert config.settings.timeouts == Timeouts() + + @pytest.mark.unit + def test_metrics_tokenizer_workers_is_flat_settings_field(self): + config = BenchmarkConfig(**_MINIMAL_KWARGS) + assert config.settings.metrics_tokenizer_workers == 4 + + +class TestTimeoutsValidation: + @pytest.mark.unit + @pytest.mark.parametrize( + "field", + [ + "run_timeout_s", + "warmup_drain_timeout_s", + "performance_drain_timeout_s", + "accuracy_drain_timeout_s", + "metrics_drain_timeout_s", + ], + ) + @pytest.mark.parametrize("value", [0, -1.0]) + def test_deadline_must_be_positive_or_none(self, field, value): + # The 0-sentinel is dead: unlimited is spelled None, never 0. + with pytest.raises(ValidationError): + Timeouts(**{field: value}) + + @pytest.mark.unit + @pytest.mark.parametrize( + "field", + [ + "run_timeout_s", + "warmup_drain_timeout_s", + "performance_drain_timeout_s", + "accuracy_drain_timeout_s", + "metrics_drain_timeout_s", + ], + ) + def test_deadline_none_means_unlimited(self, field): + assert getattr(Timeouts(**{field: None}), field) is None + + @pytest.mark.unit + @pytest.mark.parametrize( + "field", + [ + "service_ready_timeout_s", + "worker_initialization_timeout_s", + "worker_graceful_shutdown_wait_s", + "worker_force_kill_timeout_s", + ], + ) + def test_ge_zero_fields_accept_zero_reject_negative(self, field): + assert getattr(Timeouts(**{field: 0}), field) == 0.0 + with pytest.raises(ValidationError): + Timeouts(**{field: -1.0}) + + @pytest.mark.unit + def test_extra_fields_rejected(self): + with pytest.raises(ValidationError): + Timeouts(unknown_field=1) + + +class TestDeletedConfigSurface: + """Hard cutover: the pre-consolidation keys must error, not silently pass.""" + + @pytest.mark.unit + def test_settings_drain_block_rejected(self): + with pytest.raises(ValidationError, match="drain"): + BenchmarkConfig( + **_MINIMAL_KWARGS, + settings={"drain": {"warmup_timeout_s": 10.0}}, + ) + + @pytest.mark.unit + def test_settings_service_ready_timeout_rejected(self): + with pytest.raises(ValidationError, match="service_ready_timeout_s"): + BenchmarkConfig( + **_MINIMAL_KWARGS, + settings={"service_ready_timeout_s": 10.0}, + ) + + @pytest.mark.unit + def test_runtime_min_duration_rejected(self): + with pytest.raises(ValidationError, match="min_duration_ms"): + BenchmarkConfig( + **_MINIMAL_KWARGS, + settings={"runtime": {"min_duration_ms": 1000}}, + ) + + @pytest.mark.unit + def test_top_level_timeout_rejected(self): + with pytest.raises(ValidationError, match="timeout"): + BenchmarkConfig(**_MINIMAL_KWARGS, timeout=42.0) + + @pytest.mark.unit + def test_client_worker_knob_rejected(self): + with pytest.raises(ValidationError, match="worker_initialization_timeout"): + BenchmarkConfig( + **_MINIMAL_KWARGS, + settings={"client": {"worker_initialization_timeout": 120.0}}, + ) + + @pytest.mark.unit + def test_client_worker_knob_rejected_from_yaml(self, tmp_path): + yaml_content = """ +type: "offline" +model_params: + name: "test-model" +endpoint_config: + endpoints: ["http://test:8000"] +datasets: + - path: "test.jsonl" +settings: + client: + worker_initialization_timeout: 120 +""" + config_file = tmp_path / "stale.yaml" + config_file.write_text(yaml_content) + with pytest.raises(ValidationError, match="worker_initialization_timeout"): + BenchmarkConfig.from_yaml_file(config_file) + + +class TestWorkerFieldsHiddenFromSerialization: + @pytest.mark.unit + def test_yaml_roundtrip_excludes_worker_carrier_fields(self, tmp_path): + """The runtime-carrier worker fields on the client never serialize, so + a persisted config reloads cleanly under extra=forbid.""" + config = BenchmarkConfig(**_MINIMAL_KWARGS) + out = tmp_path / "roundtrip.yaml" + config.to_yaml_file(out) + + dumped = yaml.safe_load(out.read_text()) + client_block = dumped.get("settings", {}).get("client", {}) or {} + carrier_fields = { + "worker_initialization_timeout_s", + "worker_graceful_shutdown_wait_s", + "worker_force_kill_timeout_s", + } + assert not carrier_fields & client_block.keys() + + loaded = BenchmarkConfig.from_yaml_file(out) + assert loaded.settings.timeouts == config.settings.timeouts + + +class TestTimeoutsYAMLRoundtrip: + @pytest.mark.unit + def test_yaml_block_loads(self, tmp_path): + yaml_content = """ +type: "offline" +model_params: + name: "test-model" +endpoint_config: + endpoints: ["http://test:8000"] +datasets: + - path: "test.jsonl" +settings: + timeouts: + run_timeout_s: 900 + warmup_drain_timeout_s: 12.5 + performance_drain_timeout_s: 30.0 + accuracy_drain_timeout_s: null + metrics_drain_timeout_s: 300.0 + worker_initialization_timeout_s: 90 +""" + config_file = tmp_path / "timeouts.yaml" + config_file.write_text(yaml_content) + config = BenchmarkConfig.from_yaml_file(config_file) + timeouts = config.settings.timeouts + assert timeouts.run_timeout_s == 900.0 + assert timeouts.warmup_drain_timeout_s == 12.5 + assert timeouts.performance_drain_timeout_s == 30.0 + assert timeouts.accuracy_drain_timeout_s is None + assert timeouts.metrics_drain_timeout_s == 300.0 + assert timeouts.worker_initialization_timeout_s == 90.0 + + +class TestMaxDurationSuffix: + """max_duration_ms keeps the duration suffix parser (600s, 10m, plain ms).""" + + @pytest.mark.unit + @pytest.mark.parametrize( + "value, expected_ms", + [ + ("600s", 600000), + ("10m", 600000), + ("600000ms", 600000), + ("600000", 600000), + (600000, 600000), + ("0.5m", 30000), + ("1.5s", 1500), + ], + ) + def test_suffix_parses(self, value, expected_ms): + cfg = RuntimeConfig(max_duration_ms=value) + assert cfg.max_duration_ms == expected_ms + + @pytest.mark.unit + def test_default_is_none(self): + assert RuntimeConfig().max_duration_ms is None + + @pytest.mark.unit + @pytest.mark.parametrize("value", [0, -1, "0s"]) + def test_zero_and_negative_rejected(self, value): + # No 0-sentinel: "no cap" is spelled None. + with pytest.raises(ValidationError): + RuntimeConfig(max_duration_ms=value) + + +class TestDatasetOnceDefault: + @pytest.mark.unit + def test_sample_count_defaults_to_dataset_size(self): + """Without n_samples_to_issue and without any duration knob, a run + issues the dataset exactly once.""" + config = BenchmarkConfig(**_MINIMAL_KWARGS) + rt = RuntimeSettings.from_config(config, dataloader_num_samples=123) + assert rt.n_samples_to_issue is None + assert rt.total_samples_to_issue() == 123 + + @pytest.mark.unit + def test_explicit_n_samples_still_wins(self): + rt = RuntimeSettings( + metric_target=Throughput(10.0), + reported_metrics=[Throughput(10.0)], + min_duration_ms=0, + max_duration_ms=None, + n_samples_from_dataset=123, + n_samples_to_issue=7, + min_sample_count=1, + rng_sched=random.Random(0), + rng_sample_index=random.Random(0), + load_pattern=LoadPattern(type=LoadPatternType.MAX_THROUGHPUT), + ) + assert rt.total_samples_to_issue() == 7 diff --git a/tests/unit/config/test_yaml_loader.py b/tests/unit/config/test_yaml_loader.py index 0b0e8d473..d1d3d5181 100644 --- a/tests/unit/config/test_yaml_loader.py +++ b/tests/unit/config/test_yaml_loader.py @@ -45,13 +45,12 @@ def test_load_valid_yaml(self, tmp_path): path: "test.jsonl" settings: - runtime: - min_duration_ms: 60000 + timeouts: + worker_initialization_timeout_s: 120 load_pattern: type: "max_throughput" client: num_workers: 4 - worker_initialization_timeout: 120 transport: type: zmq recv_buffer_size: 16777216 @@ -68,7 +67,7 @@ def test_load_valid_yaml(self, tmp_path): assert config.name == "test-config" assert config.type == BenchmarkTestType.OFFLINE assert len(config.datasets) == 1 - assert config.settings.client.worker_initialization_timeout == 120.0 + assert config.settings.timeouts.worker_initialization_timeout_s == 120.0 assert config.settings.client.transport.recv_buffer_size == 16777216 assert config.settings.client.transport.send_buffer_size == 8388608 @@ -209,7 +208,7 @@ def test_create_default_offline_config(self): config = BenchmarkConfig.create_default_config(BenchmarkTestType.OFFLINE) assert isinstance(config, BenchmarkConfig) assert config.settings.load_pattern.type == LoadPatternType.MAX_THROUGHPUT - assert config.settings.runtime.min_duration_ms == 600000 + assert config.settings.timeouts.run_timeout_s is None assert config.settings.client.num_workers >= 1 # auto-resolved from -1 def test_create_default_online_config(self): @@ -217,7 +216,7 @@ def test_create_default_online_config(self): assert isinstance(config, BenchmarkConfig) assert config.settings.load_pattern.type == LoadPatternType.POISSON assert config.settings.load_pattern.target_qps == 10.0 - assert config.settings.runtime.min_duration_ms == 600000 + assert config.settings.timeouts.run_timeout_s is None def test_create_default_eval_not_implemented(self): with pytest.raises(CLIError, match="EVAL"): @@ -246,8 +245,8 @@ def test_serialize_deserialize_roundtrip(self, tmp_path): ) assert loaded.settings.load_pattern.type == original.settings.load_pattern.type assert ( - loaded.settings.client.worker_initialization_timeout - == original.settings.client.worker_initialization_timeout + loaded.settings.timeouts.worker_initialization_timeout_s + == original.settings.timeouts.worker_initialization_timeout_s ) assert ( loaded.settings.client.transport.recv_buffer_size diff --git a/tests/unit/scripts/test_metrics_preflight_tap.py b/tests/unit/scripts/test_metrics_preflight_tap.py new file mode 100644 index 000000000..9fdedbe69 --- /dev/null +++ b/tests/unit/scripts/test_metrics_preflight_tap.py @@ -0,0 +1,410 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Focused tests for the experiment-only GPT-OSS metrics preflight tap.""" + +# ruff: noqa: I001 +# Keep import layout stable across the pinned pre-commit and local uv ruff. + +from __future__ import annotations + +import csv +import importlib.util +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest +from inference_endpoint.async_utils.services.metrics_aggregator.snapshot import ( + CounterStat, + MetricsSnapshot, + MetricsSnapshotCodec, + SessionState, +) +from inference_endpoint.core.record import TOPIC_FRAME_SIZE + +pytestmark = pytest.mark.unit + + +def _load_tap(): + path = Path("scratchpad/gptoss_nvl144_pr334_vvv_20260728/metrics_preflight_tap.py") + spec = importlib.util.spec_from_file_location("metrics_preflight_tap", path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +tap = _load_tap() + + +def _write_process( + proc_root: Path, + *, + pid: int, + ppid: int, + argv: list[str], + status: str = "VmRSS:\t123 kB\nVmHWM:\t456 kB\n", +) -> None: + proc_dir = proc_root / str(pid) + proc_dir.mkdir() + (proc_dir / "stat").write_text(f"{pid} (command with spaces) S {ppid} 0 0\n") + (proc_dir / "cmdline").write_bytes(b"\0".join(a.encode() for a in argv) + b"\0") + (proc_dir / "status").write_text(status) + + +def _snapshot( + counter: int, + *, + state: SessionState = SessionState.LIVE, + pending: int = 0, + issued: int = 10, + completed: int = 5, +) -> MetricsSnapshot: + return MetricsSnapshot( + counter=counter, + timestamp_ns=counter * 100, + state=state, + n_pending_tasks=pending, + metrics=[ + CounterStat("total_samples_issued", issued), + CounterStat("total_samples_completed", completed), + CounterStat("total_samples_failed", 0), + ], + ) + + +class TestProcessDiscovery: + def test_finds_only_aggregator_below_root(self, tmp_path: Path) -> None: + proc = tmp_path / "proc" + proc.mkdir() + _write_process(proc, pid=100, ppid=1, argv=["benchmark"]) + _write_process(proc, pid=101, ppid=100, argv=["worker"]) + _write_process( + proc, + pid=102, + ppid=101, + argv=[ + "python", + "-m", + tap.AGGREGATOR_MODULE, + "--socket-dir", + "/dev/shm/zmq_a", + "--metrics-socket=metrics_a", + ], + ) + _write_process( + proc, + pid=200, + ppid=1, + argv=["python", "-m", tap.AGGREGATOR_MODULE], + ) + + found = tap.find_aggregator_descendant(100, proc) + + assert found is not None + assert found.pid == 102 + assert tap.parse_aggregator_socket_args(found.argv) == ( + "/dev/shm/zmq_a", + "metrics_a", + ) + assert ( + tap.metrics_ipc_address("/dev/shm/zmq_a", "metrics_a") + == "ipc:///dev/shm/zmq_a/metrics_a" + ) + + def test_missing_socket_arg_is_rejected(self) -> None: + with pytest.raises(ValueError, match="metrics-socket"): + tap.parse_aggregator_socket_args(["--socket-dir", "/tmp/x"]) + + +class TestSampling: + def test_reads_proc_cgroup_meminfo_and_tmpfs(self, tmp_path: Path) -> None: + proc = tmp_path / "proc" + proc.mkdir() + _write_process(proc, pid=42, ppid=1, argv=["aggregator"]) + (proc / "42" / "cgroup").write_text("0::/job/step\n") + (proc / "meminfo").write_text( + "MemTotal: 10000 kB\nMemAvailable: 2500 kB\n" + ) + + cgroup_root = tmp_path / "cgroup" + cgroup = cgroup_root / "job" / "step" + cgroup.mkdir(parents=True) + (cgroup / "memory.current").write_text("1000\n") + (cgroup / "memory.peak").write_text("2000\n") + (cgroup / "memory.max").write_text("3000\n") + (cgroup / "memory.events").write_text("oom 2\noom_kill 1\n") + + events = tmp_path / "benchmark_1" / "events" + events.mkdir(parents=True) + (events / "events.jsonl").write_bytes(b"x" * 17) + + location = tap._find_cgroup(42, proc, cgroup_root) + obs = tap.sample_memory( + 42, + location, + str(tmp_path / "benchmark_*" / "events" / "events.jsonl"), + proc, + ) + + assert obs == tap.MemoryObservation( + aggregator_alive=True, + rss_kib=123, + hwm_kib=456, + cgroup_current_bytes=1000, + cgroup_peak_bytes=2000, + cgroup_max_bytes=3000, + cgroup_oom=2, + cgroup_oom_kill=1, + mem_available_kib=2500, + mem_total_kib=10000, + tmpfs_event_files=1, + tmpfs_events_bytes=17, + ) + + +class TestSnapshotsAndArtifacts: + def test_decodes_frame_and_tracks_pending_memory_high_water( + self, tmp_path: Path + ) -> None: + codec = MetricsSnapshotCodec() + first = _snapshot(1, pending=3) + second = _snapshot( + 4, + state=SessionState.DRAINING, + pending=7, + issued=20, + completed=20, + ) + topic, payload = codec.encode(first) + assert len(topic) == TOPIC_FRAME_SIZE + assert tap.decode_metrics_frame(topic + payload, codec) == first + + stats = tap.MonitorStats( + started_wall_ns=100, + started_monotonic_ns=100, + root_pid=1, + aggregator_pid=42, + ) + stats.observe_snapshot(first, 1_000_000_000) + stats.observe_snapshot(second, 3_500_000_000) + stats.observe_memory( + tap.MemoryObservation( + aggregator_alive=True, + rss_kib=11, + hwm_kib=12, + cgroup_current_bytes=13, + cgroup_peak_bytes=14, + cgroup_max_bytes=15, + cgroup_oom=0, + cgroup_oom_kill=0, + mem_available_kib=16, + mem_total_kib=17, + tmpfs_event_files=1, + tmpfs_events_bytes=18, + ) + ) + + summary = stats.to_dict(ended_wall_ns=4_000_000_000, csv_path=tmp_path / "x") + assert summary["published_pending_high_water"] == 7 + assert summary["pending_at_first_draining"] == 7 + assert summary["counter_gap_total"] == 2 + assert summary["counter_gap_max"] == 2 + assert summary["max_snapshot_gap_s"] == 2.5 + assert summary["aggregator_rss_high_water_kib"] == 11 + assert summary["tmpfs_events_high_water_bytes"] == 18 + assert summary["telemetry_capture_valid"] is True + assert summary["telemetry_capture_failures"] == [] + + def test_capture_gate_requires_rss_and_oom_counters(self) -> None: + stats = tap.MonitorStats( + started_wall_ns=100, + started_monotonic_ns=100, + root_pid=1, + aggregator_pid=42, + cgroup_version=2, + snapshots_received=2, + published_pending_high_water=7, + aggregator_reported_hwm_high_water_kib=12, + cgroup_memory_current_high_water_bytes=13, + cgroup_memory_peak_high_water_bytes=14, + ) + + assert tap.telemetry_capture_failures(stats) == [ + "aggregator_rss_missing", + "cgroup_oom_missing", + "cgroup_oom_kill_missing", + ] + + stats.cgroup_version = 1 + assert tap.telemetry_capture_failures(stats) == [ + "aggregator_rss_missing", + "cgroup_oom_missing", + ] + + def test_csv_finalization_and_atomic_summary(self, tmp_path: Path) -> None: + csv_path = tmp_path / "telemetry.csv" + artifact = tap.AtomicCsv(csv_path, fsync_interval_s=0) + artifact.open() + row = dict.fromkeys(tap.CSV_FIELDS, "") + row["row_kind"] = "memory" + artifact.write(row) + artifact.finalize() + + with csv_path.open(newline="") as f: + rows = list(csv.DictReader(f)) + assert len(rows) == 1 + assert rows[0]["row_kind"] == "memory" + assert not csv_path.with_suffix(".csv.part").exists() + + summary_path = tmp_path / "summary.json" + payload = {"status": "complete"} + from inference_endpoint.utils.atomic_write import atomic_write_bytes + + atomic_write_bytes( + summary_path, (json.dumps(payload, sort_keys=True) + "\n").encode() + ) + assert json.loads(summary_path.read_text()) == payload + + def test_missing_aggregator_is_nonzero_and_still_atomic( + self, tmp_path: Path + ) -> None: + csv_path = tmp_path / "telemetry.csv" + summary_path = tmp_path / "summary.json" + args = tap._build_parser().parse_args( + [ + "--root-pid", + str(2**31 - 1), + "--csv", + str(csv_path), + "--summary", + str(summary_path), + "--discover-timeout-s", + "0", + ] + ) + + exit_code, summary = tap.run(args) + + assert exit_code == 2 + assert summary["status"] == "aggregator_not_found" + assert summary["telemetry_capture_valid"] is False + assert "aggregator_not_found" in summary["telemetry_capture_failures"] + assert csv_path.is_file() + assert summary_path.is_file() + assert not csv_path.with_suffix(".csv.part").exists() + + def test_end_to_end_discovers_and_taps_metrics_pub( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + socket_dir = tmp_path / "sockets" + socket_dir.mkdir() + socket_name = "metrics_test" + child_code = """ +import sys +import time +import zmq +from inference_endpoint.async_utils.services.metrics_aggregator.snapshot import ( + CounterStat, MetricsSnapshot, MetricsSnapshotCodec, SessionState, +) + +args = sys.argv[1:] +socket_dir = args[args.index("--socket-dir") + 1] +socket_name = args[args.index("--metrics-socket") + 1] +ctx = zmq.Context() +sock = ctx.socket(zmq.PUB) +sock.setsockopt(zmq.LINGER, 0) +sock.bind(f"ipc://{socket_dir}/{socket_name}") +codec = MetricsSnapshotCodec() +time.sleep(0.5) +for i in range(1, 7): + snap = MetricsSnapshot( + counter=i, + timestamp_ns=i, + state=SessionState.LIVE, + n_pending_tasks=i, + metrics=[ + CounterStat("total_samples_issued", i), + CounterStat("total_samples_completed", i), + CounterStat("total_samples_failed", 0), + ], + ) + topic, payload = codec.encode(snap) + sock.send(topic + payload) + time.sleep(0.15) +sock.close(0) +ctx.term() +""" + child = subprocess.Popen( + [ + sys.executable, + "-c", + child_code, + tap.AGGREGATOR_MODULE, + "--socket-dir", + str(socket_dir), + "--metrics-socket", + socket_name, + ] + ) + observation = tap.MemoryObservation( + aggregator_alive=True, + rss_kib=100, + hwm_kib=200, + cgroup_current_bytes=300, + cgroup_peak_bytes=400, + cgroup_max_bytes=500, + cgroup_oom=0, + cgroup_oom_kill=0, + mem_available_kib=600, + mem_total_kib=700, + tmpfs_event_files=0, + tmpfs_events_bytes=0, + ) + monkeypatch.setattr(tap, "sample_memory", lambda *args, **kwargs: observation) + csv_path = tmp_path / "telemetry.csv" + summary_path = tmp_path / "summary.json" + args = tap._build_parser().parse_args( + [ + "--root-pid", + str(os.getpid()), + "--csv", + str(csv_path), + "--summary", + str(summary_path), + "--discover-timeout-s", + "5", + "--discover-poll-s", + "0.02", + "--sample-interval-s", + "0.05", + "--poll-timeout-ms", + "20", + "--post-aggregator-exit-s", + "0.1", + "--fsync-interval-s", + "0", + ] + ) + try: + exit_code, summary = tap.run(args) + finally: + child.wait(timeout=5) + + assert exit_code == 0 + assert summary["status"] == "aggregator_exited" + assert summary["telemetry_capture_valid"] is True + assert summary["snapshots_received"] >= 2 + assert summary["published_pending_high_water"] >= 2 + assert summary["aggregator_reported_hwm_high_water_kib"] == 200 + assert summary["cgroup_memory_current_high_water_bytes"] == 300 + assert summary["cgroup_memory_peak_high_water_bytes"] == 400 + assert csv_path.is_file() + assert summary_path.is_file() + with csv_path.open(newline="") as handle: + rows = list(csv.DictReader(handle)) + assert rows[-1]["row_kind"] == "terminal_memory" From 628b30baf00d7749aa97c0d7f9786130ab2851df Mon Sep 17 00:00:00 2001 From: Viraat Chandra Date: Thu, 13 Aug 2026 13:11:52 -0700 Subject: [PATCH 2/9] feat(metrics): fail the run when the metrics drain times out with pending tokenization An expired metrics_drain_timeout_s finalizes the aggregator as COMPLETE with n_pending_tasks > 0; previously that exited 0 with complete: false buried in result_summary.json. run_benchmark now raises ExecutionError after the artifacts are written, so partial ISL/OSL/TPOT stats can never look like a clean run. (The audit path already refused to certify these.) --- AGENTS.md | 2 +- docs/CLI_QUICK_REFERENCE.md | 24 +++++----- .../commands/benchmark/execute.py | 17 +++++++ .../templates/concurrency_template_full.yaml | 2 +- .../templates/offline_template_full.yaml | 2 +- .../templates/online_template_full.yaml | 2 +- src/inference_endpoint/config/timeouts.py | 5 +- .../integration/commands/test_run_timeout.py | 47 +++++++++++++++++++ 8 files changed, 83 insertions(+), 18 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4c41fde22..e2432545d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -118,7 +118,7 @@ The aggregator is a separate process (`python -m inference_endpoint.async_utils. - **Series storage**: each `SeriesSampler` keeps three parallel views: O(1) cheap rollups (count/total/min/max/sum_sq, exact), an HDR Histogram (cheap live percentiles), and an in-memory `array.array` of raw values (for exact percentiles in the `COMPLETE` snapshot). Hot path is `registry.record(name, value)` — no allocation, no I/O. - **Counter API**: `registry.increment(name, delta=1)` for sample-event counters. `registry.set_counter(name, value)` only for the three derived-duration counters (`total_duration_ns` max-of-elapsed, `tracked_duration_ns` sum-of-blocks, `legacy_loadgen_window_duration_ns` first-issue→last-issued-completion span for LoadGen-parity QPS/TPS). -- **Lifecycle**: `INITIALIZE` (constructed, awaiting first `STARTED`) → `LIVE` (run in progress, ticking every `--publish-interval` seconds) → `DRAINING` (set on `ENDED`; tick continues; bounded by the `--drain-timeout` budget — argv 0 = unlimited; schema `settings.timeouts.metrics_drain_timeout_s` uses None = unlimited, converted at the argv boundary) → terminal: `COMPLETE` (clean end via `publish_final`, exact stats) **or** `INTERRUPTED` (signal-handler-triggered final via SIGTERM/SIGINT; best-effort partial stats). Drain timeout detected by consumers as `state == COMPLETE and n_pending_tasks > 0`; interrupted runs are detected as `state == INTERRUPTED` directly. +- **Lifecycle**: `INITIALIZE` (constructed, awaiting first `STARTED`) → `LIVE` (run in progress, ticking every `--publish-interval` seconds) → `DRAINING` (set on `ENDED`; tick continues; bounded by the `--drain-timeout` budget — argv 0 = unlimited; schema `settings.timeouts.metrics_drain_timeout_s` uses None = unlimited, converted at the argv boundary) → terminal: `COMPLETE` (clean end via `publish_final`, exact stats) **or** `INTERRUPTED` (signal-handler-triggered final via SIGTERM/SIGINT; best-effort partial stats). Drain timeout detected by consumers as `state == COMPLETE and n_pending_tasks > 0` — `run_benchmark` fails the run on it (artifacts written with `complete: false`, then non-zero exit); interrupted runs are detected as `state == INTERRUPTED` directly. - **Final delivery is dual-path with separated concerns**: `publish_final` atomically writes `final_snapshot.json` (`tmp + fsync(file) + rename + fsync(parent_dir)`) — this is the **primary** Report source — AND emits the terminal-state snapshot over pub/sub as a TUI shutdown signal. Each path is wrapped in its own try/except so one failure cannot suppress the other. Main process consumer reads `final_snapshot.json` (via `json.loads` to dict, no Struct decode); falls back to the subscriber's `latest` live snapshot only if the file is missing (e.g. SIGKILL / OOM before the signal handler ran). The dict form is the canonical consumer contract (see `snapshot_to_dict`). - **Early stopping (on by default)**: series registered with `register_series(..., tail_latency=True)` (today ttft/tpot/latency) get MLPerf early-stopping percentile estimates on the COMPLETE (exact) snapshot — a compact `early_stopping_percentiles` map in `result_summary.json` whose keys mirror the `percentiles` grid (≥ p50) with estimate-or-`null` values; rich detail is INFO-logged. On by default (cold-path only; the exact path shares one in-place sort between the percentile grid and the estimates); `settings.early_stopping.enabled: false` / `--no-early-stopping` opts out. Confidence/tolerance are LoadGen constants. Pure math in `metrics/early_stopping.py`; post-hoc recomputation from any run's `events.jsonl` via `scripts/early_stopping_estimate_from_events.py`. See docs/early_stopping.md. - **Histogram bucket edges are dynamic per snapshot**: log-spaced over the observed `[min, max]`. Bucket count is fixed at construction; consumers MUST re-render from the snapshot's `(lo, hi, count)` triples each frame and MUST NOT track bucket-by-index across snapshots. diff --git a/docs/CLI_QUICK_REFERENCE.md b/docs/CLI_QUICK_REFERENCE.md index 6f69d7b6b..c1d55dc5f 100644 --- a/docs/CLI_QUICK_REFERENCE.md +++ b/docs/CLI_QUICK_REFERENCE.md @@ -122,18 +122,18 @@ Flag names shown as `--full.dotted.path --alias`. Both forms work. All give-up deadlines live under `settings.timeouts`; the only workload duration is `settings.runtime.max_duration_ms`. `null`/unset means "wait indefinitely" (or "off") everywhere. -| YAML path | CLI flag | Semantics | -| --------------------------------------------------- | ----------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | -| `settings.runtime.max_duration_ms` | `--runtime.max-duration-ms` | Caps the performance phase (ms, or suffix: `600s`, `10m`); reaching it ends the phase NORMALLY — the report stays valid | -| `settings.timeouts.run_timeout_s` | `--timeout` | Whole-run watchdog; firing aborts the entire run — report marked INTERRUPTED, non-zero exit | -| `settings.timeouts.service_ready_timeout_s` | `--service-ready-timeout` | Wait for the metrics-aggregator/event-logger services to become ready (default 30) | -| `settings.timeouts.warmup_drain_timeout_s` | `--warmup-drain-timeout` | Bound on in-flight warmup requests after the warmup phase ends (default 240) | -| `settings.timeouts.performance_drain_timeout_s` | `--performance-drain-timeout` | Bound on in-flight performance requests after the phase ends (default: wait indefinitely) | -| `settings.timeouts.accuracy_drain_timeout_s` | `--accuracy-drain-timeout` | Bound on in-flight accuracy requests after the phase ends (default: wait indefinitely) | -| `settings.timeouts.metrics_drain_timeout_s` | `--metrics-drain-timeout` | Budget for the metrics aggregator to finish tokenizing buffered samples after the run ends (default: wait indefinitely) | -| `settings.timeouts.worker_initialization_timeout_s` | `--worker-initialization-timeout-s` | Wait for endpoint-client worker processes to start (default 60) | -| `settings.timeouts.worker_graceful_shutdown_wait_s` | `--worker-graceful-shutdown-wait-s` | Post-run wait for workers to exit gracefully (default 0.5) | -| `settings.timeouts.worker_force_kill_timeout_s` | `--worker-force-kill-timeout-s` | Wait after SIGTERM before SIGKILL during worker teardown (default 0.5) | +| YAML path | CLI flag | Semantics | +| --------------------------------------------------- | ----------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `settings.runtime.max_duration_ms` | `--runtime.max-duration-ms` | Caps the performance phase (ms, or suffix: `600s`, `10m`); reaching it ends the phase NORMALLY — the report stays valid | +| `settings.timeouts.run_timeout_s` | `--timeout` | Whole-run watchdog; firing aborts the entire run — report marked INTERRUPTED, non-zero exit | +| `settings.timeouts.service_ready_timeout_s` | `--service-ready-timeout` | Wait for the metrics-aggregator/event-logger services to become ready (default 30) | +| `settings.timeouts.warmup_drain_timeout_s` | `--warmup-drain-timeout` | Bound on in-flight warmup requests after the warmup phase ends (default 240) | +| `settings.timeouts.performance_drain_timeout_s` | `--performance-drain-timeout` | Bound on in-flight performance requests after the phase ends (default: wait indefinitely) | +| `settings.timeouts.accuracy_drain_timeout_s` | `--accuracy-drain-timeout` | Bound on in-flight accuracy requests after the phase ends (default: wait indefinitely) | +| `settings.timeouts.metrics_drain_timeout_s` | `--metrics-drain-timeout` | Budget for the metrics aggregator to finish tokenizing buffered samples after the run ends (default: wait indefinitely); expiring fails the run with `complete: false` artifacts | +| `settings.timeouts.worker_initialization_timeout_s` | `--worker-initialization-timeout-s` | Wait for endpoint-client worker processes to start (default 60) | +| `settings.timeouts.worker_graceful_shutdown_wait_s` | `--worker-graceful-shutdown-wait-s` | Post-run wait for workers to exit gracefully (default 0.5) | +| `settings.timeouts.worker_force_kill_timeout_s` | `--worker-force-kill-timeout-s` | Wait after SIGTERM before SIGKILL during worker teardown (default 0.5) | How the knobs compose: diff --git a/src/inference_endpoint/commands/benchmark/execute.py b/src/inference_endpoint/commands/benchmark/execute.py index 239159584..ac9fe547d 100644 --- a/src/inference_endpoint/commands/benchmark/execute.py +++ b/src/inference_endpoint/commands/benchmark/execute.py @@ -1314,6 +1314,23 @@ def run_benchmark( f"Run timeout ({run_timeout_s}s) reached; run aborted and " "report marked INTERRUPTED" ) + if ( + bench.report is not None + and bench.report.state == "complete" + and not bench.report.complete + ): + # The aggregator gave up on its tokenization backlog when + # metrics_drain_timeout_s expired (state "complete" with pending + # tasks). The artifacts above are already written with + # complete: false; fail loudly instead of exiting 0 on partial + # ISL/OSL/TPOT stats. + raise ExecutionError( + "Metrics drain timed out " + f"(metrics_drain_timeout_s=" + f"{config.settings.timeouts.metrics_drain_timeout_s}): " + "tokenization did not finish before the deadline; report is " + "partial (complete: false in result_summary.json)" + ) except KeyboardInterrupt: # Salvage results (finally), then propagate to main.py -> exit 130. logger.warning("Benchmark interrupted by user") diff --git a/src/inference_endpoint/config/templates/concurrency_template_full.yaml b/src/inference_endpoint/config/templates/concurrency_template_full.yaml index d2c3efafe..97a13f6be 100644 --- a/src/inference_endpoint/config/templates/concurrency_template_full.yaml +++ b/src/inference_endpoint/config/templates/concurrency_template_full.yaml @@ -86,7 +86,7 @@ settings: warmup_drain_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) performance_drain_timeout_s: null # Performance drain timeout in seconds (None = wait indefinitely) accuracy_drain_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely; accuracy is unbounded by default because every sample must complete) - metrics_drain_timeout_s: null # Wall-clock budget (seconds) to finish tokenizing buffered samples after ENDED (None = wait indefinitely). An incomplete drain is surfaced via n_pending_tasks > 0, never silently dropped. + metrics_drain_timeout_s: null # Wall-clock budget (seconds) to finish tokenizing buffered samples after ENDED (None = wait indefinitely). An incomplete drain fails the run: artifacts are written with complete: false, then run_benchmark exits non-zero. worker_initialization_timeout_s: 60.0 # Endpoint-client worker init timeout (seconds) worker_graceful_shutdown_wait_s: 0.5 # Endpoint-client post-run graceful shutdown wait (seconds) worker_force_kill_timeout_s: 0.5 # Endpoint-client force kill timeout after graceful wait (seconds) diff --git a/src/inference_endpoint/config/templates/offline_template_full.yaml b/src/inference_endpoint/config/templates/offline_template_full.yaml index 7250f2ca5..d232f702f 100644 --- a/src/inference_endpoint/config/templates/offline_template_full.yaml +++ b/src/inference_endpoint/config/templates/offline_template_full.yaml @@ -86,7 +86,7 @@ settings: warmup_drain_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) performance_drain_timeout_s: null # Performance drain timeout in seconds (None = wait indefinitely) accuracy_drain_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely; accuracy is unbounded by default because every sample must complete) - metrics_drain_timeout_s: null # Wall-clock budget (seconds) to finish tokenizing buffered samples after ENDED (None = wait indefinitely). An incomplete drain is surfaced via n_pending_tasks > 0, never silently dropped. + metrics_drain_timeout_s: null # Wall-clock budget (seconds) to finish tokenizing buffered samples after ENDED (None = wait indefinitely). An incomplete drain fails the run: artifacts are written with complete: false, then run_benchmark exits non-zero. worker_initialization_timeout_s: 60.0 # Endpoint-client worker init timeout (seconds) worker_graceful_shutdown_wait_s: 0.5 # Endpoint-client post-run graceful shutdown wait (seconds) worker_force_kill_timeout_s: 0.5 # Endpoint-client force kill timeout after graceful wait (seconds) diff --git a/src/inference_endpoint/config/templates/online_template_full.yaml b/src/inference_endpoint/config/templates/online_template_full.yaml index 8b0d18f39..aad246771 100644 --- a/src/inference_endpoint/config/templates/online_template_full.yaml +++ b/src/inference_endpoint/config/templates/online_template_full.yaml @@ -87,7 +87,7 @@ settings: warmup_drain_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) performance_drain_timeout_s: null # Performance drain timeout in seconds (None = wait indefinitely) accuracy_drain_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely; accuracy is unbounded by default because every sample must complete) - metrics_drain_timeout_s: null # Wall-clock budget (seconds) to finish tokenizing buffered samples after ENDED (None = wait indefinitely). An incomplete drain is surfaced via n_pending_tasks > 0, never silently dropped. + metrics_drain_timeout_s: null # Wall-clock budget (seconds) to finish tokenizing buffered samples after ENDED (None = wait indefinitely). An incomplete drain fails the run: artifacts are written with complete: false, then run_benchmark exits non-zero. worker_initialization_timeout_s: 60.0 # Endpoint-client worker init timeout (seconds) worker_graceful_shutdown_wait_s: 0.5 # Endpoint-client post-run graceful shutdown wait (seconds) worker_force_kill_timeout_s: 0.5 # Endpoint-client force kill timeout after graceful wait (seconds) diff --git a/src/inference_endpoint/config/timeouts.py b/src/inference_endpoint/config/timeouts.py index c767ad445..abd22c3f8 100644 --- a/src/inference_endpoint/config/timeouts.py +++ b/src/inference_endpoint/config/timeouts.py @@ -126,8 +126,9 @@ class Timeouts(WithUpdatesMixin, BaseModel): gt=0, description=( "Wall-clock budget (seconds) to finish tokenizing buffered samples " - "after ENDED (None = wait indefinitely). An incomplete drain is " - "surfaced via n_pending_tasks > 0, never silently dropped." + "after ENDED (None = wait indefinitely). An incomplete drain fails " + "the run: artifacts are written with complete: false, then " + "run_benchmark exits non-zero." ), ) worker_initialization_timeout_s: float = Field( diff --git a/tests/integration/commands/test_run_timeout.py b/tests/integration/commands/test_run_timeout.py index e4d811b01..96835c0c4 100644 --- a/tests/integration/commands/test_run_timeout.py +++ b/tests/integration/commands/test_run_timeout.py @@ -170,3 +170,50 @@ def test_run_timeout_during_metrics_drain_interrupts(mock_http_echo_server, tmp_ snapshot = _read_final_snapshot(report_dir) assert snapshot["state"] == "interrupted" + + +@pytest.mark.integration +def test_metrics_drain_timeout_fails_run(mock_http_echo_server, tmp_path): + """An expired metrics_drain_timeout_s fails the run instead of exiting 0. + + The aggregator finalizes as COMPLETE with a pending tokenization backlog + (state "complete", n_pending_tasks > 0). Artifacts must still be written + with complete: false, and run_benchmark must raise so partial ISL/OSL + stats can never look like a clean exit. + """ + dataset_path = tmp_path / "big_prompts.jsonl" + prompt = "lorem ipsum " * 21_000 # ~250 KB per sample + with dataset_path.open("w") as f: + for i in range(100): + f.write(json.dumps({"prompt": f"{i} {prompt}"}) + "\n") + + report_dir = tmp_path / "report" + config = BenchmarkConfig( + type=TestType.OFFLINE, + endpoint_config=EndpointConfig(endpoints=[mock_http_echo_server.url]), + model_params=ModelParams( + name=str(_CHAR_TOKENIZER_DIR), streaming=StreamingMode.OFF + ), + datasets=[Dataset(path=str(dataset_path), type=DatasetType.PERFORMANCE)], + report_dir=report_dir, + settings=Settings( + load_pattern=LoadPattern(type=LoadPatternType.MAX_THROUGHPUT), + client=_FAST_CLIENT, + # Defer every ISL/OSL tokenization to the end-of-run drain, then + # give the drain a budget far below the ~50M-char backlog. No run + # watchdog: the drain deadline itself must fail the run. + metrics_tokenizer_workers=0, + timeouts=Timeouts(metrics_drain_timeout_s=1.0), + warmup=WarmupConfig(enabled=False), + ), + ) + + with pytest.raises(ExecutionError, match="Metrics drain timed out"): + run_benchmark(config, TestMode.PERF) + + snapshot = _read_final_snapshot(report_dir) + assert snapshot["state"] == "complete" + assert snapshot["n_pending_tasks"] > 0 + + summary = _read_result_summary(report_dir) + assert summary["complete"] is False From 8fa399f29e628d2fb2025448f94002474130a84c Mon Sep 17 00:00:00 2001 From: Viraat Chandra Date: Thu, 13 Aug 2026 19:45:56 -0700 Subject: [PATCH 3/9] refactor(config): dissolve enums.py into its owning domain modules Every enum had exactly one consumer module, so the kind-based enums.py bought no cycle-breaking and no sharing: LoadPatternType/ProfilerEngine now live in settings.py beside LoadPattern/ProfilingConfig, OSLDistributionType/StreamingMode in model_params.py, DatasetType/EvalMethod/ScorerMethod in datasets.py, and the root-level TestType/TestMode in schema.py. The split criterion is now uniform: one module per config domain, every name beside its owner, schema.py = root aggregate + cross-domain validation + re-export hub (import sites unchanged). --- .pre-commit-config.yaml | 2 +- AGENTS.md | 5 +- src/inference_endpoint/config/datasets.py | 32 ++++- src/inference_endpoint/config/enums.py | 131 ------------------ src/inference_endpoint/config/model_params.py | 24 +++- src/inference_endpoint/config/schema.py | 55 ++++++-- src/inference_endpoint/config/settings.py | 27 +++- 7 files changed, 126 insertions(+), 150 deletions(-) delete mode 100644 src/inference_endpoint/config/enums.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f13dbc726..f35a96932 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -58,7 +58,7 @@ repos: entry: uv run --no-sync python scripts/regenerate_templates.py language: system pass_filenames: false - files: ^(src/inference_endpoint/config/((schema|enums|audit|model_params|datasets|settings|timeouts)\.py|templates/.*)|src/inference_endpoint/endpoint_client/config\.py|scripts/regenerate_templates\.py)$ + files: ^(src/inference_endpoint/config/((schema|audit|model_params|datasets|settings|timeouts)\.py|templates/.*)|src/inference_endpoint/endpoint_client/config\.py|scripts/regenerate_templates\.py)$ - id: add-license-header name: Add license headers diff --git a/AGENTS.md b/AGENTS.md index e2432545d..d9ceeec06 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -92,7 +92,7 @@ Dataset Manager --> Load Generator --> Endpoint Client --> External Endpoint | **Dataset Manager** | `src/inference_endpoint/dataset_manager/` | Loads JSONL, HuggingFace, CSV, JSON, Parquet datasets. `Dataset` base class with `load_sample()`/`num_samples()` interface | | **Metrics Aggregator** | `src/inference_endpoint/async_utils/services/metrics_aggregator/` | Subprocess. Subscribes to events, aggregates per-sample metrics into a `MetricsRegistry` (counters + HDR-histogram series + raw values), publishes `MetricsSnapshot` over IPC PUB at a configurable cadence (`SessionState`: `INITIALIZE` → `LIVE` → `DRAINING` → {`COMPLETE` \| `INTERRUPTED`}). Final snapshot is atomically written to `final_snapshot.json` as the **primary** Report source; the terminal pub/sub frame is a TUI "run finished" signal only. | | **Report** | `src/inference_endpoint/metrics/report.py` | `Report.from_snapshot(dict)` — pure-function builder consuming the dict form (`snapshot_to_dict`). Reads `final_snapshot.json` directly via `json.loads` (no Struct decode). Plumbs `complete = (state == "complete" and n_pending_tasks == 0)`; renders an explicit warning for `INTERRUPTED` runs. | -| **Config** | `src/inference_endpoint/config/`, `endpoint_client/config.py` | Pydantic-based YAML schema split into focused modules (`schema.py` = BenchmarkConfig + EndpointConfig + re-export hub; `enums.py`, `audit.py`, `model_params.py`, `datasets.py`, `settings.py`), `Timeouts` (`config/timeouts.py` — all give-up deadlines in one frozen model at `settings.timeouts`, incl. the whole-run `run_timeout_s` watchdog; the workload duration `runtime.max_duration_ms` stays in `settings.py`), `HTTPClientConfig` (single Pydantic model for CLI/YAML/runtime), `RuntimeSettings` | +| **Config** | `src/inference_endpoint/config/`, `endpoint_client/config.py` | Pydantic-based YAML schema split into focused modules (`schema.py` = BenchmarkConfig + EndpointConfig + root TestType/TestMode + re-export hub; `audit.py`, `model_params.py`, `datasets.py`, `settings.py` — each domain owns its models AND enums), `Timeouts` (`config/timeouts.py` — all give-up deadlines in one frozen model at `settings.timeouts`, incl. the whole-run `run_timeout_s` watchdog; the workload duration `runtime.max_duration_ms` stays in `settings.py`), `HTTPClientConfig` (single Pydantic model for CLI/YAML/runtime), `RuntimeSettings` | | **CLI** | `src/inference_endpoint/main.py`, `commands/benchmark/cli.py` | cyclopts-based, auto-generated from `schema.py` and `HTTPClientConfig` Pydantic models. Flat shorthands via `cyclopts.Parameter(alias=...)` | | **Async Utils** | `src/inference_endpoint/async_utils/` | `LoopManager` (uvloop + eager_task_factory), ZMQ transport layer, generic `MessageCodec[T]`-parametrized pub/sub, event publisher | | **OpenAI/SGLang** | `src/inference_endpoint/openai/`, `sglang/` | Protocol adapters and response accumulators for different API formats. `openai_completions` adapter (`completions_adapter.py`) sends pre-tokenized token IDs to `/v1/completions`, bypassing the server chat template — required for gpt-oss-120b on vLLM. `sglang` adapter sends to `/generate` via `input_ids`. Both apply `Harmonize()` client-side. | @@ -244,8 +244,7 @@ src/inference_endpoint/ │ ├── early_stopping.py # MLPerf LoadGen early-stopping percentile estimates (pure math; see docs/early_stopping.md) │ └── results_plots.py # Standardized run-artifact plots (matplotlib-guarded); CLI: scripts/plot_results.py ├── config/ -│ ├── schema.py # BenchmarkConfig + EndpointConfig; re-export hub for the schema surface -│ ├── enums.py # Shared schema enums (TestType, LoadPatternType, StreamingMode, ...) +│ ├── schema.py # BenchmarkConfig + EndpointConfig + TestType/TestMode; re-export hub for the schema surface │ ├── audit.py # Audit config models (audit: YAML block) │ ├── model_params.py # ModelParams, OSLDistribution, SubmissionReference │ ├── datasets.py # Dataset, AccuracyConfig, AgenticInferenceConfig diff --git a/src/inference_endpoint/config/datasets.py b/src/inference_endpoint/config/datasets.py index 06fcd4d0a..da368910d 100644 --- a/src/inference_endpoint/config/datasets.py +++ b/src/inference_endpoint/config/datasets.py @@ -23,13 +23,13 @@ from __future__ import annotations +from enum import Enum from pathlib import Path from typing import Annotated, Any, Self import cyclopts from pydantic import BaseModel, ConfigDict, Field, model_validator -from .enums import DatasetType, EvalMethod, ScorerMethod from .model_params import ModelParams @@ -57,6 +57,36 @@ def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any _METRICS_DECOUPLED_OVERRIDE_KEYS = frozenset({"name", "streaming", "tokenizer_name"}) +class DatasetType(str, Enum): + """Dataset purpose type.""" + + PERFORMANCE = "performance" + ACCURACY = "accuracy" + + +class EvalMethod(str, Enum): + """Evaluation methods for accuracy testing.""" + + EXACT_MATCH = "exact_match" + CONTAINS = "contains" + JUDGE = "judge" + + +class ScorerMethod(str, Enum): + """Registered scorer methods for accuracy evaluation.""" + + PASS_AT_1 = "pass_at_1" + STRING_MATCH = "string_match" + ROUGE = "rouge" + CODE_BENCH = "code_bench_scorer" + SHOPIFY_CATEGORY_F1 = "shopify_category_f1" + AGENTIC_INFERENCE_INLINE = "agentic_inference_inline" + VBENCH = "vbench" + BFCL_V4 = "bfcl_v4" + LEGACY_MLPERF_DEEPSEEK_R1 = "legacy_mlperf_deepseek_r1" + SWE_BENCH = "swe_bench_scorer" + + class AgenticInferenceConfig(BaseModel): """Agentic inference conversation configuration. diff --git a/src/inference_endpoint/config/enums.py b/src/inference_endpoint/config/enums.py deleted file mode 100644 index e48c70e85..000000000 --- a/src/inference_endpoint/config/enums.py +++ /dev/null @@ -1,131 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Configuration enums. - -Split criterion: one module per config domain; enums shared across the config -models live here so every sibling module can import them without cycles. -``config/schema.py`` re-exports the public surface. -""" - -from __future__ import annotations - -from enum import Enum - - -class LoadPatternType(str, Enum): - """Load pattern types.""" - - MAX_THROUGHPUT = "max_throughput" # Offline: all queries at t=0 - POISSON = "poisson" # Online: fixed QPS with Poisson distribution - CONCURRENCY = "concurrency" # Online: fixed concurrent requests - AGENTIC_INFERENCE = ( - "agentic_inference" # Agentic inference conversations with turn sequencing - ) - BURST = "burst" # Burst pattern (TODO) - STEP = "step" # Step pattern (TODO) - - -class OSLDistributionType(str, Enum): - """Output Sequence Length distribution types.""" - - ORIGINAL = "original" # Use original distribution from dataset (default) - FIXED = "fixed" # Fixed length for all outputs - UNIFORM = "uniform" # Uniform distribution between min and max - NORMAL = "normal" # Normal/Gaussian distribution - - -class DatasetType(str, Enum): - """Dataset purpose type.""" - - PERFORMANCE = "performance" - ACCURACY = "accuracy" - - -class EvalMethod(str, Enum): - """Evaluation methods for accuracy testing.""" - - EXACT_MATCH = "exact_match" - CONTAINS = "contains" - JUDGE = "judge" - - -class ScorerMethod(str, Enum): - """Registered scorer methods for accuracy evaluation.""" - - PASS_AT_1 = "pass_at_1" - STRING_MATCH = "string_match" - ROUGE = "rouge" - CODE_BENCH = "code_bench_scorer" - SHOPIFY_CATEGORY_F1 = "shopify_category_f1" - AGENTIC_INFERENCE_INLINE = "agentic_inference_inline" - VBENCH = "vbench" - BFCL_V4 = "bfcl_v4" - LEGACY_MLPERF_DEEPSEEK_R1 = "legacy_mlperf_deepseek_r1" - SWE_BENCH = "swe_bench_scorer" - - -class TestMode(str, Enum): - """Test mode controlling performance issuance and response collection. - - - PERF: Run performance and ordinary configured scoring without in-process - collection; skip scorers that own an external evaluation run - - ACC: Skip performance and collect responses for configured scoring - - BOTH: Run performance and configured scoring with response collection - """ - - PERF = "perf" - ACC = "acc" - BOTH = "both" - - -class StreamingMode(str, Enum): - """Streaming mode for response handling. - - - AUTO: Automatically enable for online mode, disable for offline mode - - ON: Force streaming enabled (for TTFT metrics) - - OFF: Force streaming disabled - """ - - AUTO = "auto" - ON = "on" - OFF = "off" - - -class TestType(str, Enum): - """Test type for both config classification and execution mode. - - - OFFLINE: Max throughput benchmark (all queries at t=0) - - ONLINE: Sustained QPS benchmark (Poisson or concurrency-based) - - EVAL: Accuracy evaluation - - SUBMISSION: Official submission (may include both perf and accuracy) - """ - - OFFLINE = "offline" - ONLINE = "online" - EVAL = "eval" - SUBMISSION = "submission" - - -class ProfilerEngine(str, Enum): - """Inference engine whose profiling protocol the client should drive. - - Selects the HTTP path layout used to derive start/stop URLs from - ``endpoint_config.endpoints``. Each value corresponds to one server-side - profiling protocol; add a new variant + ``_PROFILE_PATHS`` row to support - another engine. - """ - - VLLM = "vllm" diff --git a/src/inference_endpoint/config/model_params.py b/src/inference_endpoint/config/model_params.py index a823039d7..1e47e3de3 100644 --- a/src/inference_endpoint/config/model_params.py +++ b/src/inference_endpoint/config/model_params.py @@ -22,12 +22,12 @@ from __future__ import annotations +from enum import Enum from typing import Annotated, Any, Self import cyclopts from pydantic import BaseModel, ConfigDict, Field, model_validator -from .enums import OSLDistributionType, StreamingMode from .ruleset_base import BenchmarkSuiteRuleset @@ -46,6 +46,28 @@ def _non_default_completion_controls(mp: ModelParams) -> list[str]: return [name for name, non_default in checks.items() if non_default] +class OSLDistributionType(str, Enum): + """Output Sequence Length distribution types.""" + + ORIGINAL = "original" # Use original distribution from dataset (default) + FIXED = "fixed" # Fixed length for all outputs + UNIFORM = "uniform" # Uniform distribution between min and max + NORMAL = "normal" # Normal/Gaussian distribution + + +class StreamingMode(str, Enum): + """Streaming mode for response handling. + + - AUTO: Automatically enable for online mode, disable for offline mode + - ON: Force streaming enabled (for TTFT metrics) + - OFF: Force streaming disabled + """ + + AUTO = "auto" + ON = "on" + OFF = "off" + + class OSLDistribution(BaseModel): """Output Sequence Length distribution configuration. diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index c809e0f8b..2c1046eb7 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -20,16 +20,19 @@ cyclopts.Parameter(alias=...) on Annotated fields to declare shorthand aliases alongside dotted paths. -Split criterion: one module per config domain (enums / audit / model_params / -datasets / settings / timeouts); this module owns only the root aggregate -(``BenchmarkConfig`` and its cross-field validation) plus the explicit -re-export hub, so every existing ``config.schema`` import site keeps working. +Split criterion: one module per config domain (audit / model_params / +datasets / settings / timeouts), with every name — model, enum, helper — +living beside its owner; this module owns only the root aggregate +(``BenchmarkConfig``, its cross-field validation, and the root-level +``TestType``/``TestMode`` enums) plus the explicit re-export hub, so every +existing ``config.schema`` import site keeps working. """ from __future__ import annotations import logging from collections import Counter +from enum import Enum from pathlib import Path from typing import Annotated, Any, Literal, Self, Union from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit @@ -51,29 +54,29 @@ from ..exceptions import CLIError from ..utils import WithUpdatesMixin from .audit import AuditConfig, AuditTestId, OutputCachingTestConfig -from .datasets import AccuracyConfig, AgenticInferenceConfig, Dataset -from .enums import ( +from .datasets import ( + AccuracyConfig, + AgenticInferenceConfig, + Dataset, DatasetType, EvalMethod, - LoadPatternType, - OSLDistributionType, - ProfilerEngine, ScorerMethod, - StreamingMode, - TestMode, - TestType, ) from .model_params import ( ModelParams, OSLDistribution, + OSLDistributionType, + StreamingMode, SubmissionReference, _non_default_completion_controls, ) from .settings import ( EarlyStoppingConfig, LoadPattern, + LoadPatternType, OfflineSettings, OnlineSettings, + ProfilerEngine, ProfilingConfig, RuntimeConfig, Settings, @@ -120,6 +123,34 @@ logger = logging.getLogger(__name__) +class TestMode(str, Enum): + """Test mode controlling performance issuance and response collection. + + - PERF: Run performance and ordinary configured scoring without in-process + collection; skip scorers that own an external evaluation run + - ACC: Skip performance and collect responses for configured scoring + - BOTH: Run performance and configured scoring with response collection + """ + + PERF = "perf" + ACC = "acc" + BOTH = "both" + + +class TestType(str, Enum): + """Test type for both config classification and execution mode. + + - OFFLINE: Max throughput benchmark (all queries at t=0) + - ONLINE: Sustained QPS benchmark (Poisson or concurrency-based) + - EVAL: Accuracy evaluation + - SUBMISSION: Official submission (may include both perf and accuracy) + """ + + OFFLINE = "offline" + ONLINE = "online" + EVAL = "eval" + SUBMISSION = "submission" + class EndpointConfig(BaseModel): diff --git a/src/inference_endpoint/config/settings.py b/src/inference_endpoint/config/settings.py index f90893859..78062be7d 100644 --- a/src/inference_endpoint/config/settings.py +++ b/src/inference_endpoint/config/settings.py @@ -22,6 +22,7 @@ from __future__ import annotations +from enum import Enum from typing import Annotated, Any, Self import cyclopts @@ -36,10 +37,22 @@ ) from ..endpoint_client.config import HTTPClientConfig -from .enums import LoadPatternType, ProfilerEngine from .timeouts import Timeouts +class LoadPatternType(str, Enum): + """Load pattern types.""" + + MAX_THROUGHPUT = "max_throughput" # Offline: all queries at t=0 + POISSON = "poisson" # Online: fixed QPS with Poisson distribution + CONCURRENCY = "concurrency" # Online: fixed concurrent requests + AGENTIC_INFERENCE = ( + "agentic_inference" # Agentic inference conversations with turn sequencing + ) + BURST = "burst" # Burst pattern (TODO) + STEP = "step" # Step pattern (TODO) + + class RuntimeConfig(BaseModel): """Runtime configuration. @@ -212,6 +225,18 @@ class WarmupConfig(BaseModel): ] = Field(42, description="RNG seed for warmup scheduling and sample ordering") +class ProfilerEngine(str, Enum): + """Inference engine whose profiling protocol the client should drive. + + Selects the HTTP path layout used to derive start/stop URLs from + ``endpoint_config.endpoints``. Each value corresponds to one server-side + profiling protocol; add a new variant + ``_PROFILE_PATHS`` row to support + another engine. + """ + + VLLM = "vllm" + + @cyclopts.Parameter(name="*") class ProfilingConfig(BaseModel): """Client-side trigger for the server's profiler. From d088fe99f6634b9f5906f3ce1d8b612799088745 Mon Sep 17 00:00:00 2001 From: Viraat Chandra Date: Thu, 13 Aug 2026 23:53:53 -0700 Subject: [PATCH 4/9] docs(examples): drop default-restating timeout keys from qwen3-vl example Examples set only non-default overrides; the null drain deadlines equal the schema defaults. --- .../offline_qwen3_vl_235b_a22b_shopify.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/08_Qwen3-VL-235B-A22B_Example/offline_qwen3_vl_235b_a22b_shopify.yaml b/examples/08_Qwen3-VL-235B-A22B_Example/offline_qwen3_vl_235b_a22b_shopify.yaml index 5b2660af9..3e393480d 100644 --- a/examples/08_Qwen3-VL-235B-A22B_Example/offline_qwen3_vl_235b_a22b_shopify.yaml +++ b/examples/08_Qwen3-VL-235B-A22B_Example/offline_qwen3_vl_235b_a22b_shopify.yaml @@ -38,8 +38,6 @@ settings: timeouts: # Increase for slow worker startup (spawn, imports). Default 60s may be too short. worker_initialization_timeout_s: 120 - performance_drain_timeout_s: null # Performance drain timeout in seconds (null = wait indefinitely) - accuracy_drain_timeout_s: null # Accuracy drain timeout in seconds (null = wait indefinitely) warmup: enabled: true # Enable warmup phase before performance run n_requests: 1600 # Warmup request count (None = full dataset once) From a04a710ab2ff8067728cbcf43fcfd2d976cf725e Mon Sep 17 00:00:00 2001 From: Viraat Chandra Date: Fri, 14 Aug 2026 00:10:10 -0700 Subject: [PATCH 5/9] refactor(config): worker lifecycle timeouts stay on settings.client MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three endpoint-client worker waits (init, graceful shutdown, force kill) are client internals, not global run deadlines — restore them on HTTPClientConfig under their original names and drop them from Timeouts. CLI_QUICK_REFERENCE gains a run-lifetime timeline visualizing where every time knob acts. --- AGENTS.md | 2 +- docs/CLI_QUICK_REFERENCE.md | 72 +++++++++++++----- ...m_gptoss_120b_per_dataset_osl_example.yaml | 4 +- ...ractive_qwen3_vl_235b_a22b_shopify_8k.yaml | 6 +- .../offline_qwen3_vl_235b_a22b_shopify.yaml | 5 +- .../server_qwen3_vl_235b_a22b_shopify.yaml | 5 +- .../commands/benchmark/execute.py | 7 -- .../templates/concurrency_template_full.yaml | 6 +- .../templates/offline_template_full.yaml | 6 +- .../templates/online_template_full.yaml | 6 +- src/inference_endpoint/config/timeouts.py | 22 ++---- .../endpoint_client/config.py | 25 ++----- .../endpoint_client/worker_manager.py | 8 +- .../commands/test_benchmark_command.py | 3 +- tests/unit/config/test_timeouts.py | 73 +++---------------- tests/unit/config/test_yaml_loader.py | 7 +- 16 files changed, 101 insertions(+), 156 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index d9ceeec06..ffe50dedd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -92,7 +92,7 @@ Dataset Manager --> Load Generator --> Endpoint Client --> External Endpoint | **Dataset Manager** | `src/inference_endpoint/dataset_manager/` | Loads JSONL, HuggingFace, CSV, JSON, Parquet datasets. `Dataset` base class with `load_sample()`/`num_samples()` interface | | **Metrics Aggregator** | `src/inference_endpoint/async_utils/services/metrics_aggregator/` | Subprocess. Subscribes to events, aggregates per-sample metrics into a `MetricsRegistry` (counters + HDR-histogram series + raw values), publishes `MetricsSnapshot` over IPC PUB at a configurable cadence (`SessionState`: `INITIALIZE` → `LIVE` → `DRAINING` → {`COMPLETE` \| `INTERRUPTED`}). Final snapshot is atomically written to `final_snapshot.json` as the **primary** Report source; the terminal pub/sub frame is a TUI "run finished" signal only. | | **Report** | `src/inference_endpoint/metrics/report.py` | `Report.from_snapshot(dict)` — pure-function builder consuming the dict form (`snapshot_to_dict`). Reads `final_snapshot.json` directly via `json.loads` (no Struct decode). Plumbs `complete = (state == "complete" and n_pending_tasks == 0)`; renders an explicit warning for `INTERRUPTED` runs. | -| **Config** | `src/inference_endpoint/config/`, `endpoint_client/config.py` | Pydantic-based YAML schema split into focused modules (`schema.py` = BenchmarkConfig + EndpointConfig + root TestType/TestMode + re-export hub; `audit.py`, `model_params.py`, `datasets.py`, `settings.py` — each domain owns its models AND enums), `Timeouts` (`config/timeouts.py` — all give-up deadlines in one frozen model at `settings.timeouts`, incl. the whole-run `run_timeout_s` watchdog; the workload duration `runtime.max_duration_ms` stays in `settings.py`), `HTTPClientConfig` (single Pydantic model for CLI/YAML/runtime), `RuntimeSettings` | +| **Config** | `src/inference_endpoint/config/`, `endpoint_client/config.py` | Pydantic-based YAML schema split into focused modules (`schema.py` = BenchmarkConfig + EndpointConfig + root TestType/TestMode + re-export hub; `audit.py`, `model_params.py`, `datasets.py`, `settings.py` — each domain owns its models AND enums), `Timeouts` (`config/timeouts.py` — all give-up deadlines in one frozen model at `settings.timeouts`, incl. the whole-run `run_timeout_s` watchdog; the workload duration `runtime.max_duration_ms` stays in `settings.py`; client worker-lifecycle timeouts stay on `settings.client`), `HTTPClientConfig` (single Pydantic model for CLI/YAML/runtime), `RuntimeSettings` | | **CLI** | `src/inference_endpoint/main.py`, `commands/benchmark/cli.py` | cyclopts-based, auto-generated from `schema.py` and `HTTPClientConfig` Pydantic models. Flat shorthands via `cyclopts.Parameter(alias=...)` | | **Async Utils** | `src/inference_endpoint/async_utils/` | `LoopManager` (uvloop + eager_task_factory), ZMQ transport layer, generic `MessageCodec[T]`-parametrized pub/sub, event publisher | | **OpenAI/SGLang** | `src/inference_endpoint/openai/`, `sglang/` | Protocol adapters and response accumulators for different API formats. `openai_completions` adapter (`completions_adapter.py`) sends pre-tokenized token IDs to `/v1/completions`, bypassing the server chat template — required for gpt-oss-120b on vLLM. `sglang` adapter sends to `/generate` via `input_ids`. Both apply `Harmonize()` client-side. | diff --git a/docs/CLI_QUICK_REFERENCE.md b/docs/CLI_QUICK_REFERENCE.md index c1d55dc5f..b09f2d80d 100644 --- a/docs/CLI_QUICK_REFERENCE.md +++ b/docs/CLI_QUICK_REFERENCE.md @@ -120,31 +120,63 @@ Flag names shown as `--full.dotted.path --alias`. Both forms work. ## Time Knobs All give-up deadlines live under `settings.timeouts`; the only workload duration is -`settings.runtime.max_duration_ms`. `null`/unset means "wait indefinitely" (or "off") everywhere. - -| YAML path | CLI flag | Semantics | -| --------------------------------------------------- | ----------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `settings.runtime.max_duration_ms` | `--runtime.max-duration-ms` | Caps the performance phase (ms, or suffix: `600s`, `10m`); reaching it ends the phase NORMALLY — the report stays valid | -| `settings.timeouts.run_timeout_s` | `--timeout` | Whole-run watchdog; firing aborts the entire run — report marked INTERRUPTED, non-zero exit | -| `settings.timeouts.service_ready_timeout_s` | `--service-ready-timeout` | Wait for the metrics-aggregator/event-logger services to become ready (default 30) | -| `settings.timeouts.warmup_drain_timeout_s` | `--warmup-drain-timeout` | Bound on in-flight warmup requests after the warmup phase ends (default 240) | -| `settings.timeouts.performance_drain_timeout_s` | `--performance-drain-timeout` | Bound on in-flight performance requests after the phase ends (default: wait indefinitely) | -| `settings.timeouts.accuracy_drain_timeout_s` | `--accuracy-drain-timeout` | Bound on in-flight accuracy requests after the phase ends (default: wait indefinitely) | -| `settings.timeouts.metrics_drain_timeout_s` | `--metrics-drain-timeout` | Budget for the metrics aggregator to finish tokenizing buffered samples after the run ends (default: wait indefinitely); expiring fails the run with `complete: false` artifacts | -| `settings.timeouts.worker_initialization_timeout_s` | `--worker-initialization-timeout-s` | Wait for endpoint-client worker processes to start (default 60) | -| `settings.timeouts.worker_graceful_shutdown_wait_s` | `--worker-graceful-shutdown-wait-s` | Post-run wait for workers to exit gracefully (default 0.5) | -| `settings.timeouts.worker_force_kill_timeout_s` | `--worker-force-kill-timeout-s` | Wait after SIGTERM before SIGKILL during worker teardown (default 0.5) | +`settings.runtime.max_duration_ms`; endpoint-client worker lifecycle timeouts are client +internals under `settings.client`. `null`/unset means "wait indefinitely" (or "off") everywhere. + +Where every knob acts over the life of a run: + +```text +run_benchmark ── run_timeout_s deadline captured here ─────────────────────────────┐ +│ │ +├─ setup: dataset + tokenizer load (counts against run_timeout_s) │ +├─ launch metrics/event-logger services ── service_ready_timeout_s │ +├─ start endpoint-client workers ── client.worker_initialization_timeout│ +│ │ +├─ WARMUP issue ──────────┤ drain ─┤ ── warmup_drain_timeout_s │ +│ │ +├─ PERFORMANCE issue ──────────┤ drain ─┤ │ +│ │ │ └─ performance_drain_timeout_s │ +│ └───────────────┴─ max_duration_ms caps ISSUING only; reaching it │ +│ ends the phase NORMALLY (valid report) and │ +│ SKIPS the drain — the two never run together │ +│ │ +├─ ACCURACY issue ──────────┤ drain ─┤ ── accuracy_drain_timeout_s │ +│ │ +├─ metrics drain (tokenize buffered ISL/OSL)── metrics_drain_timeout_s │ +│ (expiry FAILS the run: │ +│ complete: false + non-zero exit) │ +├─ worker shutdown ── client.worker_graceful_shutdown_wait│ +│ then client.worker_force_kill_timeout +└─ finalize: score accuracy, write artifacts │ + │ + run_timeout_s (whole-run watchdog) ───────────────────────────────────────────────┘ + firing at ANY point above aborts the run: report marked INTERRUPTED, non-zero exit +``` + +| YAML path | CLI flag | Semantics | +| ----------------------------------------------- | ---------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `settings.runtime.max_duration_ms` | `--runtime.max-duration-ms` | Caps performance-phase issuing (ms, or suffix: `600s`, `10m`); reaching it ends the phase NORMALLY — the report stays valid — and skips the performance drain | +| `settings.timeouts.run_timeout_s` | `--timeout` | Whole-run watchdog over everything above; firing aborts the entire run — report marked INTERRUPTED, non-zero exit | +| `settings.timeouts.service_ready_timeout_s` | `--service-ready-timeout` | Wait for the metrics-aggregator/event-logger services to become ready (default 30) | +| `settings.timeouts.warmup_drain_timeout_s` | `--warmup-drain-timeout` | Bound on in-flight warmup requests after the warmup phase ends (default 240) | +| `settings.timeouts.performance_drain_timeout_s` | `--performance-drain-timeout` | Bound on in-flight performance requests after the phase stops issuing (default: wait indefinitely) | +| `settings.timeouts.accuracy_drain_timeout_s` | `--accuracy-drain-timeout` | Bound on in-flight accuracy requests after the phase ends (default: wait indefinitely) | +| `settings.timeouts.metrics_drain_timeout_s` | `--metrics-drain-timeout` | Budget for the metrics aggregator to finish tokenizing buffered samples after the run ends (default: wait indefinitely); expiring fails the run with `complete: false` artifacts | +| `settings.client.worker_initialization_timeout` | `--client.worker-initialization-timeout` | Wait for endpoint-client worker processes to start (default 60) | +| `settings.client.worker_graceful_shutdown_wait` | `--client.worker-graceful-shutdown-wait` | Post-run wait for workers to exit gracefully (default 0.5) | +| `settings.client.worker_force_kill_timeout` | `--client.worker-force-kill-timeout` | Wait after the graceful window before force-killing workers (default 0.5) | How the knobs compose: 1. **`--num-samples` / dataset-once defines the work.** An explicit `runtime.n_samples_to_issue` sets the sample count; omitting it issues the performance dataset once. -2. **`runtime.max_duration_ms` caps the performance phase** and ends it normally — remaining - samples are not issued, the report is valid. -3. **`timeouts.run_timeout_s` aborts the whole run** (every phase, drains included) — the report - is marked INTERRUPTED and the process exits non-zero. -4. **Per-phase drain timeouts bound the post-phase wait** for requests still in flight after a - phase stops issuing. +2. **`runtime.max_duration_ms` caps performance-phase issuing** and ends the phase normally — + remaining samples are not issued, in-flight requests are abandoned (no drain), the report is + valid. It does not bound the drain: issuing and draining are consecutive, never concurrent. +3. **Per-phase drain timeouts bound the post-phase wait** for requests still in flight after a + phase stops issuing on its own. +4. **`timeouts.run_timeout_s` is the only total-wall-time bound** (setup, every phase, every + drain) — firing aborts the run, marks the report INTERRUPTED, and exits non-zero. ## Environment Variables diff --git a/examples/04_GPTOSS120B_Example/vllm_gptoss_120b_per_dataset_osl_example.yaml b/examples/04_GPTOSS120B_Example/vllm_gptoss_120b_per_dataset_osl_example.yaml index 351537fcd..85e865ba2 100644 --- a/examples/04_GPTOSS120B_Example/vllm_gptoss_120b_per_dataset_osl_example.yaml +++ b/examples/04_GPTOSS120B_Example/vllm_gptoss_120b_per_dataset_osl_example.yaml @@ -72,10 +72,8 @@ settings: type: "concurrency" target_concurrency: 1024 - timeouts: - worker_initialization_timeout_s: 300.0 - client: + worker_initialization_timeout: 300.0 num_workers: 16 log_level: "WARN" worker_gc_mode: "disabled" diff --git a/examples/08_Qwen3-VL-235B-A22B_Example/interactive_qwen3_vl_235b_a22b_shopify_8k.yaml b/examples/08_Qwen3-VL-235B-A22B_Example/interactive_qwen3_vl_235b_a22b_shopify_8k.yaml index bd8cb0854..a41425503 100644 --- a/examples/08_Qwen3-VL-235B-A22B_Example/interactive_qwen3_vl_235b_a22b_shopify_8k.yaml +++ b/examples/08_Qwen3-VL-235B-A22B_Example/interactive_qwen3_vl_235b_a22b_shopify_8k.yaml @@ -31,6 +31,8 @@ settings: target_qps: 3 client: + # Increase for slow worker startup (spawn, imports). Default 60s may be too short. + worker_initialization_timeout: 120 num_workers: 5 transport: type: zmq @@ -38,10 +40,6 @@ settings: send_buffer_size: 16777216 max_connections: 1000 - timeouts: - # Increase for slow worker startup (spawn, imports). Default 60s may be too short. - worker_initialization_timeout_s: 120 - warmup: enabled: true # Enable warmup phase before performance run n_requests: 400 # Warmup request count (None = full dataset once) diff --git a/examples/08_Qwen3-VL-235B-A22B_Example/offline_qwen3_vl_235b_a22b_shopify.yaml b/examples/08_Qwen3-VL-235B-A22B_Example/offline_qwen3_vl_235b_a22b_shopify.yaml index 3e393480d..1f02ea16f 100644 --- a/examples/08_Qwen3-VL-235B-A22B_Example/offline_qwen3_vl_235b_a22b_shopify.yaml +++ b/examples/08_Qwen3-VL-235B-A22B_Example/offline_qwen3_vl_235b_a22b_shopify.yaml @@ -29,15 +29,14 @@ settings: type: "max_throughput" client: + # Increase for slow worker startup (spawn, imports). Default 60s may be too short. + worker_initialization_timeout: 120 num_workers: 5 transport: type: zmq recv_buffer_size: 16777216 send_buffer_size: 16777216 max_connections: 1000 - timeouts: - # Increase for slow worker startup (spawn, imports). Default 60s may be too short. - worker_initialization_timeout_s: 120 warmup: enabled: true # Enable warmup phase before performance run n_requests: 1600 # Warmup request count (None = full dataset once) diff --git a/examples/08_Qwen3-VL-235B-A22B_Example/server_qwen3_vl_235b_a22b_shopify.yaml b/examples/08_Qwen3-VL-235B-A22B_Example/server_qwen3_vl_235b_a22b_shopify.yaml index 9b9110ae9..9b835b249 100644 --- a/examples/08_Qwen3-VL-235B-A22B_Example/server_qwen3_vl_235b_a22b_shopify.yaml +++ b/examples/08_Qwen3-VL-235B-A22B_Example/server_qwen3_vl_235b_a22b_shopify.yaml @@ -30,6 +30,8 @@ settings: target_qps: 5 client: + # Increase for slow worker startup (spawn, imports). Default 60s may be too short. + worker_initialization_timeout: 120 num_workers: 5 transport: type: zmq @@ -37,9 +39,6 @@ settings: send_buffer_size: 16777216 max_connections: 1000 - timeouts: - # Increase for slow worker startup (spawn, imports). Default 60s may be too short. - worker_initialization_timeout_s: 120 warmup: enabled: true # Enable warmup phase before performance run n_requests: 1600 # Warmup request count (None = full dataset once) diff --git a/src/inference_endpoint/commands/benchmark/execute.py b/src/inference_endpoint/commands/benchmark/execute.py index ac9fe547d..328f67bcd 100644 --- a/src/inference_endpoint/commands/benchmark/execute.py +++ b/src/inference_endpoint/commands/benchmark/execute.py @@ -727,7 +727,6 @@ async def _create_issuer( api_type: APIType = config.endpoint_config.api_type # client.api_type is propagated from endpoint_config.api_type by # BenchmarkConfig._propagate_client_api_type — no override needed here. - timeouts = config.settings.timeouts client_overrides: dict = { "endpoint_urls": [ urljoin(e.rstrip("/") + "/", api_type.default_route()) @@ -736,12 +735,6 @@ async def _create_issuer( "api_key": config.endpoint_config.api_key, "event_logs_dir": ctx.report_dir, "cpu_affinity": ctx.affinity_plan, - # Worker lifecycle deadlines live in settings.timeouts; the - # HTTPClientConfig fields are excluded runtime carriers populated - # only here. - "worker_initialization_timeout_s": timeouts.worker_initialization_timeout_s, - "worker_graceful_shutdown_wait_s": timeouts.worker_graceful_shutdown_wait_s, - "worker_force_kill_timeout_s": timeouts.worker_force_kill_timeout_s, } if ctx.accuracy_only: # Single-stream (num_workers=1, max_connections=1) is baked into diff --git a/src/inference_endpoint/config/templates/concurrency_template_full.yaml b/src/inference_endpoint/config/templates/concurrency_template_full.yaml index 97a13f6be..47e11e7d5 100644 --- a/src/inference_endpoint/config/templates/concurrency_template_full.yaml +++ b/src/inference_endpoint/config/templates/concurrency_template_full.yaml @@ -76,6 +76,9 @@ settings: linger: -1 # ZMQ linger on close (-1=block until sent) immediate: 1 # ZMQ IMMEDIATE (1=only enqueue on ready) stream_all_chunks: false # Stream all chunks to main thread (caution: perf overhead) + worker_initialization_timeout: 60.0 # Worker init timeout (seconds) + worker_graceful_shutdown_wait: 0.5 # Post-run graceful shutdown wait (seconds) + worker_force_kill_timeout: 0.5 # Force kill timeout after graceful wait (seconds) insecure: false # Skip TLS certificate verification max_idle_time: 4.0 # Discard connections idle longer than this (seconds) min_required_connections: -1 # Min connections to initialize (-1=auto, 0=disabled) @@ -87,9 +90,6 @@ settings: performance_drain_timeout_s: null # Performance drain timeout in seconds (None = wait indefinitely) accuracy_drain_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely; accuracy is unbounded by default because every sample must complete) metrics_drain_timeout_s: null # Wall-clock budget (seconds) to finish tokenizing buffered samples after ENDED (None = wait indefinitely). An incomplete drain fails the run: artifacts are written with complete: false, then run_benchmark exits non-zero. - worker_initialization_timeout_s: 60.0 # Endpoint-client worker init timeout (seconds) - worker_graceful_shutdown_wait_s: 0.5 # Endpoint-client post-run graceful shutdown wait (seconds) - worker_force_kill_timeout_s: 0.5 # Endpoint-client force kill timeout after graceful wait (seconds) warmup: enabled: false # Enable warmup phase before performance run n_requests: null # Warmup request count (None = full dataset once) diff --git a/src/inference_endpoint/config/templates/offline_template_full.yaml b/src/inference_endpoint/config/templates/offline_template_full.yaml index d232f702f..825440624 100644 --- a/src/inference_endpoint/config/templates/offline_template_full.yaml +++ b/src/inference_endpoint/config/templates/offline_template_full.yaml @@ -76,6 +76,9 @@ settings: linger: -1 # ZMQ linger on close (-1=block until sent) immediate: 1 # ZMQ IMMEDIATE (1=only enqueue on ready) stream_all_chunks: false # Stream all chunks to main thread (caution: perf overhead) + worker_initialization_timeout: 60.0 # Worker init timeout (seconds) + worker_graceful_shutdown_wait: 0.5 # Post-run graceful shutdown wait (seconds) + worker_force_kill_timeout: 0.5 # Force kill timeout after graceful wait (seconds) insecure: false # Skip TLS certificate verification max_idle_time: 4.0 # Discard connections idle longer than this (seconds) min_required_connections: -1 # Min connections to initialize (-1=auto, 0=disabled) @@ -87,9 +90,6 @@ settings: performance_drain_timeout_s: null # Performance drain timeout in seconds (None = wait indefinitely) accuracy_drain_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely; accuracy is unbounded by default because every sample must complete) metrics_drain_timeout_s: null # Wall-clock budget (seconds) to finish tokenizing buffered samples after ENDED (None = wait indefinitely). An incomplete drain fails the run: artifacts are written with complete: false, then run_benchmark exits non-zero. - worker_initialization_timeout_s: 60.0 # Endpoint-client worker init timeout (seconds) - worker_graceful_shutdown_wait_s: 0.5 # Endpoint-client post-run graceful shutdown wait (seconds) - worker_force_kill_timeout_s: 0.5 # Endpoint-client force kill timeout after graceful wait (seconds) warmup: enabled: false # Enable warmup phase before performance run n_requests: null # Warmup request count (None = full dataset once) diff --git a/src/inference_endpoint/config/templates/online_template_full.yaml b/src/inference_endpoint/config/templates/online_template_full.yaml index aad246771..f480473e1 100644 --- a/src/inference_endpoint/config/templates/online_template_full.yaml +++ b/src/inference_endpoint/config/templates/online_template_full.yaml @@ -77,6 +77,9 @@ settings: linger: -1 # ZMQ linger on close (-1=block until sent) immediate: 1 # ZMQ IMMEDIATE (1=only enqueue on ready) stream_all_chunks: false # Stream all chunks to main thread (caution: perf overhead) + worker_initialization_timeout: 60.0 # Worker init timeout (seconds) + worker_graceful_shutdown_wait: 0.5 # Post-run graceful shutdown wait (seconds) + worker_force_kill_timeout: 0.5 # Force kill timeout after graceful wait (seconds) insecure: false # Skip TLS certificate verification max_idle_time: 4.0 # Discard connections idle longer than this (seconds) min_required_connections: -1 # Min connections to initialize (-1=auto, 0=disabled) @@ -88,9 +91,6 @@ settings: performance_drain_timeout_s: null # Performance drain timeout in seconds (None = wait indefinitely) accuracy_drain_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely; accuracy is unbounded by default because every sample must complete) metrics_drain_timeout_s: null # Wall-clock budget (seconds) to finish tokenizing buffered samples after ENDED (None = wait indefinitely). An incomplete drain fails the run: artifacts are written with complete: false, then run_benchmark exits non-zero. - worker_initialization_timeout_s: 60.0 # Endpoint-client worker init timeout (seconds) - worker_graceful_shutdown_wait_s: 0.5 # Endpoint-client post-run graceful shutdown wait (seconds) - worker_force_kill_timeout_s: 0.5 # Endpoint-client force kill timeout after graceful wait (seconds) warmup: enabled: false # Enable warmup phase before performance run n_requests: null # Warmup request count (None = full dataset once) diff --git a/src/inference_endpoint/config/timeouts.py b/src/inference_endpoint/config/timeouts.py index abd22c3f8..1a6d2124d 100644 --- a/src/inference_endpoint/config/timeouts.py +++ b/src/inference_endpoint/config/timeouts.py @@ -16,11 +16,12 @@ """Global waits and deadlines (the ``settings.timeouts`` block). Split criterion: one module per config domain; every global time knob that -bounds how long the harness waits — startup readiness, per-phase drains, the -worker lifecycle, and the whole-run watchdog — lives here. Workload durations +bounds how long the harness waits — startup readiness, per-phase drains, and +the whole-run watchdog — lives here. Workload durations (``runtime.max_duration_ms``) are part of the benchmark definition, not waits, -and stay in ``runtime``. Dataset-scoped time knobs (e.g. agentic -``turn_timeout_s``) stay in their dataset config blocks. +and stay in ``runtime``. Client worker-lifecycle timeouts stay on +``settings.client`` (endpoint-client internals). Dataset-scoped time knobs +(e.g. agentic ``turn_timeout_s``) stay in their dataset config blocks. """ from __future__ import annotations @@ -131,16 +132,3 @@ class Timeouts(WithUpdatesMixin, BaseModel): "run_benchmark exits non-zero." ), ) - worker_initialization_timeout_s: float = Field( - 60.0, ge=0, description="Endpoint-client worker init timeout (seconds)" - ) - worker_graceful_shutdown_wait_s: float = Field( - 0.5, - ge=0, - description="Endpoint-client post-run graceful shutdown wait (seconds)", - ) - worker_force_kill_timeout_s: float = Field( - 0.5, - ge=0, - description="Endpoint-client force kill timeout after graceful wait (seconds)", - ) diff --git a/src/inference_endpoint/endpoint_client/config.py b/src/inference_endpoint/endpoint_client/config.py index 6802a2851..b2839996d 100644 --- a/src/inference_endpoint/endpoint_client/config.py +++ b/src/inference_endpoint/endpoint_client/config.py @@ -187,24 +187,15 @@ class HTTPClientConfig(WithUpdatesMixin, BaseModel): False, description="Stream all chunks to main thread (caution: perf overhead)" ) - # Worker lifecycle timeouts — runtime carriers. The authoritative user - # knobs live in settings.timeouts; setup copies them here (no CLI flag, - # never serialized). WithUpdatesMixin.with_updates reads exclude=True - # fields directly, so copies preserve the injected values. - worker_initialization_timeout_s: Annotated[ - float, cyclopts.Parameter(parse=False) - ] = Field(60.0, exclude=True, description="Worker init timeout (seconds)") - worker_graceful_shutdown_wait_s: Annotated[ - float, cyclopts.Parameter(parse=False) - ] = Field( - 0.5, exclude=True, description="Post-run graceful shutdown wait (seconds)" + # Worker lifecycle timeouts + worker_initialization_timeout: float = Field( + 60.0, description="Worker init timeout (seconds)" + ) + worker_graceful_shutdown_wait: float = Field( + 0.5, description="Post-run graceful shutdown wait (seconds)" ) - worker_force_kill_timeout_s: Annotated[float, cyclopts.Parameter(parse=False)] = ( - Field( - 0.5, - exclude=True, - description="Force kill timeout after graceful wait (seconds)", - ) + worker_force_kill_timeout: float = Field( + 0.5, description="Force kill timeout after graceful wait (seconds)" ) # Set to True to skip certificate verification (e.g. self-signed certs). diff --git a/src/inference_endpoint/endpoint_client/worker_manager.py b/src/inference_endpoint/endpoint_client/worker_manager.py index bd0304447..ae0d194df 100644 --- a/src/inference_endpoint/endpoint_client/worker_manager.py +++ b/src/inference_endpoint/endpoint_client/worker_manager.py @@ -92,7 +92,7 @@ async def initialize(self) -> None: except TimeoutError as e: raise TimeoutError( - f"Workers failed to initialize within {self.http_config.worker_initialization_timeout_s}s" + f"Workers failed to initialize within {self.http_config.worker_initialization_timeout}s" ) from e finally: @@ -130,7 +130,7 @@ def _pin_workers(self) -> None: async def _wait_for_workers_with_liveness_check(self) -> None: """Wait for workers, checking liveness at 10% intervals.""" - timeout = self.http_config.worker_initialization_timeout_s + timeout = self.http_config.worker_initialization_timeout check_interval = timeout * 0.10 if timeout else 1.0 start = time.monotonic() @@ -165,7 +165,7 @@ async def shutdown(self) -> None: if worker.is_alive(): worker.terminate() - await asyncio.sleep(self.http_config.worker_graceful_shutdown_wait_s) + await asyncio.sleep(self.http_config.worker_graceful_shutdown_wait) # Force kill remaining for worker in self.workers: @@ -176,7 +176,7 @@ async def shutdown(self) -> None: await asyncio.gather( *( asyncio.to_thread( - worker.join, timeout=self.http_config.worker_force_kill_timeout_s + worker.join, timeout=self.http_config.worker_force_kill_timeout ) for worker in self.workers ) diff --git a/tests/integration/commands/test_benchmark_command.py b/tests/integration/commands/test_benchmark_command.py index 1aec94185..6023f7692 100644 --- a/tests/integration/commands/test_benchmark_command.py +++ b/tests/integration/commands/test_benchmark_command.py @@ -375,8 +375,7 @@ def _resolve_template(template_path: Path, server_url: str) -> dict: # The other 5 templates benefit from warm module / IPC caches and don't # need the headroom. 120 s is a generous safety margin that does not # change the production default, only this integration test. - data["settings"].setdefault("timeouts", {}) - data["settings"]["timeouts"]["worker_initialization_timeout_s"] = 120.0 + data["settings"].setdefault("client", {})["worker_initialization_timeout"] = 120.0 # Accuracy datasets can't run e2e against echo server (no scorer), so keep only performance datasets. data["datasets"] = [ diff --git a/tests/unit/config/test_timeouts.py b/tests/unit/config/test_timeouts.py index 0c61b6d6a..ef958a3f1 100644 --- a/tests/unit/config/test_timeouts.py +++ b/tests/unit/config/test_timeouts.py @@ -22,7 +22,6 @@ import random import pytest -import yaml from inference_endpoint.config.runtime_settings import RuntimeSettings from inference_endpoint.config.schema import ( BenchmarkConfig, @@ -53,9 +52,6 @@ def test_defaults(self): assert cfg.performance_drain_timeout_s is None assert cfg.accuracy_drain_timeout_s is None assert cfg.metrics_drain_timeout_s is None - assert cfg.worker_initialization_timeout_s == 60.0 - assert cfg.worker_graceful_shutdown_wait_s == 0.5 - assert cfg.worker_force_kill_timeout_s == 0.5 @pytest.mark.unit def test_mounted_on_settings_by_default(self): @@ -101,15 +97,7 @@ def test_deadline_none_means_unlimited(self, field): assert getattr(Timeouts(**{field: None}), field) is None @pytest.mark.unit - @pytest.mark.parametrize( - "field", - [ - "service_ready_timeout_s", - "worker_initialization_timeout_s", - "worker_graceful_shutdown_wait_s", - "worker_force_kill_timeout_s", - ], - ) + @pytest.mark.parametrize("field", ["service_ready_timeout_s"]) def test_ge_zero_fields_accept_zero_reject_negative(self, field): assert getattr(Timeouts(**{field: 0}), field) == 0.0 with pytest.raises(ValidationError): @@ -153,54 +141,19 @@ def test_top_level_timeout_rejected(self): with pytest.raises(ValidationError, match="timeout"): BenchmarkConfig(**_MINIMAL_KWARGS, timeout=42.0) - @pytest.mark.unit - def test_client_worker_knob_rejected(self): - with pytest.raises(ValidationError, match="worker_initialization_timeout"): - BenchmarkConfig( - **_MINIMAL_KWARGS, - settings={"client": {"worker_initialization_timeout": 120.0}}, - ) - - @pytest.mark.unit - def test_client_worker_knob_rejected_from_yaml(self, tmp_path): - yaml_content = """ -type: "offline" -model_params: - name: "test-model" -endpoint_config: - endpoints: ["http://test:8000"] -datasets: - - path: "test.jsonl" -settings: - client: - worker_initialization_timeout: 120 -""" - config_file = tmp_path / "stale.yaml" - config_file.write_text(yaml_content) - with pytest.raises(ValidationError, match="worker_initialization_timeout"): - BenchmarkConfig.from_yaml_file(config_file) - -class TestWorkerFieldsHiddenFromSerialization: +class TestClientWorkerKnobs: @pytest.mark.unit - def test_yaml_roundtrip_excludes_worker_carrier_fields(self, tmp_path): - """The runtime-carrier worker fields on the client never serialize, so - a persisted config reloads cleanly under extra=forbid.""" - config = BenchmarkConfig(**_MINIMAL_KWARGS) - out = tmp_path / "roundtrip.yaml" - config.to_yaml_file(out) - - dumped = yaml.safe_load(out.read_text()) - client_block = dumped.get("settings", {}).get("client", {}) or {} - carrier_fields = { - "worker_initialization_timeout_s", - "worker_graceful_shutdown_wait_s", - "worker_force_kill_timeout_s", - } - assert not carrier_fields & client_block.keys() - - loaded = BenchmarkConfig.from_yaml_file(out) - assert loaded.settings.timeouts == config.settings.timeouts + def test_worker_lifecycle_knobs_live_on_client(self): + """Worker lifecycle timeouts are endpoint-client internals and stay on + settings.client, not in the timeouts block.""" + config = BenchmarkConfig( + **_MINIMAL_KWARGS, + settings={"client": {"worker_initialization_timeout": 120.0}}, + ) + assert config.settings.client.worker_initialization_timeout == 120.0 + with pytest.raises(ValidationError): + Timeouts(worker_initialization_timeout_s=90.0) class TestTimeoutsYAMLRoundtrip: @@ -221,7 +174,6 @@ def test_yaml_block_loads(self, tmp_path): performance_drain_timeout_s: 30.0 accuracy_drain_timeout_s: null metrics_drain_timeout_s: 300.0 - worker_initialization_timeout_s: 90 """ config_file = tmp_path / "timeouts.yaml" config_file.write_text(yaml_content) @@ -232,7 +184,6 @@ def test_yaml_block_loads(self, tmp_path): assert timeouts.performance_drain_timeout_s == 30.0 assert timeouts.accuracy_drain_timeout_s is None assert timeouts.metrics_drain_timeout_s == 300.0 - assert timeouts.worker_initialization_timeout_s == 90.0 class TestMaxDurationSuffix: diff --git a/tests/unit/config/test_yaml_loader.py b/tests/unit/config/test_yaml_loader.py index d1d3d5181..ef582b31a 100644 --- a/tests/unit/config/test_yaml_loader.py +++ b/tests/unit/config/test_yaml_loader.py @@ -45,8 +45,6 @@ def test_load_valid_yaml(self, tmp_path): path: "test.jsonl" settings: - timeouts: - worker_initialization_timeout_s: 120 load_pattern: type: "max_throughput" client: @@ -67,7 +65,6 @@ def test_load_valid_yaml(self, tmp_path): assert config.name == "test-config" assert config.type == BenchmarkTestType.OFFLINE assert len(config.datasets) == 1 - assert config.settings.timeouts.worker_initialization_timeout_s == 120.0 assert config.settings.client.transport.recv_buffer_size == 16777216 assert config.settings.client.transport.send_buffer_size == 8388608 @@ -245,8 +242,8 @@ def test_serialize_deserialize_roundtrip(self, tmp_path): ) assert loaded.settings.load_pattern.type == original.settings.load_pattern.type assert ( - loaded.settings.timeouts.worker_initialization_timeout_s - == original.settings.timeouts.worker_initialization_timeout_s + loaded.settings.client.worker_initialization_timeout + == original.settings.client.worker_initialization_timeout ) assert ( loaded.settings.client.transport.recv_buffer_size From 06d42ef7358dc76ceeebe045d8ce9f1bd55b9264 Mon Sep 17 00:00:00 2001 From: Viraat Chandra Date: Fri, 14 Aug 2026 00:19:08 -0700 Subject: [PATCH 6/9] chore(tests): drop stray untracked scratch tests swept into the branch test_metrics_preflight_tap.py references a local scratchpad path and test_protocol.py targets transport features that do not exist on main; neither belongs to this PR. --- scripts/bench_drain_tokenize.py | 301 ------------- .../async_utils/transport/test_protocol.py | 113 ----- .../scripts/test_metrics_preflight_tap.py | 410 ------------------ 3 files changed, 824 deletions(-) delete mode 100644 scripts/bench_drain_tokenize.py delete mode 100644 tests/unit/async_utils/transport/test_protocol.py delete mode 100644 tests/unit/scripts/test_metrics_preflight_tap.py diff --git a/scripts/bench_drain_tokenize.py b/scripts/bench_drain_tokenize.py deleted file mode 100644 index 5a0ca00d0..000000000 --- a/scripts/bench_drain_tokenize.py +++ /dev/null @@ -1,301 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Apples-to-apples benchmark of OUTPUT-tokenization strategies for the -metrics-aggregator drain (OSL / TPOT). - -Why this exists: at the end of a run the aggregator tokenizes every sample's -output to derive OSL/TPOT. The live impl fires one asyncio task per sample, -each awaiting ``loop.run_in_executor(thread_pool, len(tok.tokenize(text)))`` -(see ``metrics_aggregator/metrics_table.py::AsyncTokenTrigger.fire`` + -``token_metrics.py::TokenizePool.token_count_async``). This script reproduces -that exact pattern standalone and pits it against a single batched -``tokenizer(texts)`` call (the batched strategy from the prior ISL ablation) -so the cost of the current design — and the win from replacing it — is measured -on identical inputs. Measured (Qwen2.5-0.5B, 48-core, 12 workers): encode_batch -is ~4.6x the current per-sample async pattern on short outputs, ~2.0x on the -realistic right-skewed OSL distribution (mean ~3.8k tok) — and, more -importantly, removes the per-sample asyncio-task backlog (1 task/sample) that -drives the drain timeout. The single batched Rust call beats thread-sharding -(the HF fast tokenizer already parallelises a batch internally). - -Strategies (all plain ``tokenize``, no chat template — matches the OSL/TPOT -text path taken when the output has no tool_calls): - - current_async EXACT live drain pattern: per-sample loop.create_task -> - TokenizePool.token_count_async -> run_in_executor, gathered. - sync_loop Serial ``len(tok.tokenize(t))`` — isolates raw tokenize cost - from asyncio/thread-pool overhead. - batch One ``tokenizer(texts)`` Rust call over all texts. - thread_batch Shard texts across ``--workers`` threads, each batch-tokenizes - its shard (GIL released inside the Rust call). - -Usage: - uv run python scripts/bench_drain_tokenize.py \ - --model Qwen/Qwen2.5-0.5B-Instruct --n-samples 20000 --runs 3 -""" - -from __future__ import annotations - -import argparse -import asyncio -import json -import os -import random -import threading -import time -from concurrent.futures import ThreadPoolExecutor -from typing import Any - -from inference_endpoint.async_utils.services.metrics_aggregator.token_metrics import ( - TokenizePool, -) -from transformers import AutoTokenizer - -_WORDS = ( - "the quick brown fox jumps over the lazy dog inference benchmark " - "tokenization latency throughput performance model weights attention " - "transformer layer norm softmax gradient embedding sequence decode " -).split() - - -# Measured OSL token-length distribution (max_new_tokens=20000 cap; heavily -# right-skewed: median 2153, mean 3824). Piecewise-linear inverse-CDF from the -# measured percentiles so generated lengths match the real drain workload. -_OSL_PCTL: tuple[tuple[float, int], ...] = ( - (0, 177), - (1, 303), - (5, 463), - (10, 578), - (25, 951), - (50, 2153), - (75, 4977), - (80, 6001), - (90, 9564), - (95, 13510), - (97, 16422), - (99, 20000), - (100, 20000), -) - - -def _sample_osl(rng: random.Random) -> int: - p = rng.random() * 100.0 - for (p0, v0), (p1, v1) in zip(_OSL_PCTL, _OSL_PCTL[1:], strict=False): - if p <= p1: - frac = (p - p0) / (p1 - p0) if p1 > p0 else 0.0 - return int(v0 + frac * (v1 - v0)) - return _OSL_PCTL[-1][1] - - -def _make_outputs( - n: int, profile: str, min_words: int, max_words: int, seed: int = 42 -) -> list[str]: - """Synthetic model-output texts (plain text, the OSL/TPOT common case). - - profile='mlperf' draws word counts from the measured OSL distribution - (token≈word for these common words); 'uniform' uses [min_words, max_words]. - """ - rng = random.Random(seed) - if profile == "mlperf": - lengths = [_sample_osl(rng) for _ in range(n)] - else: - lengths = [rng.randint(min_words, max_words) for _ in range(n)] - return [" ".join(rng.choices(_WORDS, k=length)) for length in lengths] - - -def _result(name: str, secs: float, n: int, total_tokens: int) -> dict[str, Any]: - return { - "strategy": name, - "wall_s": round(secs, 4), - "samples_per_s": round(n / secs) if secs else 0, - "tokens_per_s": round(total_tokens / secs) if secs else 0, - } - - -def bench_sync_loop(texts: list[str], tok: Any) -> tuple[float, int]: - t0 = time.perf_counter() - total = 0 - for t in texts: - total += len(tok.tokenize(t)) - return time.perf_counter() - t0, total - - -def bench_batch(texts: list[str], tok: Any) -> tuple[float, int]: - t0 = time.perf_counter() - enc = tok(texts, add_special_tokens=False, return_attention_mask=False) - total = sum(len(ids) for ids in enc["input_ids"]) - return time.perf_counter() - t0, total - - -def bench_encode_batch(texts: list[str], tok: Any) -> tuple[float, int]: - """Raw Rust ``encode_batch`` on the backend tokenizer — skips the - BatchEncoding/padding wrapper that ``tokenizer(...)`` builds. We only need - counts, so this is the leanest count-only path.""" - backend = tok.backend_tokenizer - # encode_batch_fast (tokenizers>=0.20) skips offset computation; fall back - # to encode_batch where unavailable. - fn = getattr(backend, "encode_batch_fast", None) or backend.encode_batch - t0 = time.perf_counter() - encs = fn(texts, add_special_tokens=False) - total = sum(len(e.ids) for e in encs) - return time.perf_counter() - t0, total - - -def bench_batch_chunked( - texts: list[str], tok: Any, chunk: int = 50_000 -) -> tuple[float, int]: - """Chunked batches — bounds peak memory for very large drains while still - feeding the Rust parallel path large slices.""" - t0 = time.perf_counter() - total = 0 - for i in range(0, len(texts), chunk): - enc = tok( - texts[i : i + chunk], - add_special_tokens=False, - return_attention_mask=False, - ) - total += sum(len(ids) for ids in enc["input_ids"]) - return time.perf_counter() - t0, total - - -def bench_thread_batch( - texts: list[str], tokenizer_name: str, workers: int -) -> tuple[float, int]: - # Each worker loads its own tokenizer (thread-local, like TokenizePool) and - # batch-tokenizes a contiguous shard. - shards: list[list[str]] = [texts[i::workers] for i in range(workers)] - tls = threading.local() - - def _work_tls(shard: list[str]) -> int: - tok = getattr(tls, "tok", None) - if tok is None: - tok = AutoTokenizer.from_pretrained(tokenizer_name) - tls.tok = tok - if not shard: - return 0 - enc = tok(shard, add_special_tokens=False, return_attention_mask=False) - return sum(len(ids) for ids in enc["input_ids"]) - - with ThreadPoolExecutor(max_workers=workers) as ex: - # Warm tokenizers on every thread before timing. - list(ex.map(lambda _: _work_tls([]), range(workers))) - t0 = time.perf_counter() - total = sum(ex.map(_work_tls, shards)) - return time.perf_counter() - t0, total - - -async def bench_current_async( - texts: list[str], pool: TokenizePool -) -> tuple[float, int]: - """EXACT live drain pattern: one asyncio task per sample, each awaiting - pool.token_count_async (-> loop.run_in_executor), then gathered.""" - loop = asyncio.get_running_loop() - t0 = time.perf_counter() - tasks = [loop.create_task(pool.token_count_async(t, loop)) for t in texts] - counts = await asyncio.gather(*tasks) - return time.perf_counter() - t0, sum(counts) - - -def _run_current_async(texts: list[str], pool: TokenizePool) -> tuple[float, int]: - try: - import uvloop # the aggregator runs on uvloop; match it. - - runner = uvloop.run - except ImportError: - runner = asyncio.run - return runner(bench_current_async(texts, pool)) - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__) - ap.add_argument("--model", default="Qwen/Qwen2.5-0.5B-Instruct") - ap.add_argument("--n-samples", type=int, default=20000) - ap.add_argument("--runs", type=int, default=3) - ap.add_argument( - "--workers", - type=int, - default=max(2, (os.cpu_count() or 16) // 4), - help="TokenizePool / thread_batch worker count (aggregator default).", - ) - ap.add_argument("--osl-profile", choices=("mlperf", "uniform"), default="mlperf") - ap.add_argument("--min-words", type=int, default=20) - ap.add_argument("--max-words", type=int, default=200) - ap.add_argument("--output", default="") - args = ap.parse_args() - - print(f"Loading tokenizer: {args.model}") - AutoTokenizer.from_pretrained(args.model) # warm cache before timing - tok = AutoTokenizer.from_pretrained(args.model) - - print( - f"Generating {args.n_samples} synthetic outputs (profile={args.osl_profile})..." - ) - texts = _make_outputs( - args.n_samples, args.osl_profile, args.min_words, args.max_words - ) - avg_words = sum(t.count(" ") + 1 for t in texts) / len(texts) - print( - f"profile={args.osl_profile} | avg {avg_words:.0f} words/output " - f"| workers={args.workers}\n" - ) - - pool = TokenizePool(args.model, n_workers=args.workers) - results: list[dict[str, Any]] = [] - try: - strategies = [ - ("current_async", lambda: _run_current_async(texts, pool)), - ("sync_loop", lambda: bench_sync_loop(texts, tok)), - ("batch", lambda: bench_batch(texts, tok)), - ("batch_chunked", lambda: bench_batch_chunked(texts, tok)), - ("encode_batch", lambda: bench_encode_batch(texts, tok)), - ( - "thread_batch", - lambda: bench_thread_batch(texts, args.model, args.workers), - ), - ] - for name, fn in strategies: - best_secs = float("inf") - total_tokens = 0 - for _ in range(args.runs): - secs, total_tokens = fn() - best_secs = min(best_secs, secs) - r = _result(name, best_secs, args.n_samples, total_tokens) - results.append(r) - print( - f"{name:<16} {r['wall_s']:>9.4f}s " - f"{r['samples_per_s']:>12,} samples/s " - f"{r['tokens_per_s']:>14,} tok/s" - ) - finally: - pool.close() - - base = next(r for r in results if r["strategy"] == "current_async") - print("\nspeedup vs current_async (best wall):") - for r in results: - if r["strategy"] != "current_async" and r["samples_per_s"]: - print( - f" {r['strategy']:<16} {r['samples_per_s'] / base['samples_per_s']:>6.1f}x" - ) - - if args.output: - with open(args.output, "w") as f: - json.dump({"args": vars(args), "results": results}, f, indent=2) - print(f"\nwrote {args.output}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/unit/async_utils/transport/test_protocol.py b/tests/unit/async_utils/transport/test_protocol.py deleted file mode 100644 index 16dc8da6a..000000000 --- a/tests/unit/async_utils/transport/test_protocol.py +++ /dev/null @@ -1,113 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -import asyncio -from collections import deque - -import pytest -from inference_endpoint.async_utils.transport.protocol import MessageSubscriber - - -class _IntCodec: - def encode(self, item: int) -> tuple[bytes, bytes]: - return b"test____", str(item).encode() - - def decode(self, payload: bytes) -> int: - return int(payload) - - def on_decode_error(self, payload: bytes, exc: Exception) -> int | None: - return None - - -class _QueueSubscriber(MessageSubscriber[int]): - def __init__( - self, - loop: asyncio.AbstractEventLoop, - payloads: list[bytes], - *, - max_read_batch_size: int, - ) -> None: - super().__init__(_IntCodec(), "test://subscriber", loop) - self._payloads = deque(payloads) - self._max_read_batch_size = max_read_batch_size - self.batches: list[list[int]] = [] - self.received: list[int] = [] - self.done = asyncio.Event() - self.expected = len(payloads) - self.release = asyncio.Event() - self.block_processing = False - self.active = 0 - self.max_active = 0 - - def receive(self) -> bytes | None: - if not self._payloads: - raise StopIteration - return self._payloads.popleft() - - async def process(self, items: list[int]) -> None: - self.active += 1 - self.max_active = max(self.max_active, self.active) - if self.block_processing: - await self.release.wait() - self.batches.append(items) - self.received.extend(items) - self.active -= 1 - if len(self.received) >= self.expected: - self.done.set() - - -@pytest.mark.unit -@pytest.mark.asyncio -async def test_subscriber_caps_each_read_and_reschedules_without_new_edge(): - subscriber = _QueueSubscriber( - asyncio.get_running_loop(), - [str(i).encode() for i in range(5)], - max_read_batch_size=2, - ) - - subscriber._on_readable() - await asyncio.wait_for(subscriber.done.wait(), timeout=1) - - assert subscriber.received == [0, 1, 2, 3, 4] - assert subscriber.batches == [[0, 1], [2, 3], [4]] - - -@pytest.mark.unit -@pytest.mark.asyncio -async def test_subscriber_processes_batches_single_flight_in_fifo_order(): - subscriber = _QueueSubscriber( - asyncio.get_running_loop(), [b"1"], max_read_batch_size=4 - ) - subscriber.block_processing = True - subscriber.expected = 2 - - subscriber._on_readable() - subscriber._payloads.append(b"2") - subscriber._on_readable() - await asyncio.sleep(0) - subscriber.release.set() - await asyncio.wait_for(subscriber.done.wait(), timeout=1) - - assert subscriber.received == [1, 2] - assert subscriber.max_active == 1 - - -@pytest.mark.unit -def test_none_payloads_count_toward_read_budget_and_close_cancels_resume(): - subscriber = _QueueSubscriber( - asyncio.new_event_loop(), - [None, None, b"3"], # type: ignore[list-item] - max_read_batch_size=2, - ) - try: - subscriber._on_readable() - - assert list(subscriber._payloads) == [b"3"] - assert subscriber._read_continuation is not None - - subscriber.close() - assert subscriber._read_continuation is None - finally: - subscriber.loop.close() diff --git a/tests/unit/scripts/test_metrics_preflight_tap.py b/tests/unit/scripts/test_metrics_preflight_tap.py deleted file mode 100644 index 9fdedbe69..000000000 --- a/tests/unit/scripts/test_metrics_preflight_tap.py +++ /dev/null @@ -1,410 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Focused tests for the experiment-only GPT-OSS metrics preflight tap.""" - -# ruff: noqa: I001 -# Keep import layout stable across the pinned pre-commit and local uv ruff. - -from __future__ import annotations - -import csv -import importlib.util -import json -import os -import subprocess -import sys -from pathlib import Path - -import pytest -from inference_endpoint.async_utils.services.metrics_aggregator.snapshot import ( - CounterStat, - MetricsSnapshot, - MetricsSnapshotCodec, - SessionState, -) -from inference_endpoint.core.record import TOPIC_FRAME_SIZE - -pytestmark = pytest.mark.unit - - -def _load_tap(): - path = Path("scratchpad/gptoss_nvl144_pr334_vvv_20260728/metrics_preflight_tap.py") - spec = importlib.util.spec_from_file_location("metrics_preflight_tap", path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - -tap = _load_tap() - - -def _write_process( - proc_root: Path, - *, - pid: int, - ppid: int, - argv: list[str], - status: str = "VmRSS:\t123 kB\nVmHWM:\t456 kB\n", -) -> None: - proc_dir = proc_root / str(pid) - proc_dir.mkdir() - (proc_dir / "stat").write_text(f"{pid} (command with spaces) S {ppid} 0 0\n") - (proc_dir / "cmdline").write_bytes(b"\0".join(a.encode() for a in argv) + b"\0") - (proc_dir / "status").write_text(status) - - -def _snapshot( - counter: int, - *, - state: SessionState = SessionState.LIVE, - pending: int = 0, - issued: int = 10, - completed: int = 5, -) -> MetricsSnapshot: - return MetricsSnapshot( - counter=counter, - timestamp_ns=counter * 100, - state=state, - n_pending_tasks=pending, - metrics=[ - CounterStat("total_samples_issued", issued), - CounterStat("total_samples_completed", completed), - CounterStat("total_samples_failed", 0), - ], - ) - - -class TestProcessDiscovery: - def test_finds_only_aggregator_below_root(self, tmp_path: Path) -> None: - proc = tmp_path / "proc" - proc.mkdir() - _write_process(proc, pid=100, ppid=1, argv=["benchmark"]) - _write_process(proc, pid=101, ppid=100, argv=["worker"]) - _write_process( - proc, - pid=102, - ppid=101, - argv=[ - "python", - "-m", - tap.AGGREGATOR_MODULE, - "--socket-dir", - "/dev/shm/zmq_a", - "--metrics-socket=metrics_a", - ], - ) - _write_process( - proc, - pid=200, - ppid=1, - argv=["python", "-m", tap.AGGREGATOR_MODULE], - ) - - found = tap.find_aggregator_descendant(100, proc) - - assert found is not None - assert found.pid == 102 - assert tap.parse_aggregator_socket_args(found.argv) == ( - "/dev/shm/zmq_a", - "metrics_a", - ) - assert ( - tap.metrics_ipc_address("/dev/shm/zmq_a", "metrics_a") - == "ipc:///dev/shm/zmq_a/metrics_a" - ) - - def test_missing_socket_arg_is_rejected(self) -> None: - with pytest.raises(ValueError, match="metrics-socket"): - tap.parse_aggregator_socket_args(["--socket-dir", "/tmp/x"]) - - -class TestSampling: - def test_reads_proc_cgroup_meminfo_and_tmpfs(self, tmp_path: Path) -> None: - proc = tmp_path / "proc" - proc.mkdir() - _write_process(proc, pid=42, ppid=1, argv=["aggregator"]) - (proc / "42" / "cgroup").write_text("0::/job/step\n") - (proc / "meminfo").write_text( - "MemTotal: 10000 kB\nMemAvailable: 2500 kB\n" - ) - - cgroup_root = tmp_path / "cgroup" - cgroup = cgroup_root / "job" / "step" - cgroup.mkdir(parents=True) - (cgroup / "memory.current").write_text("1000\n") - (cgroup / "memory.peak").write_text("2000\n") - (cgroup / "memory.max").write_text("3000\n") - (cgroup / "memory.events").write_text("oom 2\noom_kill 1\n") - - events = tmp_path / "benchmark_1" / "events" - events.mkdir(parents=True) - (events / "events.jsonl").write_bytes(b"x" * 17) - - location = tap._find_cgroup(42, proc, cgroup_root) - obs = tap.sample_memory( - 42, - location, - str(tmp_path / "benchmark_*" / "events" / "events.jsonl"), - proc, - ) - - assert obs == tap.MemoryObservation( - aggregator_alive=True, - rss_kib=123, - hwm_kib=456, - cgroup_current_bytes=1000, - cgroup_peak_bytes=2000, - cgroup_max_bytes=3000, - cgroup_oom=2, - cgroup_oom_kill=1, - mem_available_kib=2500, - mem_total_kib=10000, - tmpfs_event_files=1, - tmpfs_events_bytes=17, - ) - - -class TestSnapshotsAndArtifacts: - def test_decodes_frame_and_tracks_pending_memory_high_water( - self, tmp_path: Path - ) -> None: - codec = MetricsSnapshotCodec() - first = _snapshot(1, pending=3) - second = _snapshot( - 4, - state=SessionState.DRAINING, - pending=7, - issued=20, - completed=20, - ) - topic, payload = codec.encode(first) - assert len(topic) == TOPIC_FRAME_SIZE - assert tap.decode_metrics_frame(topic + payload, codec) == first - - stats = tap.MonitorStats( - started_wall_ns=100, - started_monotonic_ns=100, - root_pid=1, - aggregator_pid=42, - ) - stats.observe_snapshot(first, 1_000_000_000) - stats.observe_snapshot(second, 3_500_000_000) - stats.observe_memory( - tap.MemoryObservation( - aggregator_alive=True, - rss_kib=11, - hwm_kib=12, - cgroup_current_bytes=13, - cgroup_peak_bytes=14, - cgroup_max_bytes=15, - cgroup_oom=0, - cgroup_oom_kill=0, - mem_available_kib=16, - mem_total_kib=17, - tmpfs_event_files=1, - tmpfs_events_bytes=18, - ) - ) - - summary = stats.to_dict(ended_wall_ns=4_000_000_000, csv_path=tmp_path / "x") - assert summary["published_pending_high_water"] == 7 - assert summary["pending_at_first_draining"] == 7 - assert summary["counter_gap_total"] == 2 - assert summary["counter_gap_max"] == 2 - assert summary["max_snapshot_gap_s"] == 2.5 - assert summary["aggregator_rss_high_water_kib"] == 11 - assert summary["tmpfs_events_high_water_bytes"] == 18 - assert summary["telemetry_capture_valid"] is True - assert summary["telemetry_capture_failures"] == [] - - def test_capture_gate_requires_rss_and_oom_counters(self) -> None: - stats = tap.MonitorStats( - started_wall_ns=100, - started_monotonic_ns=100, - root_pid=1, - aggregator_pid=42, - cgroup_version=2, - snapshots_received=2, - published_pending_high_water=7, - aggregator_reported_hwm_high_water_kib=12, - cgroup_memory_current_high_water_bytes=13, - cgroup_memory_peak_high_water_bytes=14, - ) - - assert tap.telemetry_capture_failures(stats) == [ - "aggregator_rss_missing", - "cgroup_oom_missing", - "cgroup_oom_kill_missing", - ] - - stats.cgroup_version = 1 - assert tap.telemetry_capture_failures(stats) == [ - "aggregator_rss_missing", - "cgroup_oom_missing", - ] - - def test_csv_finalization_and_atomic_summary(self, tmp_path: Path) -> None: - csv_path = tmp_path / "telemetry.csv" - artifact = tap.AtomicCsv(csv_path, fsync_interval_s=0) - artifact.open() - row = dict.fromkeys(tap.CSV_FIELDS, "") - row["row_kind"] = "memory" - artifact.write(row) - artifact.finalize() - - with csv_path.open(newline="") as f: - rows = list(csv.DictReader(f)) - assert len(rows) == 1 - assert rows[0]["row_kind"] == "memory" - assert not csv_path.with_suffix(".csv.part").exists() - - summary_path = tmp_path / "summary.json" - payload = {"status": "complete"} - from inference_endpoint.utils.atomic_write import atomic_write_bytes - - atomic_write_bytes( - summary_path, (json.dumps(payload, sort_keys=True) + "\n").encode() - ) - assert json.loads(summary_path.read_text()) == payload - - def test_missing_aggregator_is_nonzero_and_still_atomic( - self, tmp_path: Path - ) -> None: - csv_path = tmp_path / "telemetry.csv" - summary_path = tmp_path / "summary.json" - args = tap._build_parser().parse_args( - [ - "--root-pid", - str(2**31 - 1), - "--csv", - str(csv_path), - "--summary", - str(summary_path), - "--discover-timeout-s", - "0", - ] - ) - - exit_code, summary = tap.run(args) - - assert exit_code == 2 - assert summary["status"] == "aggregator_not_found" - assert summary["telemetry_capture_valid"] is False - assert "aggregator_not_found" in summary["telemetry_capture_failures"] - assert csv_path.is_file() - assert summary_path.is_file() - assert not csv_path.with_suffix(".csv.part").exists() - - def test_end_to_end_discovers_and_taps_metrics_pub( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - socket_dir = tmp_path / "sockets" - socket_dir.mkdir() - socket_name = "metrics_test" - child_code = """ -import sys -import time -import zmq -from inference_endpoint.async_utils.services.metrics_aggregator.snapshot import ( - CounterStat, MetricsSnapshot, MetricsSnapshotCodec, SessionState, -) - -args = sys.argv[1:] -socket_dir = args[args.index("--socket-dir") + 1] -socket_name = args[args.index("--metrics-socket") + 1] -ctx = zmq.Context() -sock = ctx.socket(zmq.PUB) -sock.setsockopt(zmq.LINGER, 0) -sock.bind(f"ipc://{socket_dir}/{socket_name}") -codec = MetricsSnapshotCodec() -time.sleep(0.5) -for i in range(1, 7): - snap = MetricsSnapshot( - counter=i, - timestamp_ns=i, - state=SessionState.LIVE, - n_pending_tasks=i, - metrics=[ - CounterStat("total_samples_issued", i), - CounterStat("total_samples_completed", i), - CounterStat("total_samples_failed", 0), - ], - ) - topic, payload = codec.encode(snap) - sock.send(topic + payload) - time.sleep(0.15) -sock.close(0) -ctx.term() -""" - child = subprocess.Popen( - [ - sys.executable, - "-c", - child_code, - tap.AGGREGATOR_MODULE, - "--socket-dir", - str(socket_dir), - "--metrics-socket", - socket_name, - ] - ) - observation = tap.MemoryObservation( - aggregator_alive=True, - rss_kib=100, - hwm_kib=200, - cgroup_current_bytes=300, - cgroup_peak_bytes=400, - cgroup_max_bytes=500, - cgroup_oom=0, - cgroup_oom_kill=0, - mem_available_kib=600, - mem_total_kib=700, - tmpfs_event_files=0, - tmpfs_events_bytes=0, - ) - monkeypatch.setattr(tap, "sample_memory", lambda *args, **kwargs: observation) - csv_path = tmp_path / "telemetry.csv" - summary_path = tmp_path / "summary.json" - args = tap._build_parser().parse_args( - [ - "--root-pid", - str(os.getpid()), - "--csv", - str(csv_path), - "--summary", - str(summary_path), - "--discover-timeout-s", - "5", - "--discover-poll-s", - "0.02", - "--sample-interval-s", - "0.05", - "--poll-timeout-ms", - "20", - "--post-aggregator-exit-s", - "0.1", - "--fsync-interval-s", - "0", - ] - ) - try: - exit_code, summary = tap.run(args) - finally: - child.wait(timeout=5) - - assert exit_code == 0 - assert summary["status"] == "aggregator_exited" - assert summary["telemetry_capture_valid"] is True - assert summary["snapshots_received"] >= 2 - assert summary["published_pending_high_water"] >= 2 - assert summary["aggregator_reported_hwm_high_water_kib"] == 200 - assert summary["cgroup_memory_current_high_water_bytes"] == 300 - assert summary["cgroup_memory_peak_high_water_bytes"] == 400 - assert csv_path.is_file() - assert summary_path.is_file() - with csv_path.open(newline="") as handle: - rows = list(csv.DictReader(handle)) - assert rows[-1]["row_kind"] == "terminal_memory" From 2d9cc3d61e7f25c9ff485927e6e1df4aae45532b Mon Sep 17 00:00:00 2001 From: Viraat Chandra Date: Fri, 14 Aug 2026 00:36:46 -0700 Subject: [PATCH 7/9] refactor(watchdog): extract _RunWatchdog beside _PerfPhaseTimeout One object owns the whole-run deadline: timer handle, fired flag, and the late-bound session, replacing the nonlocal flag + mutable-holder closure threaded through _run_benchmark_async. Behavior unchanged. --- .../commands/benchmark/execute.py | 102 ++++++++++-------- 1 file changed, 57 insertions(+), 45 deletions(-) diff --git a/src/inference_endpoint/commands/benchmark/execute.py b/src/inference_endpoint/commands/benchmark/execute.py index 328f67bcd..bd735ad56 100644 --- a/src/inference_endpoint/commands/benchmark/execute.py +++ b/src/inference_endpoint/commands/benchmark/execute.py @@ -716,6 +716,56 @@ def cancel(self) -> None: self._handle = None +class _RunWatchdog: + """Whole-run deadline timer for ``settings.timeouts.run_timeout_s``. + + Armed before the metrics pipeline starts (so service-launch and + endpoint-connect stalls are bounded) and kept armed through the metrics + drain (so a stuck aggregator drain is bounded too). On fire: stop the + session first — it short-circuits its drain and publishes ENDED promptly, + so the event logger flushes and the aggregator records the buffered + tokenizer-drain samples — then SIGTERM the aggregator, whose handler + writes the INTERRUPTED final snapshot (``publish_final`` is first-wins, + so INTERRUPTED stays authoritative). ``run_benchmark`` raises + ``ExecutionError`` after finalization whenever ``fired`` is set, so a + timed-out run always fails loudly even if a still-draining aggregator + finalized COMPLETE first. + """ + + def __init__( + self, + loop: asyncio.AbstractEventLoop, + deadline: float | None, + pipe: MetricsPipeline, + ) -> None: + self.fired = False + self._session: BenchmarkSession | None = None + self._pipe = pipe + self._handle = ( + loop.call_later(max(0.0, deadline - time.monotonic()), self._fire) + if deadline is not None + else None + ) + + def bind_session(self, session: BenchmarkSession) -> None: + """Late-bind the session: it is created after the timer is armed.""" + self._session = session + + def _fire(self) -> None: + self.fired = True + logger.error( + "Run timeout reached; aborting run — report will be marked " "INTERRUPTED." + ) + if self._session is not None: + self._session.stop() + self._pipe.terminate_metrics_aggregator() + + def cancel(self) -> None: + if self._handle is not None: + self._handle.cancel() + self._handle = None + + async def _create_issuer( ctx: BenchmarkContext, loop: asyncio.AbstractEventLoop ) -> tuple[HttpClientSampleIssuer, HTTPEndpointClient]: @@ -848,44 +898,7 @@ async def _run_benchmark_async( # idempotent, so the clean-path shutdown below is a harmless second call. http_client: HTTPEndpointClient | None = None - # Whole-run watchdog. Armed before the pipeline starts so setup stalls - # (service launch, endpoint connect) are bounded too, and kept armed - # through the metrics drain so run_timeout_s can SIGTERM a stuck - # aggregator drain. Cancelled in the outermost finally. - run_timed_out = False - # The session is created later inside the pipeline scope; bind it through - # a mutable holder so the callback never touches a possibly-unbound local - # (a NameError inside a loop callback is swallowed by the loop's exception - # handler, which would leave the watchdog inert). - session_ref: list[BenchmarkSession] = [] - run_timeout_s = config.settings.timeouts.run_timeout_s - - def _on_run_timeout() -> None: - nonlocal run_timed_out - run_timed_out = True - logger.error( - "Run timeout (%.1fs) reached; aborting run — report will be " - "marked INTERRUPTED.", - run_timeout_s, - ) - # Stop the session first: it short-circuits _drain_inflight and - # run()'s finally publishes ENDED promptly, so the aggregator still - # records the buffered tokenizer-drain samples. Then SIGTERM the - # aggregator: its handler writes the INTERRUPTED final snapshot - # (publish_final is first-wins, so INTERRUPTED stays authoritative; - # even if a still-draining aggregator finalizes as COMPLETE first, - # run_benchmark raises on run_timed_out, so a timed-out run always - # fails loudly). Targeted (not all services): the event logger - # flushes on ENDED, which session.stop() still delivers. - if session_ref: - session_ref[0].stop() - pipe.terminate_metrics_aggregator() - - run_watchdog = ( - loop.call_later(max(0.0, deadline - time.monotonic()), _on_run_timeout) - if deadline is not None - else None - ) + watchdog = _RunWatchdog(loop, deadline, pipe) try: tmpfs_dir.mkdir(parents=True, exist_ok=True) @@ -923,7 +936,7 @@ def _on_run_timeout() -> None: on_sample_complete=on_sample_complete, session_id=session_id, ) - session_ref.append(session) + watchdog.bind_session(session) phases = _build_phases(ctx, perf_strategy=agentic_inference_strategy) max_duration_ms = ( @@ -963,7 +976,7 @@ def _on_phase_start(phase: PhaseConfig) -> None: loop.add_signal_handler(signal.SIGINT, session.stop) try: - if run_timed_out: + if watchdog.fired: # Deadline elapsed during setup — never start issuing # load after it. Run the already-stopped session so # STARTED/ENDED still flow: the event logger exits only @@ -978,7 +991,7 @@ def _on_phase_start(phase: PhaseConfig) -> None: ) session_completed_normally = True except Exception as e: - if run_timed_out: + if watchdog.fired: # The watchdog already aborted the run; a teardown race # can surface here as a generic exception. Fall through # with an empty session result so finalize still writes @@ -1004,7 +1017,7 @@ def _on_phase_start(phase: PhaseConfig) -> None: # Unifies the clean phase-end path and the abort path — both # reach this block. A watchdog abort counts as an abort even # when session.run returned normally after session.stop(). - profiler.stop(session_completed_normally and not run_timed_out) + profiler.stop(session_completed_normally and not watchdog.fired) # Graceful drain runs on both the clean-finish and session- # failure paths (BenchmarkSession.run publishes ENDED in its own # finally, so a failed run still has a terminal snapshot worth @@ -1058,8 +1071,7 @@ def _on_phase_start(phase: PhaseConfig) -> None: ) raise finally: - if run_watchdog is not None: - run_watchdog.cancel() + watchdog.cancel() return BenchmarkResult( session=result, @@ -1067,7 +1079,7 @@ def _on_phase_start(phase: PhaseConfig) -> None: report=report, tmpfs_dir=tmpfs_dir, profiling=profiler.payload(), - run_timed_out=run_timed_out, + run_timed_out=watchdog.fired, ) From 7a496058b841d75e842a8a1dca05da9daa358bf4 Mon Sep 17 00:00:00 2001 From: Viraat Chandra Date: Fri, 14 Aug 2026 00:36:49 -0700 Subject: [PATCH 8/9] docs: current-state pass over the timeout surface Fix references left behind by the consolidation: renamed drain knob in session.py docstring, argv-vs-schema 0-sentinel wording in the aggregator snapshot/help text, config module pointers after the schema split, the from-config flag surface in CLI_QUICK_REFERENCE, regenerate-templates trigger lists, the config DESIGN nested-model table, and the compliance plan's duration-floor note. schema.py re-exports HTTPClientConfig again. --- AGENTS.md | 4 ++-- docs/CLI_DESIGN.md | 2 +- docs/CLI_QUICK_REFERENCE.md | 9 ++++----- docs/DEVELOPMENT.md | 2 +- .../services/metrics_aggregator/DESIGN.md | 3 ++- docs/compliance_audit_plan.md | 14 ++++++-------- docs/config/DESIGN.md | 19 ++++++++++--------- .../services/metrics_aggregator/__main__.py | 4 ++-- .../services/metrics_aggregator/snapshot.py | 2 +- src/inference_endpoint/config/schema.py | 2 ++ .../load_generator/session.py | 2 +- .../integration/commands/test_run_timeout.py | 6 +++--- tests/unit/config/test_timeouts.py | 11 +++++------ 13 files changed, 40 insertions(+), 40 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ffe50dedd..7c21a474d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -128,7 +128,7 @@ The aggregator is a separate process (`python -m inference_endpoint.async_utils. CLI is auto-generated from `config/schema.py` Pydantic models via cyclopts. Fields annotated with `cyclopts.Parameter(alias="--flag")` get flat shorthands; all other fields get auto-generated dotted flags (kebab-case). - **CLI mode** (`offline`/`online`): cyclopts constructs `OfflineBenchmarkConfig`/`OnlineBenchmarkConfig` (subclasses in `config/schema.py`) directly from CLI args. Type locked via `Literal`. `--dataset` is repeatable with TOML-style format `[perf|acc:][,key=value...]` (e.g. `--dataset data.csv,samples=500,parser.prompt=article`). Full accuracy support via `accuracy_config.eval_method=pass_at_1` etc. -- **YAML mode** (`from-config`): `BenchmarkConfig.from_yaml_file()` loads YAML, resolves env vars, and auto-selects the right subclass via Pydantic discriminated union. Optional `--timeout` (maps to `settings.timeouts.run_timeout_s`)/`--mode` overrides via `config.with_updates()`. +- **YAML mode** (`from-config`): `BenchmarkConfig.from_yaml_file()` loads YAML, resolves env vars, and auto-selects the right subclass via Pydantic discriminated union. Optional `--timeout` override maps to `settings.timeouts.run_timeout_s` via `config.with_updates()`; `--mode` selects the `TestMode` passed to the runner. - **eval**: Not yet implemented (raises `CLIError` with a tracking issue link) ### Config Construction & Validation @@ -315,7 +315,7 @@ All of these run automatically on commit: - `mypy` type checking - `prettier` for YAML/JSON/Markdown - License header enforcement -- `regenerate-templates`: auto-regenerates YAML config templates from schema defaults when `schema.py`, `config.py`, or `regenerate_templates.py` change +- `regenerate-templates`: auto-regenerates YAML config templates from schema defaults when any config schema module (`schema|audit|model_params|datasets|settings|timeouts`.py), `endpoint_client/config.py`, or `regenerate_templates.py` changes **IMPORTANT: Always run `pre-commit run --all-files` before every commit.** Hooks may modify files (prettier, ruff-format, license headers). If files are modified, stage the changes and commit once. Never commit without running pre-commit first. diff --git a/docs/CLI_DESIGN.md b/docs/CLI_DESIGN.md index 474c81fb9..9b9807a1f 100644 --- a/docs/CLI_DESIGN.md +++ b/docs/CLI_DESIGN.md @@ -67,7 +67,7 @@ Both paths produce the **same subclass with the same defaults**. A YAML file wit 2. **Auto-selects subclass.** `type: "offline"` → `OfflineBenchmarkConfig`, `type: "online"` → `OnlineBenchmarkConfig`, others → base `BenchmarkConfig`. -3. **Optional CLI overrides.** `--timeout` and `--mode` applied via `config.with_updates(...)` which re-runs validators. +3. **Optional CLI overrides.** `--timeout` maps into `settings.timeouts.run_timeout_s` via `with_updates(...)` (re-runs validators); `--mode` selects the `TestMode` passed to the runner and never touches the config object. ### Why subclasses? diff --git a/docs/CLI_QUICK_REFERENCE.md b/docs/CLI_QUICK_REFERENCE.md index b09f2d80d..5c6a4dbf7 100644 --- a/docs/CLI_QUICK_REFERENCE.md +++ b/docs/CLI_QUICK_REFERENCE.md @@ -102,9 +102,8 @@ Flag names shown as `--full.dotted.path --alias`. Both forms work. - `--endpoint-config.api-key --api-key` - API authentication - `--endpoint-config.api-type --api-type` - API type: openai/sglang (default: openai) - `--report-dir` - Report output directory - Note: applies to CLI-driven `benchmark offline` / `benchmark online`; `benchmark from-config` - does not expose a CLI override for `report_dir`. Set it in the YAML only if you need to control - the output location; otherwise a default report directory is used. + Note: `benchmark from-config` also accepts `--report-dir` as an override of the YAML value; + when neither is set a default report directory is used. - `--timeout` - Whole-run watchdog in seconds (off by default). If it fires, the run is aborted, the report is marked INTERRUPTED, and the process exits non-zero. - `--enable-cpu-affinity / --no-cpu-affinity` - NUMA-aware CPU pinning (default: true) - `--no-early-stopping` - opt out of the MLPerf early-stopping percentile estimates in `result_summary.json` (default: on; see [early_stopping.md](early_stopping.md)) @@ -307,8 +306,8 @@ inference-endpoint init submission # 3. Run (YAML mode) inference-endpoint benchmark from-config \ --config submission_template.yaml -# Note: from-config only accepts --config, --timeout, and --mode via CLI. -# Set report_dir in the YAML if you need a specific output location. +# from-config accepts --config, --timeout, --mode, --accuracy-only, and +# --report-dir; everything else comes from the YAML. ``` ### Validate First diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index c9dd2a3c8..aef0f5c9d 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -201,7 +201,7 @@ docs/short-description ## YAML Config Templates -Config templates in `src/inference_endpoint/config/templates/` are auto-generated from schema defaults. When you change `config/schema.py`, regenerate them: +Config templates in `src/inference_endpoint/config/templates/` are auto-generated from schema defaults. When you change any config schema module (`config/schema.py` and its sibling domain modules), regenerate them: ```bash uv run python scripts/regenerate_templates.py diff --git a/docs/async_utils/services/metrics_aggregator/DESIGN.md b/docs/async_utils/services/metrics_aggregator/DESIGN.md index 951cd9b24..48f0ed33f 100644 --- a/docs/async_utils/services/metrics_aggregator/DESIGN.md +++ b/docs/async_utils/services/metrics_aggregator/DESIGN.md @@ -122,7 +122,8 @@ COMPLETE event ─► trigger.fire ─► queue.enqueue(text, on_count) [ `--drain-timeout` and `--tokenizer-workers` have service-side defaults (`0` and `2`) so the service is launchable by hand without tuning knobs, but -`config/schema.py` is the single source of truth: the benchmark always +the config schema is the single source of truth (`settings.timeouts.metrics_drain_timeout_s` +in `config/timeouts.py`, `settings.metrics_tokenizer_workers` in `config/settings.py`): the benchmark always forwards the schema values (`--metrics-drain-timeout`, `--metrics-tokenizer-workers`), overriding these defaults in normal runs. diff --git a/docs/compliance_audit_plan.md b/docs/compliance_audit_plan.md index 4f0625b44..fa24f09cc 100644 --- a/docs/compliance_audit_plan.md +++ b/docs/compliance_audit_plan.md @@ -523,14 +523,12 @@ Two scenarios must be covered: **Offline** (`max_throughput`) and **SingleStream > catches a crashed run — but the examples default to equal for the clearest, least-contentious > comparison. -> **`min_duration` is not a duration floor (current limitation).** The load-generator stop -> check (`session.py`) halts a phase on **sample count** or **`max_duration_ms`** only; -> `min_duration_ms` merely _derives_ a count when no explicit count is set. Because TEST04 -> drives an explicit `samples` count, each phase stops at `samples` and `min_duration_ms` is -> **not** honored as a "run for at least 10 minutes" floor. MLCommons' 10-minute compliance -> minimum therefore is **not** enforced today; combining a count floor with a duration floor -> ("AND-semantics") is future work. Set `samples` large enough that each phase reaches a -> stable throughput on its own. +> **No duration floor (current limitation).** Runs are count-driven: the load-generator stop +> check (`session.py`) halts a phase on **sample count** or **`runtime.max_duration_ms`** +> only, and TEST04 drives explicit `samples` / `audit_samples` counts. MLCommons' 10-minute +> compliance minimum therefore is **not** enforced today; combining a count floor with a +> duration floor ("AND-semantics") is future work. Set `samples` large enough that each phase +> reaches a stable throughput on its own. Both scenarios ship as committed configs (see also [`compliance/audit_test/README.md`](../src/inference_endpoint/compliance/audit_test/README.md)): diff --git a/docs/config/DESIGN.md b/docs/config/DESIGN.md index b8208c909..e302b2f1d 100644 --- a/docs/config/DESIGN.md +++ b/docs/config/DESIGN.md @@ -45,13 +45,14 @@ the benchmark `type` and `endpoint_config`. Key nested models: -| Model | Purpose | -| ---------------- | --------------------------------------------------- | -| `LoadPattern` | Pattern type + parameters (target QPS, concurrency) | -| `RuntimeConfig` | Duration, sample count, RNG seeds | -| `ClientSettings` | Worker count and HTTP client settings | -| `EndpointConfig` | Endpoint URLs, API key | -| `Dataset` | Dataset path, type (performance / accuracy) | +| Model | Purpose | +| ------------------ | ----------------------------------------------------------- | +| `LoadPattern` | Pattern type + parameters (target QPS, concurrency) | +| `RuntimeConfig` | Sample count, perf-phase cap (`max_duration_ms`), RNG seeds | +| `Timeouts` | All global waits/deadlines (`settings.timeouts`) | +| `HTTPClientConfig` | Worker count and HTTP client settings (`settings.client`) | +| `EndpointConfig` | Endpoint URLs, API key | +| `Dataset` | Dataset path, type (performance / accuracy) | ### `RuntimeSettings` (frozen dataclass) @@ -145,8 +146,8 @@ to the report output. | Consumer | Usage | | ------------------------------- | ------------------------------------------------------------ | | `load_generator/session.py` | Receives `RuntimeSettings` at construction | -| `load_generator/scheduler.py` | Reads `load_pattern`, `n_samples_to_issue`, RNG seeds | +| `load_generator/strategy.py` | Reads `load_pattern`, `n_samples_to_issue`, RNG seeds | | `endpoint_client/config.py` | Reads `api_type`, `num_workers`, streaming mode | -| `metrics/reporter.py` | Reads `reported_metrics`, duration bounds | +| `metrics/report.py` | Reads `reported_metrics`, duration bounds | | `commands/benchmark/cli.py` | Defines benchmark subcommands and resolves CLI vs YAML input | | `commands/benchmark/execute.py` | Runs the benchmark lifecycle from resolved configuration | diff --git a/src/inference_endpoint/async_utils/services/metrics_aggregator/__main__.py b/src/inference_endpoint/async_utils/services/metrics_aggregator/__main__.py index d65bf3ab3..4320c9fa0 100644 --- a/src/inference_endpoint/async_utils/services/metrics_aggregator/__main__.py +++ b/src/inference_endpoint/async_utils/services/metrics_aggregator/__main__.py @@ -145,7 +145,7 @@ async def main() -> None: "Wall-clock budget (seconds) to finish tokenizing buffered samples " "after ENDED before the aggregator emits the final snapshot with " "n_pending_tasks > 0 (0 = wait indefinitely, the default; the " - "benchmark forwards the schema default, see config/schema.py). " + "benchmark forwards the schema default, see config/timeouts.py). " "Increase for very large datasets where the end-of-run tokenize " "batch is big." ), @@ -176,7 +176,7 @@ async def main() -> None: "In-process tokenizer threads for live (mid-run) ISL/OSL/TPOT " "(0 = no mid-run tokenization, everything defers to the " "end-of-run drain; the benchmark forwards the schema default, " - "see config/schema.py). The drain always uses the auto-sized " + "see config/settings.py). The drain always uses the auto-sized " "sharded pool — one worker process per 8-core block." ), ) diff --git a/src/inference_endpoint/async_utils/services/metrics_aggregator/snapshot.py b/src/inference_endpoint/async_utils/services/metrics_aggregator/snapshot.py index 5c5691d2b..8284bdf0a 100644 --- a/src/inference_endpoint/async_utils/services/metrics_aggregator/snapshot.py +++ b/src/inference_endpoint/async_utils/services/metrics_aggregator/snapshot.py @@ -45,7 +45,7 @@ class SessionState(str, Enum): LIVE → run in progress; tick task publishing live HDR-derived stats. DRAINING → ``SessionEventType.ENDED`` has been received; the aggregator is tokenizing the buffered samples (bounded by the - ``--drain-timeout`` budget — schema default 0 = unlimited). Tick task + ``--drain-timeout`` budget — argv 0 = unlimited; the schema knob ``settings.timeouts.metrics_drain_timeout_s`` uses None, converted at the argv boundary). Tick task continues at this stage, still HDR-derived; no new events will arrive. COMPLETE → terminal clean state. The ``publish_final()`` snapshot diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index 2c1046eb7..10a31eeb1 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -51,6 +51,7 @@ ) from ..core.types import APIType +from ..endpoint_client.config import HTTPClientConfig from ..exceptions import CLIError from ..utils import WithUpdatesMixin from .audit import AuditConfig, AuditTestId, OutputCachingTestConfig @@ -97,6 +98,7 @@ "EarlyStoppingConfig", "EndpointConfig", "EvalMethod", + "HTTPClientConfig", "LoadPattern", "LoadPatternType", "ModelParams", diff --git a/src/inference_endpoint/load_generator/session.py b/src/inference_endpoint/load_generator/session.py index 319a8be85..3aa98d64f 100644 --- a/src/inference_endpoint/load_generator/session.py +++ b/src/inference_endpoint/load_generator/session.py @@ -404,7 +404,7 @@ def stop_current_phase(self) -> None: Also sets the drain event: if the cap fires while the phase is already inside its ``_drain_inflight`` wait (strategy task finished), cancelling - the task is a no-op, so an unbounded (``performance_timeout_s: null``) + the task is a no-op, so an unbounded (``performance_drain_timeout_s: null``) drain would otherwise hang forever on a stuck in-flight response. """ self._current_phase_stopped = True diff --git a/tests/integration/commands/test_run_timeout.py b/tests/integration/commands/test_run_timeout.py index 96835c0c4..86dd0a358 100644 --- a/tests/integration/commands/test_run_timeout.py +++ b/tests/integration/commands/test_run_timeout.py @@ -16,9 +16,9 @@ """Whole-run watchdog (settings.timeouts.run_timeout_s) integration tests. Locking invariant: a fired run watchdog must never produce a COMPLETE -report. The watchdog SIGTERMs the metrics aggregator (whose handler writes -an INTERRUPTED final snapshot) before stopping the session, and -``run_benchmark`` exits non-zero via ``ExecutionError``. +report. The watchdog stops the session (ENDED still flows) and then +SIGTERMs the metrics aggregator, whose handler writes an INTERRUPTED +final snapshot; ``run_benchmark`` exits non-zero via ``ExecutionError``. """ import json diff --git a/tests/unit/config/test_timeouts.py b/tests/unit/config/test_timeouts.py index ef958a3f1..3abf8cd88 100644 --- a/tests/unit/config/test_timeouts.py +++ b/tests/unit/config/test_timeouts.py @@ -13,11 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Tests for the consolidated ``settings.timeouts`` block (Timeouts model), -the reworked ``runtime.max_duration_ms`` knob, and the hard removal of the -pre-consolidation config surface (``settings.drain``, top-level ``timeout``, -``settings.service_ready_timeout_s``, ``runtime.min_duration_ms``, and the -``settings.client.worker_*`` knobs).""" +"""Tests for the ``settings.timeouts`` block (Timeouts model), the +``runtime.max_duration_ms`` perf-phase cap, and the rejection of config +keys that do not exist (``settings.drain``, top-level ``timeout``, +``settings.service_ready_timeout_s``, ``runtime.min_duration_ms``).""" import random @@ -110,7 +109,7 @@ def test_extra_fields_rejected(self): class TestDeletedConfigSurface: - """Hard cutover: the pre-consolidation keys must error, not silently pass.""" + """Removed config keys must error via extra=forbid, not silently pass.""" @pytest.mark.unit def test_settings_drain_block_rejected(self): From b7c30c47db20679ce17d8ee3fddbcd32ea64ab0b Mon Sep 17 00:00:00 2001 From: Viraat Chandra Date: Fri, 14 Aug 2026 22:01:22 -0700 Subject: [PATCH 9/9] style(config): drop extra blank line left by rebase conflict resolution --- src/inference_endpoint/config/schema.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index 10a31eeb1..4de5a3864 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -154,7 +154,6 @@ class TestType(str, Enum): SUBMISSION = "submission" - class EndpointConfig(BaseModel): """Endpoint connection configuration.