Add Openvino support

src/winml/modelkit/commands/_perf_genai.py

@@ -328,6 +328,7 @@ class GenaiPerfConfig:
     model_id: str | None = None
     ep: EPNameOrAlias | None = None
     device: str = "auto"
+    provider_options: dict[str, str] | None = None
     prompt: str = _DEFAULT_PROMPT
     apply_template: bool = True
     max_new_tokens: int = 128
@@ -357,6 +358,7 @@ class _RequestSample:
     decode_token_durations_ms: list[float]
     sequence_fetch_duration_ms: float
     detokenization_duration_ms: float
+    response_text: str = ""
 
     @property
     def model_ttft_duration_ms(self) -> float:
@@ -457,6 +459,7 @@ class GenaiBenchmarkResult:
     timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
     prompt_tokens: int = 0
     generated_tokens: int = 0
+    response_text: str = ""
     context_length: int | None = None
     effective_ep: str | None = None
     effective_device: str | None = None
@@ -497,6 +500,7 @@ def to_dict(self) -> dict[str, Any]:
             },
             "requests": [sample.to_dict() for sample in self.requests],
             "aggregate": self._round_aggregate(),
+            "response_text": self.response_text,
         }
         if self.memory_profile:
             result["memory"] = self.memory_profile
@@ -557,6 +561,7 @@ def _build_session(self) -> GenaiSession:
             self._config.bundle_dir,
             self._config.ep,
             device=self._session_device(),
+            provider_options=self._config.provider_options,
             context_length=self._config.context_length,
             compile=self._config.compile,
             compile_timeout=self._config.compile_timeout,
@@ -747,6 +752,7 @@ def _time_one_generation(
             decode_token_durations_ms=[value * 1000.0 for value in timing.decode_s],
             sequence_fetch_duration_ms=timing.sequence_fetch_s * 1000.0,
             detokenization_duration_ms=timing.detokenization_s * 1000.0,
+            response_text=timing.response_text,
         )
 
     def _aggregate(
@@ -788,6 +794,7 @@ def _aggregate(
             effective_device=getattr(self._session, "effective_device", None),
             prompt_tokens=timed[0].prompt_tokens if timed else 0,
             generated_tokens=timed[0].generated_tokens if timed else 0,
+            response_text=timed[0].response_text if timed else "",
             context_length=self._session.context_length if self._session else None,
             load=load,
             requests=samples,
@@ -849,6 +856,10 @@ def display_genai_report(result: GenaiBenchmarkResult, console: Console) -> None
         f"[dim]Generated:[/dim] {result.generated_tokens} tokens "
         f"(max_new_tokens={cfg.max_new_tokens})"
     )
+    if result.response_text:
+        console.print()
+        console.print("[bold]Response[/bold]")
+        console.print(result.response_text, markup=False)
 
     load = result.load
     aggregate = result.aggregate

src/winml/modelkit/commands/perf.py

@@ -2390,7 +2390,6 @@ def _run_simple_loop(
 _GENAI_IGNORED_FLAGS: dict[str, str] = {
     "task": "--task",
     "precision": "--precision",
-    "ep_options": "--ep-options",
     "shape_config_path": "--shape-config",
     "input_specs": "--input-specs",
     "export_config": "--export-config",
@@ -2616,6 +2615,11 @@ def _run_genai_runtime(
         ep = cast("EPNameOrAlias", short_ep_name(target.ep)) if target is not None else None
         if target is not None:
             device = target.device
+    provider_options: dict[str, str] | None = p.get("ep_options")
+    if provider_options and ep is None:
+        raise click.UsageError(
+            "--ep-options requires --ep or a concrete --device with --runtime ort-genai."
+        )
 
     # Keep any bundle-lifetime resources alive across the benchmark.
     with contextlib.ExitStack() as stack:
@@ -2678,6 +2682,7 @@ def _run_genai_runtime(
             model_id=model,
             ep=ep,
             device=device,
+            provider_options=provider_options,
             prompt=prompt,
             apply_template=p["apply_template"],
             max_new_tokens=p["max_new_tokens"],
@@ -3042,6 +3047,7 @@ def perf(
         raise click.UsageError("A model is required via -m/--model.")
 
     ep_provider_options = cli_utils.parse_ep_options(ep_options)
+    ctx.params["ep_options"] = ep_provider_options
     if device_luid is not None and "device_id" in (ep_provider_options or {}):
         raise click.UsageError(
             "--device-luid cannot be combined with --ep-options device_id=...; "