diff --git a/families/qwen3_8/build_request.py b/families/qwen3_8/build_request.py new file mode 100644 index 0000000000..698784cc54 --- /dev/null +++ b/families/qwen3_8/build_request.py @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""qwen3_8 build inputs and strict compatibility for existing Python callers.""" + +from __future__ import annotations + +from dataclasses import dataclass, fields +from pathlib import Path +import re +from typing import ClassVar + +from tensorrt_model_connect.graph_transform import GraphTransform + + +def _validate_id(field: str, value: str) -> None: + if not isinstance(value, str) or re.fullmatch(r"[a-z][a-z0-9_]*", value) is None: + raise ValueError(f"{field} must be a lowercase identifier") + + +@dataclass(frozen=True) +class BuildRequest: + """qwen3_8-owned inputs; unsupported legacy controls are read-only defaults.""" + + model_dir: Path + output_path: Path + family: str + task: str + precision: str + backend: str = "trt" + max_sequence_length: int | None = None + image_height: ClassVar[int | None] = None + image_width: ClassVar[int | None] = None + video_num_frames: ClassVar[int | None] = None + max_batch_size: ClassVar[int] = 1 + tensor_parallel_size: int = 1 + context_parallel_size: ClassVar[int] = 1 + quantization: str | None = None + fp32_layers: tuple[int, ...] = () + dynamic_kv_cache: ClassVar[bool] = False + verbose: bool = False + graph_transform: GraphTransform | None = None + + def __post_init__(self) -> None: + if not self.precision: + raise ValueError("precision must be non-empty") + _validate_id("family", self.family) + _validate_id("task", self.task) + if self.backend not in {"trt", "trt_rtx"}: + raise ValueError("backend must be 'trt' or 'trt_rtx'") + if self.max_sequence_length is not None and self.max_sequence_length < 1: + raise ValueError("max_sequence_length must be positive") + for field in ("image_height", "image_width", "video_num_frames"): + value = getattr(self, field) + if value is not None and value < 1: + raise ValueError(f"{field} must be positive") + if self.max_batch_size < 1: + raise ValueError("max_batch_size must be positive") + if self.tensor_parallel_size < 1: + raise ValueError("tensor_parallel_size must be positive") + if self.context_parallel_size < 1: + raise ValueError("context_parallel_size must be positive") + if self.quantization is not None and not self.quantization: + raise ValueError("quantization must be non-empty when provided") + if any(layer < 0 for layer in self.fp32_layers): + raise ValueError("fp32_layers must contain non-negative indices") + if not isinstance(self.dynamic_kv_cache, bool): + raise ValueError("dynamic_kv_cache must be a bool") + if self.graph_transform is not None and not callable(self.graph_transform): + raise ValueError("graph_transform must be callable when provided") + + +def coerce_request(request: object) -> BuildRequest: + """Reject unsupported/unknown legacy inputs before converting to owner fields.""" + if isinstance(request, BuildRequest): + return request + unsupported = { + "image_height": None, + "image_width": None, + "video_num_frames": None, + "max_batch_size": 1, + "context_parallel_size": 1, + "dynamic_kv_cache": False, + } + for name, default in unsupported.items(): + value = getattr(request, name, default) + if name == "quantization" and value == "none": + continue + if value != default: + raise NotImplementedError(f"qwen3_8 does not support {name}") + names = {field.name for field in fields(BuildRequest)} + if unknown := set(vars(request)) - names - set(unsupported): + raise ValueError(f"unknown qwen3_8 build inputs: {sorted(unknown)}") + return BuildRequest(**{name: getattr(request, name) for name in names}) diff --git a/families/qwen3_8/cli.json b/families/qwen3_8/cli.json new file mode 100644 index 0000000000..77f79702fc --- /dev/null +++ b/families/qwen3_8/cli.json @@ -0,0 +1,142 @@ +{ + "version": 1, + "commands": [ + { + "name": "build", + "help": "Build one qwen3_8 TensorRT bundle", + "executor": "python", + "handler": "cli:build", + "arguments": [ + { + "name": "model", + "type": "string", + "help": "Hugging Face model ID or local snapshot" + }, + { + "name": "output", + "flags": [ + "-o", + "--output" + ], + "type": "path", + "required": true + }, + { + "name": "revision", + "flags": [ + "--revision" + ], + "type": "string" + }, + { + "name": "task", + "flags": [ + "--task" + ], + "type": "string", + "choices": [ + "text_generation" + ], + "default": "text_generation" + }, + { + "name": "precision", + "flags": [ + "--precision" + ], + "type": "string", + "choices": [ + "fp32", + "fp16", + "bf16" + ], + "help": "Compute precision (default: fp16 for paired execution, bf16 otherwise)" + }, + { + "name": "backend", + "flags": [ + "--backend" + ], + "type": "string", + "choices": [ + "trt", + "trt_rtx" + ], + "default": "trt" + }, + { + "name": "max_sequence_length", + "flags": [ + "--max-sequence-length" + ], + "type": "int" + }, + { + "name": "tensor_parallel_size", + "flags": [ + "--tensor-parallel-size" + ], + "type": "int", + "choices": [ + 1, + 2, + 4, + 8 + ], + "default": 1 + }, + { + "name": "verbose", + "flags": [ + "--verbose" + ], + "type": "bool", + "action": "store_true", + "default": false + }, + { + "name": "fp32_layers", + "flags": [ + "--fp32-layer" + ], + "type": "int", + "action": "append", + "default": [] + }, + { + "name": "quantization", + "flags": [ + "--quantization" + ], + "type": "string", + "choices": [ + "none", + "nvfp4", + "fp8" + ] + }, + { + "name": "execution_variant", + "flags": [ + "--execution-variant" + ], + "type": "string", + "choices": [ + "dspark" + ], + "help": "Explicit paired execution mode; no automatic draft discovery" + }, + { + "name": "companion", + "flags": [ + "--companion" + ], + "type": "string", + "action": "append", + "default": [], + "help": "Local companion checkpoint as ROLE=LOCAL_DIR" + } + ] + } + ] +} diff --git a/families/qwen3_8/cli.py b/families/qwen3_8/cli.py new file mode 100644 index 0000000000..db920cd573 --- /dev/null +++ b/families/qwen3_8/cli.py @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""qwen3_8-owned build command and typed inputs; importing this module is CPU-only.""" + +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path + +from tensorrt_model_connect.build import select_backend +from tensorrt_model_connect.bundle_writer import BundleWriter +from tensorrt_model_connect.graph_transform import graph_transform +from tensorrt_model_connect.model_support import load_model_metadata, resolve_family, resolve_model + +from .build_request import BuildRequest + +from .edge_llm.cli import execution_inputs +from .edge_llm.config import with_execution + + +def build_bundle(request: BuildRequest, output: Path) -> None: + """Build and publish through the owning model, preserving atomic failure.""" + request = replace(request, output_path=output) + select_backend(request.backend) + from .model import build as build_model + + writer = BundleWriter(output) + try: + with graph_transform(request.graph_transform): + build_model(request, writer) + writer.finish() + except BaseException: + writer.abort() + raise + +def build( + *, model: str, output: Path, revision: str | None = None, + task: str = "text_generation", precision: str | None = None, backend: str = "trt", + max_sequence_length: int | None = None, tensor_parallel_size: int = 1, + verbose: bool = False, + fp32_layers: list[int] | tuple[int, ...] = (), + quantization: str | None = None, + execution_variant: str | None = None, companion: list[str] | tuple[str, ...] = (), +) -> int: + """Run the declared owner command; help never imports this handler.""" + execution = execution_inputs(execution_variant, companion) + if precision is None: + precision = "fp16" if execution is not None else "bf16" + model_dir = resolve_model(model, revision) + resolve_family(load_model_metadata(model_dir), "qwen3_8") + request = BuildRequest( + model_dir=model_dir, output_path=output, family="qwen3_8", + task=task, precision=precision, backend=backend, + max_sequence_length=max_sequence_length, tensor_parallel_size=tensor_parallel_size, + verbose=verbose, + fp32_layers=tuple(fp32_layers), + quantization=quantization, + ) + if execution is not None: + request = with_execution(request, execution) + build_bundle(request, output) + return 0 diff --git a/families/qwen3_8/edge_llm/README.md b/families/qwen3_8/edge_llm/README.md new file mode 100644 index 0000000000..ec20c14e81 --- /dev/null +++ b/families/qwen3_8/edge_llm/README.md @@ -0,0 +1,107 @@ +# Qwen3.8 DSpark Edge-LLM adapter + +The Qwen3.8 family owns its configuration admission, ONNX command mapping, +bundle assets and C++ runtime orchestration. It does not reuse another Qwen +family. Standalone builds retain the original native path; this change admits +Edge only for an explicit mixed-NVFP4 target plus DSpark companion. + +## Build and inference + +Provision the optional native SDK with `TRTMC_EDGELLM_ALL_KERNELS=ON` and +`TRTMC_EDGELLM_ONNX=ON` using the +[pinned package instructions](../../../cmake/edge_llm/README.md). The source is +GitHub Edge-LLM 0.10.1 at `e8b29522938901f6df19ebeedd4b69bc8edbcd97`. +Configure `CMAKE_PREFIX_PATH` for the installed package and compile the runtime +with `TRTMC_ENABLE_EDGELLM=ON`. Cross compilation is unsupported. + +The Python build API accepts `BuildExecutionInputs(variant="dspark", +checkpoints=(NamedCheckpoint("draft", draft_path),))` on a family-owned typed request. The CLI equivalent adds `--execution-variant dspark` and +`--companion draft=/path/to/draft` to an ordinary build invocation. +The family invokes the original Edge Python ONNX exporter and native +`edgellm-onnx-build`; it does not alter source tensors or pad safetensors headers. +The exporter resolves the draft LM head from the target checkpoint. Both +speculative engines, embedding/head sidecars and tokenizer assets are bundled; +checkpoint weights are not duplicated in the bundle. + +The runtime calls the original Edge speculative inference constructor with +proposal block 7, verify 8, drafting topK 1/step 1 and DSpark scheduling disabled. +It preserves supported sampling controls. Engine preparation errors warn and +try native once with the same requested execution variant; native currently +rejects the unmapped DSpark variant explicitly. No failure substitutes an +ordinary base-only decoder. Inference errors propagate without fallback. + +## Validated exact profile + +- Target: `RadixArk/Qwen3.8-27B-NVFP4`, revision + `319f741cce68d7914884900c138a1fbb70a42f30`. +- Draft: `RadixArk/Qwen3.8-27B-DSpark`, revision + `b9a5dbdf03bc999c6c73c426b19c2d9041cea393`. +- Native SM120, CUDA 13.3, TensorRT 11.1.0.106, FP16 execution with source mixed + NVFP4/FP8 metadata retained, TP 1/batch 1, input/KV capacity 1024. +- Actual Model Connect paired build and public CLI inference passed. +- Independent greedy oracle: exact token match, NED 0.0 against 0.15. + The saved CPU FP32 reference was reused after checkpoint-byte and reference + function verification; it was not regenerated during this run. +- Original Edge `llm_basic` prompt and 128-token sampled profile + (temperature 1/topK 50/topP 1): ROUGE-1 **0.4246**, ROUGE-L **0.2458**, above + unchanged **0.25/0.20** gates. Chat enabled and thinking disabled. +- Publication regressions: 11 existing family Python tests plus 106 existing + builder/architecture tests pass; both existing native C++ tests pass. + +These results qualify only the exact text profile, not standalone NVFP4, +multimodal inputs, other checkpoints/platforms or statistical sampling parity. +The local run reused existing owning E2E helpers with an explicit companion; +this pair is not yet a registered pytest manifest case. + +A first inference attempt exposed incompatible development JSON headers sharing +Edge’s 3.12.0 version label. Matching the exact pinned headers fixed the crash +without changing Edge or the engines. The generic SDK now checks header content +rather than relying only on the version label. + +## Family-owned build options + +The existing family CLI reads this owner's cli.json and invokes cli.py. +Edge-specific inputs and selection remain in edge_llm/; the shared parser, +CLI protocol and build API gain no new options or hooks. +All variant validation and builder selection remain in this family. + +```sh +trtmc qwen3_8 build /path/to/target --precision fp16 \ + --execution-variant dspark --companion draft=/path/to/draft \ + -o model.bundle +``` + +Options may precede or follow MODEL. `trtmc qwen3_8 build /path/to/target --help` +shows these family options using local metadata; remote-ID help does not download +a checkpoint. For Python callers, use this family's request extension: + +```python +from tensorrt_model_connect import build +from families.qwen3_8.edge_llm.config import ( + BuildExecutionInputs, NamedCheckpoint, with_execution, +) + +# request is an ordinary BuildRequest owned by this family; draft_path is a Path. +build(with_execution(request, BuildExecutionInputs( + "dspark", (NamedCheckpoint("draft", draft_path),), +))) +``` + +A failed explicit pair is never replaced by a base-only bundle. Previously +recorded full-model results above are historical, not fresh refactor-head E2Es. + +Request controls and the existing 9–1024 capacity range are checked before any +Edge preparation. When CLI precision is omitted, paired execution defaults to +FP16 while ordinary native builds retain BF16. Explicit precision values are +unchanged; this paired profile accepts only FP16. Temporary staging uses the +output filesystem. + +## Declared build command + +This family uses the existing cli.json protocol introduced in #1310. The family +owns its declaration, typed inputs and Python handler. The handler adapts those +inputs to the unchanged builder API, preserving native/Edge dispatch and bundle +publication. The legacy flat build command remains available for its existing +ordinary options; new family options use `trtmc qwen3_8 build`. +Help is offline and does not need a local checkpoint. No shared parser hook or +family registry entry is added. diff --git a/families/qwen3_8/edge_llm/__init__.py b/families/qwen3_8/edge_llm/__init__.py new file mode 100644 index 0000000000..df93d8cabe --- /dev/null +++ b/families/qwen3_8/edge_llm/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Family-owned optional complete-network Edge offload.""" diff --git a/families/qwen3_8/edge_llm/builder.py b/families/qwen3_8/edge_llm/builder.py new file mode 100644 index 0000000000..c086d7fc09 --- /dev/null +++ b/families/qwen3_8/edge_llm/builder.py @@ -0,0 +1,210 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Family-owned paired adapter to the pinned Edge ONNX builder API.""" + +from __future__ import annotations + +import json +from pathlib import Path +import shutil +import subprocess + +from tensorrt_model_connect.build import cmake_prefixes, detect_local_platform, subprocess_environment + +EDGE_REVISION = "e8b29522938901f6df19ebeedd4b69bc8edbcd97" + + +def local_target() -> dict: + """Return the executing worker identity supplied by generic build mechanics.""" + return detect_local_platform() + + +def installed_package(target: dict) -> dict: + """Resolve CMake installation via standard prefixes; never install anything. + + Args: + target: Executing device and SDK identity. + + Returns: + Validated package metadata with absolute Python and plugin paths. + + Raises: + FileNotFoundError: No CMake installation or required artifact exists. + ValueError: Pin, architecture, SDK or contained-path contract differs. + """ + for prefix in cmake_prefixes(): + manifest = prefix / "share/trtmc/edge-llm.json" + if not manifest.is_file(): + continue + package = json.loads(manifest.read_text(encoding="utf-8")) + if package.get("schema_version") != 1 or package.get("revision") != EDGE_REVISION: + raise ValueError(f"Edge package has an unsupported revision/schema: {manifest}") + if package.get("version") != "0.10.1" or package.get("arch") != target["arch"]: + raise ValueError("Edge package version/architecture differs from executing worker") + if target["sm"] not in package.get("architectures", []): + raise ValueError("Edge package was not built for this local GPU") + cuda_version = ".".join(str(package.get("cuda_version", "")).split(".")[:2]) + if cuda_version != target["cuda_version"] or package.get("tensorrt_version") != target["tensorrt_version"]: + raise ValueError("Edge package CUDA/TensorRT differs from executing worker") + for name in ("python", "plugin") + (("onnx_builder",) if package.get("onnx") else ()): + relative = Path(package[name]) + path = (prefix / relative).resolve() + if relative.is_absolute() or not path.is_relative_to(prefix.resolve()): + raise ValueError(f"Edge package {name} must be contained in its installation") + if not path.is_file(): + raise FileNotFoundError(f"Edge package {name} is missing: {path}") + package[name] = str(path) + return package + raise FileNotFoundError("Edge-LLM is not installed; enable the optional Edge-LLM CMake dependency " + "and set CMAKE_PREFIX_PATH to its install prefix") + + + +def checkpoint_quantization(model_dir: Path, raw: dict) -> str | None: + """Admit plain weights or the documented mixed ModelOpt checkpoint. + + Edge owns per-layer decoding and FP8 KV interpretation. Require matching + embedded/sidecar metadata rather than inventing or converting a format. + """ + if any((model_dir / name).exists() for name in ("quantize_config.json", "quant_config.json")): + return None + embedded = raw.get("quantization_config") + nested = raw.get("text_config", {}).get("quantization_config") + sidecar = model_dir / "hf_quant_config.json" + if not embedded and not nested and not sidecar.exists(): + return "none" + if not isinstance(embedded, dict) or (nested and nested != embedded): + return None + if embedded.get("quant_method") != "modelopt" or embedded.get("quant_algo") != "MIXED_PRECISION": + return None + layers = embedded.get("quantized_layers") + if not isinstance(layers, dict) or not layers or any(not isinstance(v, dict) for v in layers.values()): + return None + if {v.get("quant_algo") for v in layers.values()} != {"FP8", "NVFP4"}: + return None + if not sidecar.is_file(): + return None + value = json.loads(sidecar.read_text(encoding="utf-8")) + quant = value.get("quantization") if isinstance(value, dict) else None + if not isinstance(quant, dict) or quant.get("quant_algo") != "MIXED_PRECISION": + return None + if quant.get("quantized_layers") != layers: + return None + if quant.get("kv_cache_quant_algo") != "FP8" or not embedded.get("kv_cache_scheme"): + return None + return "nvfp4" + +_PROMPT_PROGRAM = r"""import json, sys +from pathlib import Path +from transformers import AutoTokenizer +checkpoint = Path(sys.argv[sys.argv.index("--model-dir") + 1]) +engine = Path(sys.argv[sys.argv.index("--engine-dir") + 1]) +tokenizer = AutoTokenizer.from_pretrained(checkpoint, local_files_only=True, trust_remote_code=False) +slot = "Qwen38SingleUserContentSlot" +formats = {} +for thinking in (False, True): + options = dict(tokenize=False, add_generation_prompt=True, enable_thinking=thinking) + rendered = tokenizer.apply_chat_template([dict(role="user", content=slot)], **options) + if rendered.count(slot) != 1: + raise ValueError("Qwen3.8 source does not preserve a single user prompt") + prefix, suffix = rendered.split(slot) + whitespace = "".join(chr(cp) for cp in range(sys.maxunicode + 1) if chr(cp).isspace()) + for probe in ("", " leading and trailing ", "first\nsecond", "世界", whitespace + "text" + whitespace): + actual = tokenizer.apply_chat_template([dict(role="user", content=probe)], **options) + if actual != prefix + probe.strip() + suffix: + raise ValueError("Qwen3.8 source user content is not a prefix/suffix mapping") + formats[str(thinking).lower()] = dict(prefix=prefix, suffix=suffix) +(engine / "trtmc_single_user_prompts.json").write_text(json.dumps(formats, ensure_ascii=False)) +""" + + + +def prepare_dspark(request, raw: dict, target: dict, staging: Path, log_path: Path, + draft_dir: Path) -> tuple[dict, dict]: + """Map the paired request to the original exporter and native ONNX builder. + + Edge supplies the draft LM head from its target checkpoint. No safetensors + padding or weight conversion is needed, and baked weights are not bundled + twice. Both plans and their complete runtime assets must exist to publish. + """ + package = installed_package(target) + if package.get("onnx") is not True: + raise ValueError("Qwen3.8 DSpark requires an ONNX-enabled Edge SDK") + source, draft_dir = Path(request.model_dir).resolve(), draft_dir.resolve() + if checkpoint_quantization(source, raw) != "nvfp4": + raise ValueError("Qwen3.8 DSpark requires the mixed NVFP4 target") + if not list(source.glob("*.safetensors")) or not list(draft_dir.glob("*.safetensors")): + raise ValueError("Qwen3.8 DSpark requires both local safetensors checkpoints") + config = raw.get("text_config", raw) + limit = request.max_sequence_length or min(int(config["max_position_embeddings"]), 256) + if not 8 < limit <= 1024: + raise ValueError("Qwen3.8 DSpark requires capacity above verification size8 and at most1024") + checkpoint = staging / "edge_llm/checkpoint" + checkpoint.mkdir(parents=True) + for name in ("config.json", "tokenizer.json", "tokenizer_config.json", "generation_config.json", + "chat_template.jinja"): + if (source / name).is_file(): + shutil.copy2(source / name, checkpoint / name) + (checkpoint / "draft").mkdir() + shutil.copy2(draft_dir / "config.json", checkpoint / "draft/config.json") + engine, onnx = staging / "edge_llm/engine", staging / "onnx" + env = subprocess_environment( + {"EDGELLM_PLUGIN_PATH": package["plugin"]}, + prepend_paths={"LD_LIBRARY_PATH": str(Path(package["plugin"]).parent)}, + ) + for role, subdirectory, flag in (("draft", "dspark_draft", "--specDraft"), + ("base", "llm", "--specBase")): + commands = [ + [package["python"], "-I", "-m", "tensorrt_edgellm.scripts.export", + str(source), str(onnx), f"--dspark-{role}", "--dspark-draft-dir", str(draft_dir), + "--skip-visual", "--skip-audio"], + [package["onnx_builder"], "--onnxDir", str(onnx / subdirectory), + "--engineDir", str(engine), flag, "--maxInputLen", str(min(limit, 1024)), + "--maxKVCacheCapacity", str(limit), "--maxBatchSize", "1", + "--maxVerifyTreeSize", "8", "--maxDraftTreeSize", "7"], + ] + with log_path.open("a", encoding="utf-8") as log: + for command in commands: + log.write(json.dumps(command) + "\n") + log.flush() + subprocess.run(command, check=True, stdout=log, stderr=subprocess.STDOUT, + cwd=staging, env=env) + shutil.rmtree(onnx) # Only this preparation's successfully consumed intermediates. + with log_path.open("a", encoding="utf-8") as log: + subprocess.run([package["python"], "-I", "-c", _PROMPT_PROGRAM, + "--model-dir", str(checkpoint), "--engine-dir", str(engine)], + check=True, stdout=log, stderr=subprocess.STDOUT, cwd=staging, env=env) + required = ("spec_base.engine", "spec_draft.engine", "base_config.json", "draft_config.json", + "embedding.safetensors", "dspark_heads.safetensors", "dspark_heads_info.json", + "tokenizer.json", "tokenizer_config.json", + "processed_chat_template.json", "trtmc_single_user_prompts.json") + for name in required: + if not (engine / name).is_file() or (engine / name).stat().st_size == 0: + raise ValueError(f"Edge ONNX builder did not produce required artifact: {name}") + for role in ("base", "draft"): + built = json.loads((engine / f"{role}_config.json").read_text(encoding="utf-8")) + if built.get("spec_decode_type") != "dspark" or built.get("dspark_config", {}).get("block_size") != 7: + raise ValueError("Edge ONNX builder returned a different DSpark contract") + files = {} + for directory in (engine, checkpoint): + for path in sorted(directory.rglob("*")): + if path.is_symlink(): + raise ValueError(f"Edge output must not contain symlinks: {path}") + if path.is_file(): + files[path.relative_to(staging).as_posix()] = path + return files, { + "version": 1, "edge_revision": EDGE_REVISION, "target": target, "precision": "fp16", + "max_sequence_length": limit, "max_input_length": min(limit, 1024), "max_batch_size": 1, + "checkpoint_quantization": "nvfp4", "artifacts": list(files), + "execution_variant": "dspark", "builder_flow": "onnx", "dspark_block_size": 7, + } + + +def publish(request, writer, files: dict, marker: dict) -> None: + """Stream complete Edge sections; publication errors must not retry native.""" + writer.set_header(family=request.family, task=request.task, backend=request.backend) + for name, path in files.items(): + with path.open("rb") as source, writer.open_section(name) as destination: + shutil.copyfileobj(source, destination, length=1024 * 1024) + writer.add_json("edge_llm.json", marker) diff --git a/families/qwen3_8/edge_llm/cli.py b/families/qwen3_8/edge_llm/cli.py new file mode 100644 index 0000000000..b9f80bae10 --- /dev/null +++ b/families/qwen3_8/edge_llm/cli.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Qwen38-owned CLI options for explicit paired Edge execution.""" + +from pathlib import Path + +from .config import BuildExecutionInputs, NamedCheckpoint + + +def execution_inputs( + execution_variant: str | None, companion: list[str] | tuple[str, ...] = (), +) -> BuildExecutionInputs | None: + """Parse only explicit local inputs; no variant list or model acquisition.""" + if execution_variant is None: + if companion: + raise ValueError("--companion requires --execution-variant") + return None + if execution_variant not in ['dspark']: + raise ValueError("unsupported qwen3_8 execution variant") + checkpoints = [] + for value in companion: + role, separator, directory = value.partition("=") + if not separator or not role or not directory: + raise ValueError("--companion must be ROLE=LOCAL_DIR") + if "://" in directory: + raise ValueError("--companion requires a local directory, not a URI") + checkpoints.append(NamedCheckpoint(role, Path(directory))) + return BuildExecutionInputs(execution_variant, tuple(checkpoints)) diff --git a/families/qwen3_8/edge_llm/config.py b/families/qwen3_8/edge_llm/config.py new file mode 100644 index 0000000000..44a4ba17da --- /dev/null +++ b/families/qwen3_8/edge_llm/config.py @@ -0,0 +1,89 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Explicit Qwen38 paired-build inputs; no shared execution-mode contract.""" + +from dataclasses import dataclass, fields +from pathlib import Path +import re + +from ..build_request import BuildRequest, coerce_request + + +_ID = re.compile(r"[a-z][a-z0-9_]*\Z") + + +def _validate_id(field: str, value: object) -> str: + if not isinstance(value, str) or _ID.fullmatch(value) is None: + raise ValueError(f"{field} must be a lowercase identifier containing only letters, digits, and underscores") + return value + + +@dataclass(frozen=True) +class NamedCheckpoint: + """One explicitly named local checkpoint; the family owns role semantics.""" + + role: str + model_dir: Path + + def __post_init__(self) -> None: + _validate_id("checkpoint role", self.role) + if not isinstance(self.model_dir, Path): + raise TypeError("checkpoint model_dir must be a Path") + if not self.model_dir.is_dir(): + raise ValueError(f"checkpoint must be an existing local directory: {self.model_dir}") + + +@dataclass(frozen=True) +class BuildExecutionInputs: + """Optional family-owned execution variant and immutable local companions. + + Qwen38 validates these inputs without fetching or inferring companions. + """ + + variant: str + checkpoints: tuple[NamedCheckpoint, ...] = () + + def __post_init__(self) -> None: + _validate_id("execution variant", self.variant) + if not isinstance(self.checkpoints, tuple) or any( + not isinstance(checkpoint, NamedCheckpoint) for checkpoint in self.checkpoints + ): + raise TypeError("checkpoints must be a tuple of NamedCheckpoint values") + roles = [checkpoint.role for checkpoint in self.checkpoints] + if len(roles) != len(set(roles)): + raise ValueError("checkpoint roles must be unique") + self.validate_local() + + def validate_local(self) -> None: + """Recheck local availability before dispatch, without acquiring inputs.""" + for checkpoint in self.checkpoints: + if not checkpoint.model_dir.is_dir(): + raise ValueError( + f"checkpoint must be an existing local directory: {checkpoint.model_dir}" + ) + + +@dataclass(frozen=True) +class Qwen38BuildRequest(BuildRequest): + """Ordinary build inputs plus an explicitly requested Qwen38 execution recipe.""" + + execution: BuildExecutionInputs | None = None + + def __post_init__(self) -> None: + super().__post_init__() + if self.family != "qwen3_8": + raise ValueError("Qwen38BuildRequest requires the qwen3_8 family") + if self.execution is not None: + if not isinstance(self.execution, BuildExecutionInputs): + raise TypeError("execution must be BuildExecutionInputs") + self.execution.validate_local() + + +def with_execution(request: BuildRequest, execution: BuildExecutionInputs) -> Qwen38BuildRequest: + """Preserve supported ordinary request fields and callback identity.""" + request = coerce_request(request) + return Qwen38BuildRequest( + **{field.name: getattr(request, field.name) for field in fields(BuildRequest)}, + execution=execution, + ) diff --git a/families/qwen3_8/edge_llm/dispatch.py b/families/qwen3_8/edge_llm/dispatch.py new file mode 100644 index 0000000000..ba744a3e16 --- /dev/null +++ b/families/qwen3_8/edge_llm/dispatch.py @@ -0,0 +1,172 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Family-owned complete-network route map; native is the default.""" + +from __future__ import annotations + +import json +import logging +import os +from pathlib import Path +import tempfile +import traceback + +from . import builder as edge_llm + +_LOG = logging.getLogger(__name__) + +# Exact qualified native DSpark route; no unvalidated platform or ordinary offload. +EDGE_DISPATCH = { + ("linux", "x86_64", 120, "fp16"): edge_llm.prepare_dspark, +} + + +def request_matches(request) -> bool: + """Validate mapped request controls independently of checkpoint metadata.""" + return ( + request.backend == "trt" and request.task == "text_generation" + and request.precision.lower() == "fp16" + and request.quantization in {None, "nvfp4"} + and request.max_batch_size == request.tensor_parallel_size == request.context_parallel_size == 1 + and not request.dynamic_kv_cache and not request.fp32_layers and request.graph_transform is None + and all(value is None for value in (request.image_height, request.image_width, request.video_num_frames)) + ) + + +def candidate(request, raw: dict) -> bool: + """Return whether this family's model/request contract can delegate to Edge.""" + config = raw.get("text_config", raw) + source_quantization = edge_llm.checkpoint_quantization(Path(request.model_dir), raw) + return ( + isinstance(config, dict) + and raw.get("model_type") == "qwen3_5" + and ("output_gate_type" in config and "mlp_only_layers" not in config) + and config.get("linear_key_head_dim") == config.get("linear_value_head_dim") == 128 + and not config.get("num_experts") + and source_quantization == "nvfp4" + and request_matches(request) + ) + + +def build(request, writer, native, *, draft_dir: Path) -> None: + """Dispatch locally or warn and retry native once with the original request. + + Args: + request: Unmodified Model Connect build request. + writer: Unpublished bundle writer. + native: This family's original native builder callback. + + Raises: + Exception: Common input/publication error, or native build error with + Edge cause after a failed preparation. Cancellation never retries. + """ + raw = json.loads((Path(request.model_dir) / "config.json").read_text(encoding="utf-8")) + if not isinstance(raw, dict): + raise ValueError("checkpoint config.json must contain an object") + config = raw.get("text_config", raw) + if not isinstance(config, dict): + raise ValueError("checkpoint text_config must contain an object") + if not candidate(request, raw): + native(request, writer) + return + capacity = config.get("max_position_embeddings") + if type(capacity) is not int or capacity <= 0: + raise ValueError("checkpoint max_position_embeddings must be a positive integer") + if request.max_sequence_length and request.max_sequence_length > capacity: + raise ValueError("max_sequence_length exceeds checkpoint context capacity") + failure = None + descriptor, name = tempfile.mkstemp(prefix=f".{request.output_path.name}.edge-", suffix=".log", + dir=request.output_path.parent) + os.close(descriptor) + log_path = Path(name) + with tempfile.TemporaryDirectory( + prefix=f".{request.output_path.name}.edge-", dir=request.output_path.parent + ) as directory: + try: + target = edge_llm.local_target() + key = (target["os"], target["arch"], target["sm"], request.precision.lower()) + adapter = EDGE_DISPATCH.get(key) + if adapter is not None: + files, marker = adapter(request, raw, target, Path(directory), log_path, draft_dir) + except Exception as error: + failure = error + with log_path.open("a", encoding="utf-8") as log: + traceback.print_exception(error, file=log) + _LOG.warning("qwen3_8 Edge build failed: %s. Diagnostics: %s. " + "Retrying native once with the unchanged request.", error, log_path, exc_info=True) + except BaseException: + log_path.unlink(missing_ok=True) + raise + else: + # Edge preparation did not touch writer; publication cannot fallback. + if adapter is not None: + edge_llm.publish(request, writer, files, marker) + log_path.unlink() + return + log_path.unlink() # A platform non-match is not an Edge failure. + try: + native(request, writer) + except Exception as error: + if failure is not None: + raise error from failure + raise + + +def build_paired(request, writer, execution) -> None: + """Build the retained mixed-NVFP4 Qwen3.8 / DSpark block7 pair.""" + execution.validate_local() + + if execution.variant != "dspark" or tuple(x.role for x in execution.checkpoints) != ("draft",): + raise ValueError("Qwen3.8 paired execution requires variant=dspark and one draft checkpoint") + if request.precision.lower() != "fp16": + raise ValueError("Qwen3.8 DSpark requires --precision fp16; the native default is unchanged") + if not request_matches(request): + raise ValueError( + "Qwen3.8 DSpark requires backend=trt, text_generation, batch/TP/CP=1, " + "quantization unset or nvfp4, and no dynamic KV, FP32 layers, graph or media overrides" + ) + draft_dir = execution.checkpoints[0].model_dir + raw = json.loads((request.model_dir / "config.json").read_text()) + draft = json.loads((draft_dir / "config.json").read_text()) + if not candidate(request, raw): + raise ValueError("The retained Qwen3.8 DSpark pair requires a matching mixed-NVFP4 base") + base = raw.get("text_config", raw) + if not isinstance(draft, dict) or draft.get("architectures") != ["DSparkDraftModel"]: + raise ValueError("Expected a DSparkDraftModel companion") + for name in ("hidden_size", "vocab_size"): + if type(draft.get(name)) is not int or draft[name] != base.get(name): + raise ValueError(f"Qwen3.8 DSpark base and draft disagree on {name}") + if draft.get("num_target_layers") != base.get("num_hidden_layers"): + raise ValueError("Qwen3.8 DSpark target layer count differs from base") + config = draft.get("dspark_config") + if not isinstance(config, dict) or config.get("block_size", draft.get("block_size")) != 7: + raise ValueError("Qwen3.8 DSpark maps the upstream block7 / verify8 profile") + layers = config.get("target_layer_ids", draft.get("target_layer_ids")) + if (not isinstance(layers, list) or not layers + or any(type(i) is not int or not 0 <= i < base["num_hidden_layers"] for i in layers) + or len(set(layers)) != len(layers)): + raise ValueError("Invalid Qwen3.8 DSpark target layer IDs") + mask = config.get("mask_token_id", draft.get("mask_token_id")) + if type(mask) is not int or not 0 <= mask < base["vocab_size"]: + raise ValueError("Invalid Qwen3.8 DSpark mask token") + limit = request.max_sequence_length or min(base["max_position_embeddings"], 256) + if not 8 < limit <= 1024: + raise ValueError("Qwen3.8 DSpark requires max_sequence_length above 8 and at most 1024") + capacity = draft.get("max_position_embeddings") + if type(capacity) is not int or not 8 < limit <= capacity: + raise ValueError("Requested context exceeds DSpark draft capacity or block minimum") + if draft.get("quantization_config") or any( + (draft_dir / name).exists() + for name in ("hf_quant_config.json", "quantize_config.json", "quant_config.json") + ): + raise ValueError("This Qwen3.8 DSpark profile requires unquantized draft weights") + + def native_pair(original_request, original_writer): + # A failure must never replace the requested pair with base-only decoding. + raise NotImplementedError( + "Native Qwen3.8 does not implement the requested DSpark variant; " + "the qualified Edge route requires Linux x86_64, SM120 and FP16" + ) + + build(request, writer, native_pair, draft_dir=draft_dir) diff --git a/families/qwen3_8/model.py b/families/qwen3_8/model.py index 15688ae363..1c7bb2558c 100644 --- a/families/qwen3_8/model.py +++ b/families/qwen3_8/model.py @@ -82,6 +82,16 @@ def _runtime_config(model_dir: Path, config: ModelConfig, model: Qwen38Model, ** def build(request, writer) -> None: """Build one Qwen3.8 hybrid text-generation bundle.""" + from .build_request import coerce_request + + request = coerce_request(request) + + from .edge_llm.config import Qwen38BuildRequest + from .edge_llm.dispatch import build_paired + + if isinstance(request, Qwen38BuildRequest) and request.execution is not None: + build_paired(request, writer, request.execution) + return if request.dynamic_kv_cache: raise NotImplementedError("qwen3_8 does not support dynamic_kv_cache") diff --git a/families/qwen3_8/runtime/CMakeLists.txt b/families/qwen3_8/runtime/CMakeLists.txt index af423a3801..ac275f707c 100644 --- a/families/qwen3_8/runtime/CMakeLists.txt +++ b/families/qwen3_8/runtime/CMakeLists.txt @@ -21,7 +21,10 @@ target_link_libraries(trtmc_model_qwen3_8 PRIVATE nlohmann_json::nlohmann_json ${TRTMC_CUDART_LIBRARY} ) -target_compile_options(trtmc_model_qwen3_8 PRIVATE -Wall -Wextra -Wpedantic) +target_compile_options(trtmc_model_qwen3_8 PRIVATE + "$<$:-Wall;-Wextra;-Wpedantic>" + "$<$:-Xcompiler=-Wall,-Wextra>" +) set_target_properties(trtmc_model_qwen3_8 PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" BUILD_RPATH "\$ORIGIN" @@ -71,3 +74,5 @@ if(TRTMC_BUILD_TESTS) SKIP_RETURN_CODE 77 ) endif() + +include(edge_llm/Adapter.cmake) diff --git a/families/qwen3_8/runtime/edge_llm/Adapter.cmake b/families/qwen3_8/runtime/edge_llm/Adapter.cmake new file mode 100644 index 0000000000..8b96763a68 --- /dev/null +++ b/families/qwen3_8/runtime/edge_llm/Adapter.cmake @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Complete-network offload is family-owned and absent from native-only builds. +if(TARGET EdgeLLM::Core) + if(NOT TARGET EdgeLLM::Plugin) + message(FATAL_ERROR "Qwen38 Edge adapter requires the complete EdgeLLM package (Core and Plugin)") + endif() + target_sources(trtmc_model_qwen3_8 PRIVATE + "${CMAKE_CURRENT_LIST_DIR}/adapter.cpp" + "${CMAKE_CURRENT_LIST_DIR}/device_link.cu" + ) + target_compile_definitions(trtmc_model_qwen3_8 PRIVATE TRTMC_HAS_EDGE_LLM=1) + target_link_libraries(trtmc_model_qwen3_8 PRIVATE EdgeLLM::Core) + set_target_properties(trtmc_model_qwen3_8 PROPERTIES + CUDA_ARCHITECTURES "${EdgeLLM_CUDA_ARCHITECTURE}" + CUDA_SEPARABLE_COMPILATION ON + CUDA_RESOLVE_DEVICE_SYMBOLS ON + ) +endif() + +if(TARGET EdgeLLM::Core) + add_custom_command(TARGET trtmc_model_qwen3_8 POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ $ + VERBATIM + ) +endif() diff --git a/families/qwen3_8/runtime/edge_llm/adapter.cpp b/families/qwen3_8/runtime/edge_llm/adapter.cpp new file mode 100644 index 0000000000..4c294c1008 --- /dev/null +++ b/families/qwen3_8/runtime/edge_llm/adapter.cpp @@ -0,0 +1,279 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#include "families/qwen3_8/runtime/edge_llm/adapter.h" + +#include "families/qwen3_8/runtime/edge_llm/request.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace trtmc::qwen3_8::edge_llm { +namespace { +namespace fs = std::filesystem; + +/// Turn CUDA failures into caller-visible load or inference errors. +void check_cuda(cudaError_t result) { + if (result != cudaSuccess) + throw std::runtime_error(std::string("Qwen3.8 Edge CUDA error: ") + + cudaGetErrorString(result)); +} + +/// Reject an engine built for a different local GPU or CUDA/TensorRT runtime. +void validate_target(const nlohmann::json& target) { + utsname host{}; + if (uname(&host) != 0) + throw std::runtime_error("Cannot identify Qwen3.8 Edge runtime host"); + std::ifstream release("/etc/os-release"); + std::string line, os_version; + while (std::getline(release, line)) { + if (line.rfind("VERSION_ID=", 0) == 0) { + os_version = line.substr(11); + if (os_version.size() >= 2 && os_version.front() == char(34) && + os_version.back() == char(34)) + os_version = os_version.substr(1, os_version.size() - 2); + } + } + int device = 0, cuda_version = 0; + check_cuda(cudaGetDevice(&device)); + check_cuda(cudaRuntimeGetVersion(&cuda_version)); + cudaDeviceProp gpu{}; + check_cuda(cudaGetDeviceProperties(&gpu, device)); + const int trt_version = getInferLibVersion(); + const std::string trt = + std::to_string(trt_version / 10000) + "." + std::to_string((trt_version % 10000) / 100) + + "." + std::to_string(trt_version % 100) + "." + std::to_string(getInferLibBuildVersion()); + const std::string cuda = + std::to_string(cuda_version / 1000) + "." + std::to_string((cuda_version % 1000) / 10); + if (target.at("os") != "linux" || target.at("os_version") != os_version || + target.at("arch") != host.machine || target.at("sm") != gpu.major * 10 + gpu.minor || + target.at("cuda_version") != cuda || target.at("tensorrt_version") != trt) + throw std::runtime_error( + "Qwen3.8 Edge bundle requires its build GPU and CUDA/TensorRT stack"); +} + +/// Own extracted engine/checkpoint files until after the Edge runtime is destroyed. +class Artifacts { + public: + explicit Artifacts(const BundleReader& bundle, const nlohmann::json& marker) { + std::set names; + for (const auto& entry : marker.at("artifacts")) { + const auto name = entry.get(); + if (!safe_artifact_path(name) || !names.insert(name).second || + !bundle.find_section(name)) + throw std::runtime_error("Invalid Qwen3.8 Edge artifact: " + name); + } + std::vector required_files{ + "edge_llm/engine/tokenizer.json", "edge_llm/engine/tokenizer_config.json", + "edge_llm/engine/processed_chat_template.json", + "edge_llm/engine/trtmc_single_user_prompts.json", "edge_llm/checkpoint/config.json"}; + + for (const auto* name : + {"spec_base.engine", "spec_draft.engine", "base_config.json", "draft_config.json", + "embedding.safetensors", "dspark_heads.safetensors", "dspark_heads_info.json"}) + required_files.push_back(std::string("edge_llm/engine/") + name); + required_files.push_back("edge_llm/checkpoint/draft/config.json"); + for (const auto& required : required_files) + if (!names.count(required) || bundle.find_section(required)->length == 0) + throw std::runtime_error("Required Qwen3.8 Edge artifact missing: " + required); + std::string pattern = (fs::temp_directory_path() / "trtmc-qwen3_8-edge-XXXXXX").string(); + if (!mkdtemp(pattern.data())) + throw std::runtime_error("Cannot create Qwen3.8 Edge artifact directory"); + root_ = pattern; + try { + for (const auto& name : names) { + const auto destination = root_ / name; + fs::create_directories(destination.parent_path()); + std::ofstream output(destination, std::ios::binary); + bundle.copy_section(name, output); + output.close(); + if (!output) + throw std::runtime_error("Cannot extract Qwen3.8 Edge artifact: " + name); + } + } catch (...) { + cleanup(); + throw; + } + } + ~Artifacts() { cleanup(); } + Artifacts(const Artifacts&) = delete; + Artifacts& operator=(const Artifacts&) = delete; + std::string engine() const { return (root_ / "edge_llm/engine").string(); } + std::string checkpoint() const { return (root_ / "edge_llm/checkpoint").string(); } + + private: + void cleanup() noexcept { + std::error_code ignored; + fs::remove_all(root_, ignored); + } + fs::path root_; +}; + +/// Close the plugin handle after runtime destruction; registrations remain mapped. +struct CloseLibrary { + void operator()(void* handle) const noexcept { + if (handle) + dlclose(handle); + } +}; + +/// Initialize the CMake-installed adjacent plugin without process-global environment mutation. +std::unique_ptr load_plugin() { + Dl_info location{}; + if (!dladdr(reinterpret_cast(&create), &location) || !location.dli_fname) + throw std::runtime_error("Cannot locate Qwen3.8 family library"); + const auto path = + fs::absolute(location.dli_fname).parent_path() / "libNvInfer_edgellm_plugin.so"; + std::unique_ptr plugin( + dlopen(path.c_str(), RTLD_NOW | RTLD_GLOBAL | RTLD_NODELETE)); + if (!plugin) + throw std::runtime_error("Cannot load CMake-installed Edge plugin: " + + std::string(dlerror())); + using Initialize = bool (*)(void*, const char*); + auto initialize = reinterpret_cast(dlsym(plugin.get(), "initEdgellmPlugins")); + if (!initialize || !initialize(static_cast(&trt_edgellm::gLogger), "")) + throw std::runtime_error("Cannot initialize Qwen3.8 Edge plugin"); + return plugin; +} + +/// Stream ownership is independent of construction success and outlives the Edge instance. +class Stream { + public: + Stream() { check_cuda(cudaStreamCreateWithFlags(&value_, cudaStreamNonBlocking)); } + ~Stream() { cudaStreamDestroy(value_); } + Stream(const Stream&) = delete; + Stream& operator=(const Stream&) = delete; + cudaStream_t get() const { return value_; } + + private: + cudaStream_t value_{nullptr}; +}; + +/// Delegate the complete DSpark block7 algorithm to the pinned runtime. +std::unique_ptr make_runtime(const Artifacts& artifacts, + cudaStream_t stream) { + trt_edgellm::rt::SpecDecodeDraftingConfig drafting{}; + drafting.draftingTopK = 1; + drafting.draftingStep = 1; + drafting.verifySize = 8; + drafting.dflashBlockSize = 0; + drafting.dsparkSchedulerMode = trt_edgellm::rt::DSparkSchedulerMode::kOff; + drafting.dsparkConfidenceThreshold = 0.0F; + drafting.dsparkMinProposalLen = 1; + drafting.dsparkMaxProposalLen = 0; + return std::make_unique( + artifacts.engine(), "", std::unordered_map{}, drafting, stream, + trt_edgellm::rt::ContextCacheConfig{}, artifacts.checkpoint(), + (fs::path(artifacts.checkpoint()) / "draft").string()); +} + +/// Thin persistent Edge API adapter; serialization prevents concurrent use of Edge request state. +class EdgeTask final : public ITextGeneration { + public: + EdgeTask(const BundleReader& bundle, const nlohmann::json& marker) + : artifacts_(bundle, marker), plugin_(load_plugin()), + runtime_(make_runtime(artifacts_, stream_.get())), + capacity_(marker.at("max_sequence_length").get()), + input_limit_(marker.at("max_input_length").get()) { + std::ifstream input(fs::path(artifacts_.engine()) / "trtmc_single_user_prompts.json"); + source_prompts_ = nlohmann::json::parse(input); + for (const auto* mode : {"false", "true"}) + for (const auto* part : {"prefix", "suffix"}) + if (!source_prompts_.at(mode).at(part).is_string()) + throw std::runtime_error("Invalid Qwen3.8 source prompt mapping"); + } + + std::int32_t default_max_new_tokens() const override { return std::min(128, capacity_ - 1); } + + /// Drain work from failed requests before destroying the runtime and its weight buffers. + ~EdgeTask() override { cudaStreamSynchronize(stream_.get()); } + + /// Invoke Edge once; failures propagate without attempting native inference. + TextResult generate(const std::string& prompt, const TextGenerationConfig& config) override { + auto effective = config; + std::string source_prompt = prompt; + if (config.use_chat_template) { + const auto& format = source_prompts_.at(config.enable_thinking ? "true" : "false"); + source_prompt = format.at("prefix").get() + source_user_content(prompt) + + format.at("suffix").get(); + effective.use_chat_template = false; + } + auto request = make_request(source_prompt, effective, default_max_new_tokens()); + std::lock_guard lock(mutex_); + const auto counts = runtime_->countPromptTokens(request); + if (counts.size() != 1) + throw std::runtime_error("Qwen3.8 Edge returned invalid prompt counts"); + validate_capacity(counts.front(), input_limit_, capacity_, request.maxGenerateLength); + trt_edgellm::rt::LLMGenerationResponse response{}; + // Complete queued work before response/request storage is destroyed, + // including exception paths in a persistent task. + struct Drain { + cudaStream_t stream; + ~Drain() { cudaStreamSynchronize(stream); } + } drain{stream_.get()}; + if (!runtime_->handleRequest(request, response, stream_.get()) || + response.outputIds.size() != 1 || response.outputTexts.size() != 1 || + response.outputIds.front().empty() || + response.outputIds.front().size() > static_cast(request.maxGenerateLength)) + throw std::runtime_error("Qwen3.8 Edge generation failed"); + if (response.finishReasons.size() != 1 || + (response.finishReasons.front() != trt_edgellm::rt::FinishReason::kEndId && + response.finishReasons.front() != trt_edgellm::rt::FinishReason::kLength)) + throw std::runtime_error("Qwen3.8 Edge generation did not complete successfully"); + // This API does not expose per-request stage times; zero means unavailable. + return {std::move(response.outputTexts.front()), std::move(response.outputIds.front())}; + } + + private: + // Reverse destruction order keeps weights, plugin and stream alive throughout Edge teardown. + Artifacts artifacts_; + std::unique_ptr plugin_; + Stream stream_; + std::unique_ptr runtime_; + int capacity_; + int input_limit_; + nlohmann::json source_prompts_; + std::mutex mutex_; +}; +} // namespace + +ITask* create(const BundleReader& bundle) { + const auto bytes = bundle.read_section("edge_llm.json"); + const auto marker = nlohmann::json::parse(bytes.begin(), bytes.end()); + if (marker.at("version") != 1 || marker.at("edge_revision") != kRevision || + marker.at("max_sequence_length").get() <= 1 || + marker.at("max_input_length").get() <= 0 || + marker.at("max_input_length").get() > marker.at("max_sequence_length").get() || + marker.at("max_batch_size") != 1 || marker.at("precision") != "fp16" || + !marker.at("artifacts").is_array()) + throw std::runtime_error("Invalid Qwen3.8 Edge bundle contract"); + const auto variant = marker.value("execution_variant", ""); + if (!valid_execution_variant(variant, marker.value("dspark_block_size", 0))) + throw std::runtime_error("Unsupported Qwen3.8 Edge execution variant"); + if (marker.value("builder_flow", "") != "onnx" || + marker.value("checkpoint_quantization", "") != "nvfp4" || + marker.at("max_sequence_length").get() <= 8 || + marker.at("max_sequence_length").get() > 1024 || + marker.at("target").at("arch") != "x86_64" || marker.at("target").at("sm") != 120) + throw std::runtime_error("Qwen3.8 DSpark requires the ONNX builder flow"); + validate_target(marker.at("target")); + return new EdgeTask(bundle, marker); +} + +} // namespace trtmc::qwen3_8::edge_llm diff --git a/families/qwen3_8/runtime/edge_llm/adapter.h b/families/qwen3_8/runtime/edge_llm/adapter.h new file mode 100644 index 0000000000..86cf5a89ce --- /dev/null +++ b/families/qwen3_8/runtime/edge_llm/adapter.h @@ -0,0 +1,15 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include "trtmc/bundle.h" +#include "trtmc/task.h" + +namespace trtmc::qwen3_8::edge_llm { + +/// Create a persistent Edge task from a self-contained bundle; throws on load failure. +ITask* create(const BundleReader& bundle); + +} // namespace trtmc::qwen3_8::edge_llm diff --git a/families/qwen3_8/runtime/edge_llm/contract.h b/families/qwen3_8/runtime/edge_llm/contract.h new file mode 100644 index 0000000000..aedecb8588 --- /dev/null +++ b/families/qwen3_8/runtime/edge_llm/contract.h @@ -0,0 +1,61 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include "trtmc/task.h" + +#include +#include +#include +#include + +namespace trtmc::qwen3_8::edge_llm { + +inline constexpr const char* kRevision = "e8b29522938901f6df19ebeedd4b69bc8edbcd97"; + +/// DSpark proposal length excludes the one additional verification token. +inline bool valid_execution_variant(const std::string& variant, int block_size) { + return variant == "dspark" && block_size == 7; +} + +/// Return whether an artifact is a normalized file below one of the two Edge roots. +inline bool safe_artifact_path(const std::string& name) { + if (name.find('\\') != std::string::npos || name.find('\0') != std::string::npos) + return false; + const std::filesystem::path path(name); + if (path.is_absolute() || path.filename().empty()) + return false; + for (const auto& part : path) + if (part == "." || part == "..") + return false; + return path.generic_string() == name && + (name.rfind("edge_llm/engine/", 0) == 0 || name.rfind("edge_llm/checkpoint/", 0) == 0); +} + +/// Reject invalid sampling settings and controls with no equivalent Edge request API. +inline void validate_generation(const TextGenerationConfig& c) { + if (!std::isfinite(c.temperature) || c.temperature < 0 || !std::isfinite(c.top_p) || + c.top_p <= 0 || c.top_p > 1 || c.top_k < 0) + throw std::invalid_argument("Invalid Qwen3.8 Edge sampling parameters"); + if (c.min_p != 0 || c.seed != -1 || c.eos_token_id != -1 || c.repetition_penalty != 1 || + !c.lora_adapter_id.empty() || c.stop_on_boxed_answer || + (c.text_generation_mode != "auto" && c.text_generation_mode != "autoregressive") || + c.source_language_token_id != -1 || c.forced_bos_token_id != -1 || c.guidance_scale != -1 || + c.cfg_scale != -1 || c.num_steps != -1 || c.sde_gamma != -1 || !c.initial_latents.empty() || + !c.condition_latents.empty() || !c.condition_mask.empty() || !c.sampling_steps.empty() || + !c.sde_noises.empty() || c.block_length != 0 || c.confidence_threshold != -1) + throw std::invalid_argument( + "Requested generation controls are unsupported by Qwen3.8 Edge"); +} + +/// Enforce prompt and total capacity without allowing Edge to silently clip generation. +inline void validate_capacity(int prompt_tokens, int input_limit, int capacity, + std::int64_t generated_tokens) { + if (prompt_tokens <= 0 || prompt_tokens > input_limit || generated_tokens <= 0 || + generated_tokens > static_cast(capacity) - prompt_tokens) + throw std::invalid_argument("Qwen3.8 Edge prompt and generation exceed bundle capacity"); +} + +} // namespace trtmc::qwen3_8::edge_llm diff --git a/families/qwen3_8/runtime/edge_llm/device_link.cu b/families/qwen3_8/runtime/edge_llm/device_link.cu new file mode 100644 index 0000000000..1ea4768d60 --- /dev/null +++ b/families/qwen3_8/runtime/edge_llm/device_link.cu @@ -0,0 +1,5 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +// Enable the final CUDA device-link step for Edge's static runtime dependencies. diff --git a/families/qwen3_8/runtime/edge_llm/request.h b/families/qwen3_8/runtime/edge_llm/request.h new file mode 100644 index 0000000000..5a8e8a2e75 --- /dev/null +++ b/families/qwen3_8/runtime/edge_llm/request.h @@ -0,0 +1,60 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include "families/qwen3_8/runtime/edge_llm/contract.h" + +#include + +namespace trtmc::qwen3_8::edge_llm { + +/// Match the source Jinja trim filter without locale-dependent ASCII-only trimming. +inline std::string source_user_content(std::string text) { + // Python str.strip whitespace, including C0 separators and Unicode spaces. + constexpr const char* whitespace[] = { + "\t", "\n", "\v", "\f", "\r", + "\x1c", "\x1d", "\x1e", "\x1f", " ", + "\xc2\x85", "\xc2\xa0", "\xe1\x9a\x80", "\xe2\x80\x80", "\xe2\x80\x81", + "\xe2\x80\x82", "\xe2\x80\x83", "\xe2\x80\x84", "\xe2\x80\x85", "\xe2\x80\x86", + "\xe2\x80\x87", "\xe2\x80\x88", "\xe2\x80\x89", "\xe2\x80\x8a", "\xe2\x80\xa8", + "\xe2\x80\xa9", "\xe2\x80\xaf", "\xe2\x81\x9f", "\xe3\x80\x80"}; + bool changed = true; + while (changed && !text.empty()) { + changed = false; + for (const std::string space : whitespace) { + if (text.compare(0, space.size(), space) == 0) { + text.erase(0, space.size()); + changed = true; + } + if (text.size() >= space.size() && + text.compare(text.size() - space.size(), space.size(), space) == 0) { + text.resize(text.size() - space.size()); + changed = true; + } + } + } + if (text.rfind("", 0) == 0 && text.size() >= 16 && + text.compare(text.size() - 16, 16, "") == 0) + throw std::invalid_argument("Qwen3.8 single-user prompt contains no user query"); + return text; +} + +/// Map Model Connect text arguments to the pinned Edge API; rejects unmapped controls. +inline trt_edgellm::rt::LLMGenerationRequest +make_request(const std::string& prompt, const TextGenerationConfig& config, int default_length) { + validate_generation(config); + trt_edgellm::rt::LLMGenerationRequest request{}; + request.requests.resize(1); + request.requests.front().messages.push_back({"user", {{"text", prompt}}}); + request.applyChatTemplate = config.use_chat_template; + request.enableThinking = config.enable_thinking; + request.temperature = config.temperature; + request.topK = config.top_k; + request.topP = config.top_p; + request.maxGenerateLength = config.max_new_tokens > 0 ? config.max_new_tokens : default_length; + return request; +} + +} // namespace trtmc::qwen3_8::edge_llm diff --git a/families/qwen3_8/runtime/plugin.cpp b/families/qwen3_8/runtime/plugin.cpp index 49a8b4c57f..114a0f4533 100644 --- a/families/qwen3_8/runtime/plugin.cpp +++ b/families/qwen3_8/runtime/plugin.cpp @@ -9,6 +9,9 @@ #include "families/qwen3_8/runtime/plugin_helpers.h" #include "families/qwen3_8/runtime/recurrent_state.h" #include "trtmc/runtime/family_factory.h" +#ifdef TRTMC_HAS_EDGE_LLM +#include "families/qwen3_8/runtime/edge_llm/adapter.h" +#endif #include #include @@ -119,6 +122,14 @@ std::string chat_template(const BundleReader& bundle) { } // namespace ITask* create(const FamilyContext& context) { + if (context.reader.find_section("edge_llm.json")) { +#ifdef TRTMC_HAS_EDGE_LLM + return edge_llm::create(context.reader); +#else + throw std::runtime_error("Qwen3.8 Edge bundle requires a runtime configured with " + "-DTRTMC_ENABLE_EDGELLM=ON; rebuild and install Model Connect"); +#endif + } const RuntimeConfig config = parse_runtime_config(context.reader); auto decoder = load_engine(context.backend, require_section(context.reader, "engine.plan"), "qwen3_8 decoder"); diff --git a/families/qwen3_8/support.py b/families/qwen3_8/support.py index 6db925d744..a38cce3a96 100644 --- a/families/qwen3_8/support.py +++ b/families/qwen3_8/support.py @@ -13,7 +13,8 @@ default_precision="bf16", ) _SUPPORT = FamilySupport( - tasks=("text_generation",), default_task="text_generation", default_precision="bf16") + tasks=("text_generation",), default_task="text_generation", default_precision="bf16", +) def describe(metadata: ModelMetadata) -> FamilySupport | None: diff --git a/families/qwen3_8/tests/test_e2e.py b/families/qwen3_8/tests/test_e2e.py index cd5ad94989..7ea4915010 100644 --- a/families/qwen3_8/tests/test_e2e.py +++ b/families/qwen3_8/tests/test_e2e.py @@ -17,7 +17,7 @@ import pytest -from tensorrt_model_connect import BuildRequest, build +from families.qwen3_8.cli import build _TEST_DIR = Path(__file__).resolve().parent @@ -98,12 +98,13 @@ def _required_environment(tp_size: int): def _checkpoint(manifest: dict) -> Path: - from huggingface_hub import snapshot_download + from huggingface_hub import constants, snapshot_download path = Path( snapshot_download( repo_id=manifest["hf_id"], revision=manifest.get("hf_revision"), + local_files_only=constants.HF_HUB_OFFLINE, ) ) assert (path / "config.json").is_file(), path @@ -133,22 +134,23 @@ def _thresholds(case_name: str) -> dict[str, float]: return thresholds -def _build_bundle(manifest: dict, model_dir: Path, bundle: Path) -> None: +def _build_bundle(manifest: dict, model_dir: Path, bundle: Path, *, execution=None) -> None: quantization = manifest.get("quantization") assert quantization is None or isinstance(quantization, str) fp32_layers = tuple(manifest.get("fp32_layers", ())) build( - BuildRequest( - model_dir=model_dir, - output_path=bundle, - family=_FAMILY, - task="text_generation", - precision=manifest["precision"], - max_sequence_length=manifest["max_sequence_length"], - tensor_parallel_size=manifest["tensor_parallel_size"], - quantization=quantization, - fp32_layers=fp32_layers, - ) + model=str(model_dir), + output=bundle, + task="text_generation", + precision=manifest["precision"], + max_sequence_length=manifest["max_sequence_length"], + tensor_parallel_size=manifest["tensor_parallel_size"], + quantization=quantization, + fp32_layers=fp32_layers, + execution_variant=execution.variant if execution is not None else None, + companion=tuple( + f"{checkpoint.role}={checkpoint.model_dir}" for checkpoint in execution.checkpoints + ) if execution is not None else (), ) assert bundle.is_file() and bundle.stat().st_size > 0, bundle @@ -227,7 +229,7 @@ def _run_native( completed = subprocess.run( command, - check=True, + check=False, capture_output=True, text=True, timeout=600, @@ -235,6 +237,9 @@ def _run_native( ) record_evidence("commands", {"argv": getattr(completed, "args", None)}) record_evidence("native", {"stdout": getattr(completed, "stdout", None), "stderr": getattr(completed, "stderr", None)}) + (tmp_path / "native.stdout.log").write_text(completed.stdout, encoding="utf-8") + (tmp_path / "native.stderr.log").write_text(completed.stderr, encoding="utf-8") + completed.check_returncode() if tp_size == 1: return json.loads(completed.stdout) @@ -353,16 +358,102 @@ def _hf_reference( "bf16": torch.bfloat16, } assert reference_precision in dtypes, reference_precision - model = ( - AutoModelForCausalLM.from_pretrained( + if case.get("reference_decode_modelopt_mixed", False): + # Preserve the declared FP32 oracle; packed bytes are not floating weights. + # This does not emulate compiled activation or KV quantization. + from modelopt.torch.export.quant_utils import QUANTIZATION_FP8, from_quantized_weight + from modelopt.torch.quantization.qtensor import NVFP4QTensor + from safetensors.torch import load_file + from transformers import AutoConfig, GenerationConfig, Qwen3_5ForCausalLM + + assert not trust_remote_code and reference_precision == "fp32" + raw = json.loads((model_dir / "config.json").read_text()) + quant = json.loads((model_dir / "hf_quant_config.json").read_text())["quantization"] + embedded = raw["quantization_config"] + assert embedded["quant_method"] == "modelopt" + assert embedded["quant_algo"] == quant["quant_algo"] == "MIXED_PRECISION" + layers = quant["quantized_layers"] + assert layers and embedded["quantized_layers"] == layers + assert all( + policy["quant_algo"] == "FP8" + or (policy["quant_algo"] == "NVFP4" and policy["group_size"] == 16) + for policy in layers.values() + ) + state = {} + for shard in sorted(model_dir.glob("*.safetensors")): + tensors = load_file(str(shard), device="cpu") + assert not state.keys() & tensors.keys(), "duplicate checkpoint tensors" + state.update(tensors) + packed = {key for key, value in state.items() if value.dtype == torch.uint8} + fp8 = { + key for key, value in state.items() + if key.endswith(".weight") and value.dtype == torch.float8_e4m3fn + } + quantized = packed | fp8 + assert packed and fp8 and quantized == {key + ".weight" for key in layers} + assert all(layers[key.removesuffix(".weight")]["quant_algo"] == "NVFP4" for key in packed) + assert all(layers[key.removesuffix(".weight")]["quant_algo"] == "FP8" for key in fp8) + for key in packed: + weight = state[key] + assert key.endswith(".weight") and weight.ndim == 2, key + assert weight.shape[-1] % 8 == 0, key + prefix = key.removesuffix("weight") + scale = state[prefix + "weight_scale"] + double_scale = state[prefix + "weight_scale_2"] + shape = (weight.shape[0], weight.shape[1] * 2) + assert scale.dtype == torch.float8_e4m3fn + assert scale.shape == (shape[0], shape[1] // 16), key + assert torch.isfinite(scale.float()).all() and (scale.float() >= 0).all(), key + assert double_scale.numel() == 1 and torch.isfinite(double_scale).all(), key + assert (double_scale > 0).all(), key + state[key] = NVFP4QTensor(shape, torch.float32, weight).dequantize( + dtype=torch.float32, scale=scale, double_scale=double_scale, + block_sizes={-1: 16}, fast=False, + ) + assert state[key].shape == shape and torch.isfinite(state[key]).all(), key + for key in fp8: + weight = state[key] + scale = state[key.removesuffix("weight") + "weight_scale"] + assert weight.ndim == 2 and scale.numel() == 1, key + assert torch.isfinite(scale).all() and (scale > 0).all(), key + state[key] = from_quantized_weight( + weight, scale, QUANTIZATION_FP8, torch.float32, + ) + assert state[key].shape == weight.shape and torch.isfinite(state[key]).all(), key + for key in list(state): + if key.endswith((".weight_scale", ".weight_scale_2", ".input_scale")): + assert key.rsplit(".", 1)[0] + ".weight" in quantized, key + scale = state.pop(key) + assert torch.isfinite(scale.float()).all() and (scale.float() >= 0).all(), key + # Use the original official text-only class and checkpoint prefix conversion. + # HF owns its declared ignoring of unused visual/MTP keys; do not filter weights. + config = AutoConfig.from_pretrained( + model_dir, local_files_only=True, trust_remote_code=False, + ).get_text_config(decoder=True) + assert config.model_type == "qwen3_5_text" + assert not getattr(config, "quantization_config", None) + model, loading = Qwen3_5ForCausalLM.from_pretrained( + None, config=config, state_dict=state, dtype=torch.float32, + output_loading_info=True, + ) + assert all(not loading.get(key) for key in ( + "missing_keys", "unexpected_keys", "mismatched_keys", "error_msgs", + )), loading + if (model_dir / "generation_config.json").is_file(): + model.generation_config = GenerationConfig.from_pretrained( + model_dir, local_files_only=True, + ) + del state, tensors + else: + model = AutoModelForCausalLM.from_pretrained( model_dir, local_files_only=True, trust_remote_code=trust_remote_code, dtype=dtypes[reference_precision], ) - .eval() - .to("cuda") - ) + reference_device = case.get("reference_device", "cuda") + assert reference_device in {"cpu", "cuda"}, reference_device + model = model.eval().to(reference_device) inputs = _render_prompt(tokenizer, prompt, case).to(model.device) prompt_ids = inputs["input_ids"][0].tolist() if "expected_prompt_token_ids" in case: diff --git a/families/qwen3_8/tests/test_support.py b/families/qwen3_8/tests/test_support.py index af5d5169a4..619197e79a 100644 --- a/families/qwen3_8/tests/test_support.py +++ b/families/qwen3_8/tests/test_support.py @@ -1,6 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import pytest + from tensorrt_model_connect.model_support import ModelMetadata, resolve_family @@ -16,3 +18,332 @@ def test_qwen38_marker_default_task_is_checkpoint_owned() -> None: ) assert family == "qwen3_8" assert support.default_task == "text_generation" + from tensorrt_model_connect.family_cli import load_family_cli + assert load_family_cli("qwen3_8")["commands"][0]["handler"] == "cli:build" + + +def _edge_cli_source(tmp_path): + import json + + source = tmp_path / "target" + source.mkdir() + (source / "config.json").write_text(json.dumps({ + "model_type": "qwen3_5", "text_config": {"output_gate_type": "sigmoid"}, + })) + draft = tmp_path / "draft" + draft.mkdir() + return source, draft + + +@pytest.mark.parametrize("precision", [None, "fp16", "fp32", "bf16"]) +def test_edge_cli_uses_ordinary_family_build(tmp_path, monkeypatch, precision): + from tensorrt_model_connect import family_cli as build_cli + from families.qwen3_8.edge_llm import dispatch + from families.qwen3_8.edge_llm.config import Qwen38BuildRequest + + source, draft = _edge_cli_source(tmp_path) + output = tmp_path / "pair.bundle" + seen = [] + + def paired(request, writer, execution): + assert isinstance(request, Qwen38BuildRequest) + assert request.execution is execution + assert execution.variant == "dspark" + assert [(item.role, item.model_dir) for item in execution.checkpoints] == [("draft", draft)] + seen.append(request) + writer.set_header(family="qwen3_8", task=request.task, backend=request.backend) + writer.add_json("edge-test.json", {"variant": execution.variant}) + + monkeypatch.setattr(dispatch, "build_paired", paired) + options = ["--precision", precision] if precision else [] + assert build_cli.main(["qwen3_8", + "build", str(source), *options, "-o", str(output), + "--execution-variant", "dspark", "--companion", f"draft={draft}", + ]) == 0 + assert len(seen) == 1 + assert seen[0].precision == (precision or "fp16") + assert output.is_file() + + from families.qwen3_8.tests.test_e2e import _build_bundle + from families.qwen3_8.edge_llm.config import BuildExecutionInputs, NamedCheckpoint + + _build_bundle( + {"precision": "fp16", "max_sequence_length": 64, "tensor_parallel_size": 1}, + source, output, + execution=BuildExecutionInputs("dspark", (NamedCheckpoint("draft", draft),)), + ) + assert len(seen) == 2 + assert seen[-1].precision == "fp16" + assert seen[-1].max_sequence_length == 64 + assert output.is_file() + + +@pytest.mark.parametrize("options", [ + ["--companion", "draft=/missing"], + ["--execution-variant", "dspark", "--companion", "missing_separator"], + ["--execution-variant", "dspark", "--companion", "=path"], + ["--execution-variant", "dspark", "--companion", "draft="], + ["--execution-variant", "dspark", "--companion", "draft=https://example.com/model"], +]) +def test_bad_edge_cli_inputs_fail_before_backend(tmp_path, monkeypatch, options): + from tensorrt_model_connect import family_cli as build_cli + from families.qwen3_8 import cli as owner + source, _ = _edge_cli_source(tmp_path) + monkeypatch.setattr(owner, "select_backend", lambda *_: pytest.fail("backend touched")) + monkeypatch.setattr(owner, "BundleWriter", lambda *_: pytest.fail("writer created")) + with pytest.raises(ValueError): + build_cli.main(["qwen3_8", "build", str(source), "-o", str(tmp_path / "out"), *options]) + + +def test_edge_cli_help_is_family_owned(tmp_path, capsys): + from tensorrt_model_connect import family_cli as build_cli + + source, _ = _edge_cli_source(tmp_path) + with pytest.raises(SystemExit) as caught: + build_cli.main(["qwen3_8", "build", str(source), "--help"]) + assert caught.value.code == 0 + help_text = capsys.readouterr().out + assert "--execution-variant {dspark}" in help_text + assert "--companion" in help_text + + +def test_edge_request_preserves_fields_and_family_owner(tmp_path): + from families.qwen3_8.edge_llm import cli + from dataclasses import fields, replace + from families.qwen3_8.build_request import BuildRequest, coerce_request + from families.qwen3_8.edge_llm.config import ( + BuildExecutionInputs, NamedCheckpoint, with_execution, + ) + + source, draft = _edge_cli_source(tmp_path) + request = BuildRequest(source, tmp_path / "out", "qwen3_8", "text_generation", "fp16", + graph_transform=lambda layer: layer) + assert coerce_request(request) is request + execution = BuildExecutionInputs("dspark", (NamedCheckpoint("draft", draft),)) + assert cli.execution_inputs(None) is None + extended = with_execution(request, execution) + for field in fields(BuildRequest): + assert getattr(extended, field.name) is getattr(request, field.name) + with pytest.raises(ValueError, match="requires the qwen3_8 family"): + with_execution(replace(request, family="other"), execution) + with pytest.raises(ValueError, match="unique"): + BuildExecutionInputs("dspark", execution.checkpoints * 2) + + +@pytest.mark.parametrize("failure", [RuntimeError("paired build failed"), KeyboardInterrupt()]) +def test_edge_cli_failure_preserves_existing_bundle(tmp_path, monkeypatch, failure): + from tensorrt_model_connect import family_cli as build_cli + from families.qwen3_8.edge_llm import dispatch + + source, draft = _edge_cli_source(tmp_path) + output = tmp_path / "pair.bundle" + output.write_bytes(b"previous publication") + + def fail(request, writer, execution): + writer.set_header(family="qwen3_8", task=request.task, backend=request.backend) + writer.add_json("edge-test.json", {"variant": execution.variant}) + raise failure + + monkeypatch.setattr(dispatch, "build_paired", fail) + with pytest.raises(type(failure)) as caught: + build_cli.main(["qwen3_8", + "build", str(source), "-o", str(output), "--execution-variant", "dspark", + "--companion", f"draft={draft}", + ]) + assert caught.value is failure + assert output.read_bytes() == b"previous publication" + assert sorted(item.name for item in tmp_path.iterdir()) == ["draft", "pair.bundle", "target"] + + +def test_edge_pair_requires_draft_and_rechecks_local_inputs(tmp_path): + from tensorrt_model_connect.build import BuildRequest + from families.qwen3_8.edge_llm.config import BuildExecutionInputs, NamedCheckpoint + from families.qwen3_8.edge_llm.dispatch import build_paired + + source, draft = _edge_cli_source(tmp_path) + request = BuildRequest(source, tmp_path / "out", "qwen3_8", "text_generation", "fp16") + with pytest.raises(ValueError, match="paired execution requires"): + build_paired(request, None, BuildExecutionInputs("dspark")) + execution = BuildExecutionInputs("dspark", (NamedCheckpoint("draft", draft),)) + draft.rmdir() + with pytest.raises(ValueError, match="existing local directory"): + build_paired(request, None, execution) + + +@pytest.mark.parametrize("model_type", ["qwen38", "qwen3.8", "qwen3_8"]) +def test_qwen38_aliases_register_the_same_family_cli(model_type): + from families.qwen3_8.support import describe + + support = describe(ModelMetadata({"model_type": model_type}, {})) + assert support is not None + from tensorrt_model_connect.family_cli import load_family_cli + assert load_family_cli("qwen3_8")["commands"][0]["handler"] == "cli:build" + + +def _dspark_pair_request(tmp_path): + import json + from tensorrt_model_connect.build import BuildRequest + from families.qwen3_8.edge_llm.config import BuildExecutionInputs, NamedCheckpoint + + source, draft = _edge_cli_source(tmp_path) + layers = {"layer0": {"quant_algo": "FP8"}, "layer1": {"quant_algo": "NVFP4"}} + base = { + "model_type": "qwen3_5", "output_gate_type": "sigmoid", + "hidden_size": 16, "vocab_size": 32, "num_hidden_layers": 8, + "max_position_embeddings": 4096, "linear_key_head_dim": 128, + "linear_value_head_dim": 128, "quantization_config": { + "quant_method": "modelopt", "quant_algo": "MIXED_PRECISION", + "quantized_layers": layers, "kv_cache_scheme": {"type": "float"}, + }, + } + (source / "config.json").write_text(json.dumps(base)) + (source / "hf_quant_config.json").write_text(json.dumps({"quantization": { + "quant_algo": "MIXED_PRECISION", "quantized_layers": layers, + "kv_cache_quant_algo": "FP8", + }})) + (draft / "config.json").write_text(json.dumps({ + "architectures": ["DSparkDraftModel"], "hidden_size": 16, "vocab_size": 32, + "num_target_layers": 8, "max_position_embeddings": 4096, + "dspark_config": {"block_size": 7, "target_layer_ids": [1], "mask_token_id": 3}, + })) + return ( + BuildRequest(source, tmp_path / "out", "qwen3_8", "text_generation", "fp16"), + BuildExecutionInputs("dspark", (NamedCheckpoint("draft", draft),)), + ) + + +@pytest.mark.parametrize("overrides, message", [ + ({"precision": "bf16"}, "--precision fp16"), + ({"backend": "trt_rtx"}, "requires backend=trt"), + ({"max_batch_size": 2}, "batch/TP/CP=1"), + ({"tensor_parallel_size": 2}, "batch/TP/CP=1"), + ({"dynamic_kv_cache": True}, "no dynamic KV"), + ({"quantization": "fp8"}, "quantization unset or nvfp4"), + ({"max_sequence_length": 8}, "above 8 and at most 1024"), + ({"max_sequence_length": 1025}, "above 8 and at most 1024"), + ({"max_sequence_length": 2048}, "above 8 and at most 1024"), +]) +def test_dspark_request_errors_precede_adapter_work(tmp_path, monkeypatch, overrides, message): + from dataclasses import replace + from families.qwen3_8.edge_llm import dispatch + + request, execution = _dspark_pair_request(tmp_path) + monkeypatch.setattr(dispatch, "build", lambda *_args, **_kw: pytest.fail("adapter work started")) + with pytest.raises(ValueError, match=message) as caught: + dispatch.build_paired(replace(request, **overrides), None, execution) + assert caught.value.__cause__ is None + assert not list(tmp_path.glob(".out.edge-*")) + + +@pytest.mark.parametrize("limit", [9, 1024]) +def test_dspark_capacity_boundaries_keep_the_requested_pair(tmp_path, monkeypatch, limit): + from dataclasses import replace + from families.qwen3_8.edge_llm import dispatch + + request, execution = _dspark_pair_request(tmp_path) + request = replace(request, max_sequence_length=limit) + seen = [] + + def build(original, writer, native, *, draft_dir): + assert original is request and draft_dir == execution.checkpoints[0].model_dir + with pytest.raises(NotImplementedError, match="Native Qwen3.8"): + native(original, writer) + seen.append(original) + + monkeypatch.setattr(dispatch, "build", build) + dispatch.build_paired(request, None, execution) + assert seen == [request] + + +def test_dspark_checkpoint_error_is_distinct_from_request_error(tmp_path, monkeypatch): + from families.qwen3_8.edge_llm import dispatch + + request, execution = _dspark_pair_request(tmp_path) + (request.model_dir / "hf_quant_config.json").unlink() + monkeypatch.setattr(dispatch, "build", lambda *_args, **_kw: pytest.fail("adapter work started")) + with pytest.raises(ValueError, match="matching mixed-NVFP4 base"): + dispatch.build_paired(request, None, execution) + + +@pytest.mark.parametrize("arch, sm", [("x86_64", 90), ("aarch64", 120)]) +def test_dspark_unqualified_platform_names_required_route(tmp_path, monkeypatch, arch, sm): + from families.qwen3_8.edge_llm import builder, dispatch + + request, execution = _dspark_pair_request(tmp_path) + monkeypatch.setattr(builder, "local_target", lambda: {"os": "linux", "arch": arch, "sm": sm}) + with pytest.raises(NotImplementedError, match="Linux x86_64, SM120 and FP16"): + dispatch.build_paired(request, None, execution) + assert not list(tmp_path.glob(".out.edge-*")) + + +@pytest.mark.parametrize("options", [[], ["--precision", "fp16", "--max-sequence-length", "64"]]) +def test_declared_build_matches_legacy_request(tmp_path, monkeypatch, options): + """Owner command preserves ordinary request defaults and explicit controls.""" + import json + from families.qwen3_8 import cli as owner + from tensorrt_model_connect import build_cli, family_cli + + source = tmp_path / "checkpoint" + source.mkdir() + (source / "config.json").write_text(json.dumps({"model_type": "qwen3_8"})) + output = tmp_path / "model.bundle" + captured = [] + monkeypatch.setattr(owner, "build_bundle", lambda request, output: captured.append(request)) + monkeypatch.setattr(build_cli, "build", captured.append) + args = [str(source), "-o", str(output), *options] + assert family_cli.main(["qwen3_8", "build", *args]) == 0 + assert build_cli.main(["build", *args, "--family", "qwen3_8"]) == 0 + assert len(captured) == 2 + from dataclasses import fields + assert isinstance(captured[0], owner.BuildRequest) + for field in fields(captured[1]): + assert getattr(captured[0], field.name) == getattr(captured[1], field.name) + from dataclasses import replace + from families.qwen3_8.build_request import coerce_request + assert coerce_request(captured[1]) == captured[0] + with pytest.raises(NotImplementedError, match="image_height"): + coerce_request(replace(captured[1], image_height=32)) + from types import SimpleNamespace + with pytest.raises(ValueError, match="unknown"): + coerce_request(SimpleNamespace(**vars(captured[1]), unexpected_option=True)) + assert captured[0].family == "qwen3_8" + assert captured[0].task == "text_generation" + assert captured[0].precision == ("fp16" if options else "bf16") + assert not output.exists() + + from families.qwen3_8.tests.test_e2e import _build_bundle + + def capture_bundle(request, destination): + captured.append(request) + destination.write_bytes(b"test bundle") + + monkeypatch.setattr(owner, "build_bundle", capture_bundle) + _build_bundle( + {"precision": captured[0].precision, + "max_sequence_length": captured[0].max_sequence_length, + "tensor_parallel_size": captured[0].tensor_parallel_size}, + source, output, + ) + assert captured[-1] == captured[0] + + +def test_declared_help_is_offline_and_dependency_free(): + """Actual child-process help needs neither a checkpoint nor GPU imports.""" + import subprocess + import sys + + code = """ +import sys +from tensorrt_model_connect.family_cli import main +try: + main(["qwen3_8", "build", "--help"]) +except SystemExit as error: + assert error.code == 0 +else: + raise AssertionError("help did not exit") +assert "families.qwen3_8.cli" not in sys.modules +assert "tensorrt" not in sys.modules +assert "huggingface_hub" not in sys.modules +""" + result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True) + assert "trtmc qwen3_8 build" in result.stdout diff --git a/website/docs/features/model-families.md b/website/docs/features/model-families.md index f3292546e0..ae49254cc2 100644 --- a/website/docs/features/model-families.md +++ b/website/docs/features/model-families.md @@ -88,6 +88,27 @@ chat-template handling are documented in the These local qualifications are separate from the registered manifest inventory and do not imply that CI executes the paired cases. +### Qwen3.8 paired ONNX execution + +Use `trtmc qwen3_8 build MODEL -o model.bundle` with the owning +family's options. `trtmc qwen3_8 build --help` works offline without +a checkpoint or GPU imports. This uses the existing +[family CLI protocol](../extend/family-cli.md), not an extension to the shared parser. + +The Qwen3.8 family owns explicit mixed-NVFP4 target plus DSpark block7 execution +through the optional pinned native Edge-LLM SDK. The qualified profile uses +`RadixArk/Qwen3.8-27B-NVFP4` and `RadixArk/Qwen3.8-27B-DSpark`, text-only +FP16 execution with the source mixed NVFP4/FP8 metadata, TP1/batch1 on SM120. +Standalone builds retain the original native path; ordinary experimental Edge +offload and other platform routes are not enabled by this change. + +Provision the [native SDK](../user-guides/configure-runtime.md#optional-native-edge-llm-sdk), +then add `--execution-variant dspark --companion draft=/path/to/draft` to the build +CLI. See the [owning Qwen3.8 recipe](https://github.com/NVIDIA/TensorRT-Model-Connect/blob/main/families/qwen3_8/edge_llm/README.md) +for exact revisions, capacities, sampling controls and quality evidence. +The local paired qualification is not a registered manifest case and does not +imply CI coverage of that pair or statistical sampling equivalence. + ## Runtime and validation The directory name is also the runtime DSO identity: