From 6be666959617240af75cf8f10512b31fc83ae89a Mon Sep 17 00:00:00 2001 From: Joshua Calafato Date: Wed, 16 Sep 2026 23:55:01 +0000 Subject: [PATCH 01/11] feat(qwen3.8): add paired ONNX execution Forward the explicit mixed-NVFP4 target and DSpark block7 draft to the pinned native Edge-LLM ONNX exporter, builder and runtime. Keep admission, prompt mapping, artifact ownership and generation controls inside this family. Preserve native standalone builds and exclude unqualified ordinary Edge paths. Fix the existing E2E helper for companion inputs and the independent mixed-weight oracle without relaxing quality gates. Document the exact SM120 profile and the remaining automated pair-registration gap. Signed-off-by: Joshua Calafato --- families/qwen3_8/EDGE_LLM.md | 60 ++++ families/qwen3_8/dispatch.py | 100 +++++++ families/qwen3_8/edge_llm.py | 210 ++++++++++++++ families/qwen3_8/model.py | 47 +++ families/qwen3_8/runtime/CMakeLists.txt | 28 +- families/qwen3_8/runtime/edge_llm/adapter.cpp | 273 ++++++++++++++++++ families/qwen3_8/runtime/edge_llm/adapter.h | 15 + families/qwen3_8/runtime/edge_llm/contract.h | 61 ++++ .../qwen3_8/runtime/edge_llm/device_link.cu | 5 + families/qwen3_8/runtime/edge_llm/request.h | 60 ++++ families/qwen3_8/runtime/plugin.cpp | 11 + families/qwen3_8/tests/test_e2e.py | 106 ++++++- website/docs/features/model-families.md | 16 + 13 files changed, 983 insertions(+), 9 deletions(-) create mode 100644 families/qwen3_8/EDGE_LLM.md create mode 100644 families/qwen3_8/dispatch.py create mode 100644 families/qwen3_8/edge_llm.py create mode 100644 families/qwen3_8/runtime/edge_llm/adapter.cpp create mode 100644 families/qwen3_8/runtime/edge_llm/adapter.h create mode 100644 families/qwen3_8/runtime/edge_llm/contract.h create mode 100644 families/qwen3_8/runtime/edge_llm/device_link.cu create mode 100644 families/qwen3_8/runtime/edge_llm/request.h diff --git a/families/qwen3_8/EDGE_LLM.md b/families/qwen3_8/EDGE_LLM.md new file mode 100644 index 0000000000..1160f29d8e --- /dev/null +++ b/families/qwen3_8/EDGE_LLM.md @@ -0,0 +1,60 @@ +# Qwen3.8 DSpark Edge-LLM adapter + +The Qwen3.8 family owns its configuration admission, ONNX command mapping, +bundle assets and C++ runtime orchestration. It does not reuse another Qwen +family. Standalone builds retain the original native path; this change admits +Edge only for an explicit mixed-NVFP4 target plus DSpark companion. + +## Build and inference + +Provision the optional native SDK with `TRTMC_EDGELLM_ALL_KERNELS=ON` and +`TRTMC_EDGELLM_ONNX=ON` using the +[pinned package instructions](../../cmake/edgellm/README.md). The source is +GitHub Edge-LLM0.10.1 at `e8b29522938901f6df19ebeedd4b69bc8edbcd97`. +Configure `CMAKE_PREFIX_PATH` for the installed package and compile the runtime +with `TRTMC_ENABLE_EDGELLM=ON`. Cross compilation is unsupported. + +The Python build API accepts `BuildExecutionInputs(variant="dspark", +checkpoints=(NamedCheckpoint("draft", draft_path),))` through its `execution` +argument. The CLI equivalent adds `--execution-variant dspark` and +`--companion draft=/path/to/draft` to an ordinary build invocation. +The family invokes the original Edge Python ONNX exporter and native +`edgellm-onnx-build`; it does not alter source tensors or pad safetensors headers. +The exporter resolves the draft LM head from the target checkpoint. Both +speculative engines, embedding/head sidecars and tokenizer assets are bundled; +checkpoint weights are not duplicated in the bundle. + +The runtime calls the original Edge speculative inference constructor with +proposal block7, verify8, drafting topK1/step1 and DSpark scheduling disabled. +It preserves supported sampling controls. Engine preparation errors warn and +try native once with the same requested execution variant; native currently +rejects the unmapped DSpark variant explicitly. No failure substitutes an +ordinary base-only decoder. Inference errors propagate without fallback. + +## Validated exact profile + +- Target: `RadixArk/Qwen3.8-27B-NVFP4`, revision + `319f741cce68d7914884900c138a1fbb70a42f30`. +- Draft: `RadixArk/Qwen3.8-27B-DSpark`, revision + `b9a5dbdf03bc999c6c73c426b19c2d9041cea393`. +- Native SM120, CUDA13.3, TensorRT11.1.0.106, FP16 execution with source mixed + NVFP4/FP8 metadata retained, TP1/batch1, input/KV capacity1024. +- Actual Model Connect paired build and public CLI inference passed. +- Independent greedy oracle: exact token match, NED0.0 against0.15. + The saved CPU FP32 reference was reused after checkpoint-byte and reference + function verification; it was not regenerated during this run. +- Original Edge `llm_basic` prompt and128-token sampled profile + (temperature1/topK50/topP1): ROUGE-1 **0.4246**, ROUGE-L **0.2458**, above + unchanged **0.25/0.20** gates. Chat enabled and thinking disabled. +- Publication regressions:11 existing family Python tests plus106 existing + builder/architecture tests pass; both existing native C++ tests pass. + +These results qualify only the exact text profile, not standalone NVFP4, +multimodal inputs, other checkpoints/platforms or statistical sampling parity. +The local run reused existing owning E2E helpers with an explicit companion; +this pair is not yet a registered pytest manifest case. + +A first inference attempt exposed incompatible development JSON headers sharing +Edge’s3.12.0 version label. Matching the exact pinned headers fixed the crash +without changing Edge or the engines. The generic SDK now checks header content +rather than relying only on the version label. diff --git a/families/qwen3_8/dispatch.py b/families/qwen3_8/dispatch.py new file mode 100644 index 0000000000..d833bf71ac --- /dev/null +++ b/families/qwen3_8/dispatch.py @@ -0,0 +1,100 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Family-owned complete-network route map; native is the default.""" + +from __future__ import annotations + +import json +import logging +import os +from pathlib import Path +import tempfile +import traceback + +from . import edge_llm + +_LOG = logging.getLogger(__name__) + +# Exact qualified native DSpark route; no unvalidated platform or ordinary offload. +EDGE_DISPATCH = { + ("linux", "x86_64", 120, "fp16"): edge_llm.prepare_dspark, +} + + +def candidate(request, raw: dict) -> bool: + """Return whether this family's model/request contract can delegate to Edge.""" + config = raw.get("text_config", raw) + source_quantization = edge_llm.checkpoint_quantization(Path(request.model_dir), raw) + return ( + isinstance(config, dict) + and raw.get("model_type") == "qwen3_5" + and ("output_gate_type" in config and "mlp_only_layers" not in config) + and config.get("linear_key_head_dim") == config.get("linear_value_head_dim") == 128 + and not config.get("num_experts") + and source_quantization == "nvfp4" + and request.backend == "trt" and request.task == "text_generation" + and request.precision.lower() == "fp16" + and request.quantization in {None, source_quantization} + and request.max_batch_size == request.tensor_parallel_size == request.context_parallel_size == 1 + and not request.dynamic_kv_cache and not request.fp32_layers and request.graph_transform is None + and all(value is None for value in (request.image_height, request.image_width, request.video_num_frames)) + ) + + +def build(request, writer, native, *, draft_dir: Path) -> None: + """Dispatch locally or warn and retry native once with the original request. + + Args: + request: Unmodified Model Connect build request. + writer: Unpublished bundle writer. + native: This family's original native builder callback. + + Raises: + Exception: Common input/publication error, or native build error with + Edge cause after a failed preparation. Cancellation never retries. + """ + raw = json.loads((Path(request.model_dir) / "config.json").read_text(encoding="utf-8")) + if not isinstance(raw, dict): + raise ValueError("checkpoint config.json must contain an object") + config = raw.get("text_config", raw) + if not isinstance(config, dict): + raise ValueError("checkpoint text_config must contain an object") + if not candidate(request, raw): + native(request, writer) + return + capacity = config.get("max_position_embeddings") + if type(capacity) is not int or capacity <= 0: + raise ValueError("checkpoint max_position_embeddings must be a positive integer") + if request.max_sequence_length and request.max_sequence_length > capacity: + raise ValueError("max_sequence_length exceeds checkpoint context capacity") + failure = None + descriptor, name = tempfile.mkstemp(prefix=f".{request.output_path.name}.edge-", suffix=".log", + dir=request.output_path.parent) + os.close(descriptor) + log_path = Path(name) + with tempfile.TemporaryDirectory(prefix="trtmc-qwen3_8-edge-") as directory: + try: + target = edge_llm.local_target() + key = (target["os"], target["arch"], target["sm"], request.precision.lower()) + adapter = EDGE_DISPATCH.get(key) + if adapter is not None: + files, marker = adapter(request, raw, target, Path(directory), log_path, draft_dir) + except Exception as error: + failure = error + with log_path.open("a", encoding="utf-8") as log: + traceback.print_exception(error, file=log) + _LOG.warning("qwen3_8 Edge build failed: %s. Diagnostics: %s. " + "Retrying native once with the unchanged request.", error, log_path, exc_info=True) + else: + # Edge preparation did not touch writer; publication cannot fallback. + if adapter is not None: + edge_llm.publish(request, writer, files, marker) + return + log_path.unlink() # A platform non-match is not an Edge failure. + try: + native(request, writer) + except Exception as error: + if failure is not None: + raise error from failure + raise diff --git a/families/qwen3_8/edge_llm.py b/families/qwen3_8/edge_llm.py new file mode 100644 index 0000000000..c086d7fc09 --- /dev/null +++ b/families/qwen3_8/edge_llm.py @@ -0,0 +1,210 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Family-owned paired adapter to the pinned Edge ONNX builder API.""" + +from __future__ import annotations + +import json +from pathlib import Path +import shutil +import subprocess + +from tensorrt_model_connect.build import cmake_prefixes, detect_local_platform, subprocess_environment + +EDGE_REVISION = "e8b29522938901f6df19ebeedd4b69bc8edbcd97" + + +def local_target() -> dict: + """Return the executing worker identity supplied by generic build mechanics.""" + return detect_local_platform() + + +def installed_package(target: dict) -> dict: + """Resolve CMake installation via standard prefixes; never install anything. + + Args: + target: Executing device and SDK identity. + + Returns: + Validated package metadata with absolute Python and plugin paths. + + Raises: + FileNotFoundError: No CMake installation or required artifact exists. + ValueError: Pin, architecture, SDK or contained-path contract differs. + """ + for prefix in cmake_prefixes(): + manifest = prefix / "share/trtmc/edge-llm.json" + if not manifest.is_file(): + continue + package = json.loads(manifest.read_text(encoding="utf-8")) + if package.get("schema_version") != 1 or package.get("revision") != EDGE_REVISION: + raise ValueError(f"Edge package has an unsupported revision/schema: {manifest}") + if package.get("version") != "0.10.1" or package.get("arch") != target["arch"]: + raise ValueError("Edge package version/architecture differs from executing worker") + if target["sm"] not in package.get("architectures", []): + raise ValueError("Edge package was not built for this local GPU") + cuda_version = ".".join(str(package.get("cuda_version", "")).split(".")[:2]) + if cuda_version != target["cuda_version"] or package.get("tensorrt_version") != target["tensorrt_version"]: + raise ValueError("Edge package CUDA/TensorRT differs from executing worker") + for name in ("python", "plugin") + (("onnx_builder",) if package.get("onnx") else ()): + relative = Path(package[name]) + path = (prefix / relative).resolve() + if relative.is_absolute() or not path.is_relative_to(prefix.resolve()): + raise ValueError(f"Edge package {name} must be contained in its installation") + if not path.is_file(): + raise FileNotFoundError(f"Edge package {name} is missing: {path}") + package[name] = str(path) + return package + raise FileNotFoundError("Edge-LLM is not installed; enable the optional Edge-LLM CMake dependency " + "and set CMAKE_PREFIX_PATH to its install prefix") + + + +def checkpoint_quantization(model_dir: Path, raw: dict) -> str | None: + """Admit plain weights or the documented mixed ModelOpt checkpoint. + + Edge owns per-layer decoding and FP8 KV interpretation. Require matching + embedded/sidecar metadata rather than inventing or converting a format. + """ + if any((model_dir / name).exists() for name in ("quantize_config.json", "quant_config.json")): + return None + embedded = raw.get("quantization_config") + nested = raw.get("text_config", {}).get("quantization_config") + sidecar = model_dir / "hf_quant_config.json" + if not embedded and not nested and not sidecar.exists(): + return "none" + if not isinstance(embedded, dict) or (nested and nested != embedded): + return None + if embedded.get("quant_method") != "modelopt" or embedded.get("quant_algo") != "MIXED_PRECISION": + return None + layers = embedded.get("quantized_layers") + if not isinstance(layers, dict) or not layers or any(not isinstance(v, dict) for v in layers.values()): + return None + if {v.get("quant_algo") for v in layers.values()} != {"FP8", "NVFP4"}: + return None + if not sidecar.is_file(): + return None + value = json.loads(sidecar.read_text(encoding="utf-8")) + quant = value.get("quantization") if isinstance(value, dict) else None + if not isinstance(quant, dict) or quant.get("quant_algo") != "MIXED_PRECISION": + return None + if quant.get("quantized_layers") != layers: + return None + if quant.get("kv_cache_quant_algo") != "FP8" or not embedded.get("kv_cache_scheme"): + return None + return "nvfp4" + +_PROMPT_PROGRAM = r"""import json, sys +from pathlib import Path +from transformers import AutoTokenizer +checkpoint = Path(sys.argv[sys.argv.index("--model-dir") + 1]) +engine = Path(sys.argv[sys.argv.index("--engine-dir") + 1]) +tokenizer = AutoTokenizer.from_pretrained(checkpoint, local_files_only=True, trust_remote_code=False) +slot = "Qwen38SingleUserContentSlot" +formats = {} +for thinking in (False, True): + options = dict(tokenize=False, add_generation_prompt=True, enable_thinking=thinking) + rendered = tokenizer.apply_chat_template([dict(role="user", content=slot)], **options) + if rendered.count(slot) != 1: + raise ValueError("Qwen3.8 source does not preserve a single user prompt") + prefix, suffix = rendered.split(slot) + whitespace = "".join(chr(cp) for cp in range(sys.maxunicode + 1) if chr(cp).isspace()) + for probe in ("", " leading and trailing ", "first\nsecond", "世界", whitespace + "text" + whitespace): + actual = tokenizer.apply_chat_template([dict(role="user", content=probe)], **options) + if actual != prefix + probe.strip() + suffix: + raise ValueError("Qwen3.8 source user content is not a prefix/suffix mapping") + formats[str(thinking).lower()] = dict(prefix=prefix, suffix=suffix) +(engine / "trtmc_single_user_prompts.json").write_text(json.dumps(formats, ensure_ascii=False)) +""" + + + +def prepare_dspark(request, raw: dict, target: dict, staging: Path, log_path: Path, + draft_dir: Path) -> tuple[dict, dict]: + """Map the paired request to the original exporter and native ONNX builder. + + Edge supplies the draft LM head from its target checkpoint. No safetensors + padding or weight conversion is needed, and baked weights are not bundled + twice. Both plans and their complete runtime assets must exist to publish. + """ + package = installed_package(target) + if package.get("onnx") is not True: + raise ValueError("Qwen3.8 DSpark requires an ONNX-enabled Edge SDK") + source, draft_dir = Path(request.model_dir).resolve(), draft_dir.resolve() + if checkpoint_quantization(source, raw) != "nvfp4": + raise ValueError("Qwen3.8 DSpark requires the mixed NVFP4 target") + if not list(source.glob("*.safetensors")) or not list(draft_dir.glob("*.safetensors")): + raise ValueError("Qwen3.8 DSpark requires both local safetensors checkpoints") + config = raw.get("text_config", raw) + limit = request.max_sequence_length or min(int(config["max_position_embeddings"]), 256) + if not 8 < limit <= 1024: + raise ValueError("Qwen3.8 DSpark requires capacity above verification size8 and at most1024") + checkpoint = staging / "edge_llm/checkpoint" + checkpoint.mkdir(parents=True) + for name in ("config.json", "tokenizer.json", "tokenizer_config.json", "generation_config.json", + "chat_template.jinja"): + if (source / name).is_file(): + shutil.copy2(source / name, checkpoint / name) + (checkpoint / "draft").mkdir() + shutil.copy2(draft_dir / "config.json", checkpoint / "draft/config.json") + engine, onnx = staging / "edge_llm/engine", staging / "onnx" + env = subprocess_environment( + {"EDGELLM_PLUGIN_PATH": package["plugin"]}, + prepend_paths={"LD_LIBRARY_PATH": str(Path(package["plugin"]).parent)}, + ) + for role, subdirectory, flag in (("draft", "dspark_draft", "--specDraft"), + ("base", "llm", "--specBase")): + commands = [ + [package["python"], "-I", "-m", "tensorrt_edgellm.scripts.export", + str(source), str(onnx), f"--dspark-{role}", "--dspark-draft-dir", str(draft_dir), + "--skip-visual", "--skip-audio"], + [package["onnx_builder"], "--onnxDir", str(onnx / subdirectory), + "--engineDir", str(engine), flag, "--maxInputLen", str(min(limit, 1024)), + "--maxKVCacheCapacity", str(limit), "--maxBatchSize", "1", + "--maxVerifyTreeSize", "8", "--maxDraftTreeSize", "7"], + ] + with log_path.open("a", encoding="utf-8") as log: + for command in commands: + log.write(json.dumps(command) + "\n") + log.flush() + subprocess.run(command, check=True, stdout=log, stderr=subprocess.STDOUT, + cwd=staging, env=env) + shutil.rmtree(onnx) # Only this preparation's successfully consumed intermediates. + with log_path.open("a", encoding="utf-8") as log: + subprocess.run([package["python"], "-I", "-c", _PROMPT_PROGRAM, + "--model-dir", str(checkpoint), "--engine-dir", str(engine)], + check=True, stdout=log, stderr=subprocess.STDOUT, cwd=staging, env=env) + required = ("spec_base.engine", "spec_draft.engine", "base_config.json", "draft_config.json", + "embedding.safetensors", "dspark_heads.safetensors", "dspark_heads_info.json", + "tokenizer.json", "tokenizer_config.json", + "processed_chat_template.json", "trtmc_single_user_prompts.json") + for name in required: + if not (engine / name).is_file() or (engine / name).stat().st_size == 0: + raise ValueError(f"Edge ONNX builder did not produce required artifact: {name}") + for role in ("base", "draft"): + built = json.loads((engine / f"{role}_config.json").read_text(encoding="utf-8")) + if built.get("spec_decode_type") != "dspark" or built.get("dspark_config", {}).get("block_size") != 7: + raise ValueError("Edge ONNX builder returned a different DSpark contract") + files = {} + for directory in (engine, checkpoint): + for path in sorted(directory.rglob("*")): + if path.is_symlink(): + raise ValueError(f"Edge output must not contain symlinks: {path}") + if path.is_file(): + files[path.relative_to(staging).as_posix()] = path + return files, { + "version": 1, "edge_revision": EDGE_REVISION, "target": target, "precision": "fp16", + "max_sequence_length": limit, "max_input_length": min(limit, 1024), "max_batch_size": 1, + "checkpoint_quantization": "nvfp4", "artifacts": list(files), + "execution_variant": "dspark", "builder_flow": "onnx", "dspark_block_size": 7, + } + + +def publish(request, writer, files: dict, marker: dict) -> None: + """Stream complete Edge sections; publication errors must not retry native.""" + writer.set_header(family=request.family, task=request.task, backend=request.backend) + for name, path in files.items(): + with path.open("rb") as source, writer.open_section(name) as destination: + shutil.copyfileobj(source, destination, length=1024 * 1024) + writer.add_json("edge_llm.json", marker) diff --git a/families/qwen3_8/model.py b/families/qwen3_8/model.py index 15688ae363..12f6fe7535 100644 --- a/families/qwen3_8/model.py +++ b/families/qwen3_8/model.py @@ -171,3 +171,50 @@ def build(request, writer) -> None: path = model_dir / filename if path.is_file(): writer.add_bytes(filename, path.read_bytes()) + + +def build_with_inputs(request, writer, execution) -> None: + """Build the retained mixed-NVFP4 Qwen3.8 / DSpark block7 pair.""" + from . import dispatch, edge_llm + + if execution.variant != "dspark" or tuple(x.role for x in execution.checkpoints) != ("draft",): + raise ValueError("Qwen3.8 paired execution requires variant=dspark and one draft checkpoint") + draft_dir = execution.checkpoints[0].model_dir + raw = json.loads((request.model_dir / "config.json").read_text()) + draft = json.loads((draft_dir / "config.json").read_text()) + if not dispatch.candidate(request, raw) or edge_llm.checkpoint_quantization(request.model_dir, raw) != "nvfp4": + raise ValueError("The retained Qwen3.8 DSpark pair requires a matching mixed-NVFP4 base") + base = raw.get("text_config", raw) + if not isinstance(draft, dict) or draft.get("architectures") != ["DSparkDraftModel"]: + raise ValueError("Expected a DSparkDraftModel companion") + for name in ("hidden_size", "vocab_size"): + if type(draft.get(name)) is not int or draft[name] != base.get(name): + raise ValueError(f"Qwen3.8 DSpark base and draft disagree on {name}") + if draft.get("num_target_layers") != base.get("num_hidden_layers"): + raise ValueError("Qwen3.8 DSpark target layer count differs from base") + config = draft.get("dspark_config") + if not isinstance(config, dict) or config.get("block_size", draft.get("block_size")) != 7: + raise ValueError("Qwen3.8 DSpark maps the upstream block7 / verify8 profile") + layers = config.get("target_layer_ids", draft.get("target_layer_ids")) + if (not isinstance(layers, list) or not layers + or any(type(i) is not int or not 0 <= i < base["num_hidden_layers"] for i in layers) + or len(set(layers)) != len(layers)): + raise ValueError("Invalid Qwen3.8 DSpark target layer IDs") + mask = config.get("mask_token_id", draft.get("mask_token_id")) + if type(mask) is not int or not 0 <= mask < base["vocab_size"]: + raise ValueError("Invalid Qwen3.8 DSpark mask token") + limit = request.max_sequence_length or min(base["max_position_embeddings"], 256) + capacity = draft.get("max_position_embeddings") + if type(capacity) is not int or not 8 < limit <= capacity: + raise ValueError("Requested context exceeds DSpark draft capacity or block minimum") + if draft.get("quantization_config") or any( + (draft_dir / name).exists() + for name in ("hf_quant_config.json", "quantize_config.json", "quant_config.json") + ): + raise ValueError("This Qwen3.8 DSpark profile requires unquantized draft weights") + + def native_pair(original_request, original_writer): + # A failure must never replace the requested pair with base-only decoding. + raise NotImplementedError("Native Qwen3.8 does not implement the requested DSpark variant") + + dispatch.build(request, writer, native_pair, draft_dir=draft_dir) diff --git a/families/qwen3_8/runtime/CMakeLists.txt b/families/qwen3_8/runtime/CMakeLists.txt index af423a3801..b74b007112 100644 --- a/families/qwen3_8/runtime/CMakeLists.txt +++ b/families/qwen3_8/runtime/CMakeLists.txt @@ -21,7 +21,10 @@ target_link_libraries(trtmc_model_qwen3_8 PRIVATE nlohmann_json::nlohmann_json ${TRTMC_CUDART_LIBRARY} ) -target_compile_options(trtmc_model_qwen3_8 PRIVATE -Wall -Wextra -Wpedantic) +target_compile_options(trtmc_model_qwen3_8 PRIVATE + "$<$:-Wall;-Wextra;-Wpedantic>" + "$<$:-Xcompiler=-Wall,-Wextra>" +) set_target_properties(trtmc_model_qwen3_8 PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" BUILD_RPATH "\$ORIGIN" @@ -71,3 +74,26 @@ if(TRTMC_BUILD_TESTS) SKIP_RETURN_CODE 77 ) endif() + +# Complete-network offload is family-owned and absent from native-only builds. +if(TARGET EdgeLLM::Core) + target_sources(trtmc_model_qwen3_8 PRIVATE + edge_llm/adapter.cpp + edge_llm/device_link.cu + ) + target_compile_definitions(trtmc_model_qwen3_8 PRIVATE TRTMC_HAS_EDGE_LLM=1) + target_link_libraries(trtmc_model_qwen3_8 PRIVATE EdgeLLM::Core) + set_target_properties(trtmc_model_qwen3_8 PROPERTIES + CUDA_ARCHITECTURES "${EdgeLLM_CUDA_ARCHITECTURE}" + CUDA_SEPARABLE_COMPILATION ON + CUDA_RESOLVE_DEVICE_SYMBOLS ON + ) +endif() + +if(TARGET EdgeLLM::Core) + add_custom_command(TARGET trtmc_model_qwen3_8 POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ $ + VERBATIM + ) +endif() diff --git a/families/qwen3_8/runtime/edge_llm/adapter.cpp b/families/qwen3_8/runtime/edge_llm/adapter.cpp new file mode 100644 index 0000000000..6690cd07e2 --- /dev/null +++ b/families/qwen3_8/runtime/edge_llm/adapter.cpp @@ -0,0 +1,273 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#include "families/qwen3_8/runtime/edge_llm/adapter.h" + +#include "families/qwen3_8/runtime/edge_llm/request.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace trtmc::qwen3_8::edge_llm { +namespace { +namespace fs = std::filesystem; + +/// Turn CUDA failures into caller-visible load or inference errors. +void check_cuda(cudaError_t result) { + if (result != cudaSuccess) + throw std::runtime_error(std::string("Qwen3.8 Edge CUDA error: ") + + cudaGetErrorString(result)); +} + +/// Reject an engine built for a different local GPU or CUDA/TensorRT runtime. +void validate_target(const nlohmann::json& target) { + utsname host{}; + if (uname(&host) != 0) + throw std::runtime_error("Cannot identify Qwen3.8 Edge runtime host"); + std::ifstream release("/etc/os-release"); + std::string line, os_version; + while (std::getline(release, line)) { + if (line.rfind("VERSION_ID=", 0) == 0) { + os_version = line.substr(11); + if (os_version.size() >= 2 && os_version.front() == char(34) && + os_version.back() == char(34)) + os_version = os_version.substr(1, os_version.size() - 2); + } + } + int device = 0, cuda_version = 0; + check_cuda(cudaGetDevice(&device)); + check_cuda(cudaRuntimeGetVersion(&cuda_version)); + cudaDeviceProp gpu{}; + check_cuda(cudaGetDeviceProperties(&gpu, device)); + const int trt_version = getInferLibVersion(); + const std::string trt = + std::to_string(trt_version / 10000) + "." + std::to_string((trt_version % 10000) / 100) + + "." + std::to_string(trt_version % 100) + "." + std::to_string(getInferLibBuildVersion()); + const std::string cuda = + std::to_string(cuda_version / 1000) + "." + std::to_string((cuda_version % 1000) / 10); + if (target.at("os") != "linux" || target.at("os_version") != os_version || + target.at("arch") != host.machine || target.at("sm") != gpu.major * 10 + gpu.minor || + target.at("cuda_version") != cuda || target.at("tensorrt_version") != trt) + throw std::runtime_error( + "Qwen3.8 Edge bundle requires its build GPU and CUDA/TensorRT stack"); +} + +/// Own extracted engine/checkpoint files until after the Edge runtime is destroyed. +class Artifacts { + public: + explicit Artifacts(const BundleReader& bundle, const nlohmann::json& marker) { + std::set names; + for (const auto& entry : marker.at("artifacts")) { + const auto name = entry.get(); + if (!safe_artifact_path(name) || !names.insert(name).second || + !bundle.find_section(name)) + throw std::runtime_error("Invalid Qwen3.8 Edge artifact: " + name); + } + std::vector required_files{ + "edge_llm/engine/tokenizer.json", "edge_llm/engine/tokenizer_config.json", + "edge_llm/engine/processed_chat_template.json", + "edge_llm/engine/trtmc_single_user_prompts.json", "edge_llm/checkpoint/config.json"}; + + for (const auto* name : + {"spec_base.engine", "spec_draft.engine", "base_config.json", "draft_config.json", + "embedding.safetensors", "dspark_heads.safetensors", "dspark_heads_info.json"}) + required_files.push_back(std::string("edge_llm/engine/") + name); + required_files.push_back("edge_llm/checkpoint/draft/config.json"); + for (const auto& required : required_files) + if (!names.count(required) || bundle.find_section(required)->length == 0) + throw std::runtime_error("Required Qwen3.8 Edge artifact missing: " + required); + std::string pattern = (fs::temp_directory_path() / "trtmc-qwen3_8-edge-XXXXXX").string(); + if (!mkdtemp(pattern.data())) + throw std::runtime_error("Cannot create Qwen3.8 Edge artifact directory"); + root_ = pattern; + try { + for (const auto& name : names) { + const auto destination = root_ / name; + fs::create_directories(destination.parent_path()); + std::ofstream output(destination, std::ios::binary); + bundle.copy_section(name, output); + output.close(); + if (!output) + throw std::runtime_error("Cannot extract Qwen3.8 Edge artifact: " + name); + } + } catch (...) { + cleanup(); + throw; + } + } + ~Artifacts() { cleanup(); } + Artifacts(const Artifacts&) = delete; + Artifacts& operator=(const Artifacts&) = delete; + std::string engine() const { return (root_ / "edge_llm/engine").string(); } + std::string checkpoint() const { return (root_ / "edge_llm/checkpoint").string(); } + + private: + void cleanup() noexcept { + std::error_code ignored; + fs::remove_all(root_, ignored); + } + fs::path root_; +}; + +/// Close the plugin handle after runtime destruction; registrations remain mapped. +struct CloseLibrary { + void operator()(void* handle) const noexcept { + if (handle) + dlclose(handle); + } +}; + +/// Initialize the CMake-installed adjacent plugin without process-global environment mutation. +std::unique_ptr load_plugin() { + Dl_info location{}; + if (!dladdr(reinterpret_cast(&create), &location) || !location.dli_fname) + throw std::runtime_error("Cannot locate Qwen3.8 family library"); + const auto path = + fs::absolute(location.dli_fname).parent_path() / "libNvInfer_edgellm_plugin.so"; + std::unique_ptr plugin( + dlopen(path.c_str(), RTLD_NOW | RTLD_GLOBAL | RTLD_NODELETE)); + if (!plugin) + throw std::runtime_error("Cannot load CMake-installed Edge plugin: " + + std::string(dlerror())); + using Initialize = bool (*)(void*, const char*); + auto initialize = reinterpret_cast(dlsym(plugin.get(), "initEdgellmPlugins")); + if (!initialize || !initialize(static_cast(&trt_edgellm::gLogger), "")) + throw std::runtime_error("Cannot initialize Qwen3.8 Edge plugin"); + return plugin; +} + +/// Stream ownership is independent of construction success and outlives the Edge instance. +class Stream { + public: + Stream() { check_cuda(cudaStreamCreateWithFlags(&value_, cudaStreamNonBlocking)); } + ~Stream() { cudaStreamDestroy(value_); } + Stream(const Stream&) = delete; + Stream& operator=(const Stream&) = delete; + cudaStream_t get() const { return value_; } + + private: + cudaStream_t value_{nullptr}; +}; + +/// Delegate the complete DSpark block7 algorithm to the pinned runtime. +std::unique_ptr +make_runtime(const Artifacts& artifacts, cudaStream_t stream) { + trt_edgellm::rt::SpecDecodeDraftingConfig drafting{}; + drafting.draftingTopK = 1; + drafting.draftingStep = 1; + drafting.verifySize = 8; + drafting.dflashBlockSize = 0; + drafting.dsparkSchedulerMode = trt_edgellm::rt::DSparkSchedulerMode::kOff; + drafting.dsparkConfidenceThreshold = 0.0F; + drafting.dsparkMinProposalLen = 1; + drafting.dsparkMaxProposalLen = 0; + return std::make_unique( + artifacts.engine(), "", std::unordered_map{}, drafting, + stream, trt_edgellm::rt::ContextCacheConfig{}, artifacts.checkpoint(), + (fs::path(artifacts.checkpoint()) / "draft").string()); +} + +/// Thin persistent Edge API adapter; serialization prevents concurrent use of Edge request state. +class EdgeTask final : public ITextGeneration { + public: + EdgeTask(const BundleReader& bundle, const nlohmann::json& marker) + : artifacts_(bundle, marker), plugin_(load_plugin()), + runtime_(make_runtime(artifacts_, stream_.get())), + capacity_(marker.at("max_sequence_length").get()), + input_limit_(marker.at("max_input_length").get()) { + std::ifstream input(fs::path(artifacts_.engine()) / "trtmc_single_user_prompts.json"); + source_prompts_ = nlohmann::json::parse(input); + for (const auto* mode : {"false", "true"}) + for (const auto* part : {"prefix", "suffix"}) + if (!source_prompts_.at(mode).at(part).is_string()) + throw std::runtime_error("Invalid Qwen3.8 source prompt mapping"); + } + + std::int32_t default_max_new_tokens() const override { return std::min(128, capacity_ - 1); } + + /// Drain work from failed requests before destroying the runtime and its weight buffers. + ~EdgeTask() override { cudaStreamSynchronize(stream_.get()); } + + /// Invoke Edge once; failures propagate without attempting native inference. + TextResult generate(const std::string& prompt, const TextGenerationConfig& config) override { + auto effective = config; + std::string source_prompt = prompt; + if (config.use_chat_template) { + const auto& format = source_prompts_.at(config.enable_thinking ? "true" : "false"); + source_prompt = format.at("prefix").get() + source_user_content(prompt) + + format.at("suffix").get(); + effective.use_chat_template = false; + } + auto request = make_request(source_prompt, effective, default_max_new_tokens()); + std::lock_guard lock(mutex_); + const auto counts = runtime_->countPromptTokens(request); + if (counts.size() != 1) + throw std::runtime_error("Qwen3.8 Edge returned invalid prompt counts"); + validate_capacity(counts.front(), input_limit_, capacity_, request.maxGenerateLength); + trt_edgellm::rt::LLMGenerationResponse response{}; + if (!runtime_->handleRequest(request, response, stream_.get()) || + response.outputIds.size() != 1 || response.outputTexts.size() != 1 || + response.outputIds.front().empty() || + response.outputIds.front().size() > static_cast(request.maxGenerateLength)) + throw std::runtime_error("Qwen3.8 Edge generation failed"); + if (response.finishReasons.size() != 1 || + (response.finishReasons.front() != trt_edgellm::rt::FinishReason::kEndId && + response.finishReasons.front() != trt_edgellm::rt::FinishReason::kLength)) + throw std::runtime_error("Qwen3.8 Edge generation did not complete successfully"); + // This API does not expose per-request stage times; zero means unavailable. + return {std::move(response.outputTexts.front()), std::move(response.outputIds.front())}; + } + + private: + // Reverse destruction order keeps weights, plugin and stream alive throughout Edge teardown. + Artifacts artifacts_; + std::unique_ptr plugin_; + Stream stream_; + std::unique_ptr runtime_; + int capacity_; + int input_limit_; + nlohmann::json source_prompts_; + std::mutex mutex_; +}; +} // namespace + +ITask* create(const BundleReader& bundle) { + const auto bytes = bundle.read_section("edge_llm.json"); + const auto marker = nlohmann::json::parse(bytes.begin(), bytes.end()); + if (marker.at("version") != 1 || marker.at("edge_revision") != kRevision || + marker.at("max_sequence_length").get() <= 1 || + marker.at("max_input_length").get() <= 0 || + marker.at("max_input_length").get() > marker.at("max_sequence_length").get() || + marker.at("max_batch_size") != 1 || marker.at("precision") != "fp16" || + !marker.at("artifacts").is_array()) + throw std::runtime_error("Invalid Qwen3.8 Edge bundle contract"); + const auto variant = marker.value("execution_variant", ""); + if (!valid_execution_variant(variant, marker.value("dspark_block_size", 0))) + throw std::runtime_error("Unsupported Qwen3.8 Edge execution variant"); + if (marker.value("builder_flow", "") != "onnx" || + marker.value("checkpoint_quantization", "") != "nvfp4" || + marker.at("max_sequence_length").get() <= 8 || + marker.at("max_sequence_length").get() > 1024 || + marker.at("target").at("arch") != "x86_64" || marker.at("target").at("sm") != 120) + throw std::runtime_error("Qwen3.8 DSpark requires the ONNX builder flow"); + validate_target(marker.at("target")); + return new EdgeTask(bundle, marker); +} + +} // namespace trtmc::qwen3_8::edge_llm diff --git a/families/qwen3_8/runtime/edge_llm/adapter.h b/families/qwen3_8/runtime/edge_llm/adapter.h new file mode 100644 index 0000000000..86cf5a89ce --- /dev/null +++ b/families/qwen3_8/runtime/edge_llm/adapter.h @@ -0,0 +1,15 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include "trtmc/bundle.h" +#include "trtmc/task.h" + +namespace trtmc::qwen3_8::edge_llm { + +/// Create a persistent Edge task from a self-contained bundle; throws on load failure. +ITask* create(const BundleReader& bundle); + +} // namespace trtmc::qwen3_8::edge_llm diff --git a/families/qwen3_8/runtime/edge_llm/contract.h b/families/qwen3_8/runtime/edge_llm/contract.h new file mode 100644 index 0000000000..aedecb8588 --- /dev/null +++ b/families/qwen3_8/runtime/edge_llm/contract.h @@ -0,0 +1,61 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include "trtmc/task.h" + +#include +#include +#include +#include + +namespace trtmc::qwen3_8::edge_llm { + +inline constexpr const char* kRevision = "e8b29522938901f6df19ebeedd4b69bc8edbcd97"; + +/// DSpark proposal length excludes the one additional verification token. +inline bool valid_execution_variant(const std::string& variant, int block_size) { + return variant == "dspark" && block_size == 7; +} + +/// Return whether an artifact is a normalized file below one of the two Edge roots. +inline bool safe_artifact_path(const std::string& name) { + if (name.find('\\') != std::string::npos || name.find('\0') != std::string::npos) + return false; + const std::filesystem::path path(name); + if (path.is_absolute() || path.filename().empty()) + return false; + for (const auto& part : path) + if (part == "." || part == "..") + return false; + return path.generic_string() == name && + (name.rfind("edge_llm/engine/", 0) == 0 || name.rfind("edge_llm/checkpoint/", 0) == 0); +} + +/// Reject invalid sampling settings and controls with no equivalent Edge request API. +inline void validate_generation(const TextGenerationConfig& c) { + if (!std::isfinite(c.temperature) || c.temperature < 0 || !std::isfinite(c.top_p) || + c.top_p <= 0 || c.top_p > 1 || c.top_k < 0) + throw std::invalid_argument("Invalid Qwen3.8 Edge sampling parameters"); + if (c.min_p != 0 || c.seed != -1 || c.eos_token_id != -1 || c.repetition_penalty != 1 || + !c.lora_adapter_id.empty() || c.stop_on_boxed_answer || + (c.text_generation_mode != "auto" && c.text_generation_mode != "autoregressive") || + c.source_language_token_id != -1 || c.forced_bos_token_id != -1 || c.guidance_scale != -1 || + c.cfg_scale != -1 || c.num_steps != -1 || c.sde_gamma != -1 || !c.initial_latents.empty() || + !c.condition_latents.empty() || !c.condition_mask.empty() || !c.sampling_steps.empty() || + !c.sde_noises.empty() || c.block_length != 0 || c.confidence_threshold != -1) + throw std::invalid_argument( + "Requested generation controls are unsupported by Qwen3.8 Edge"); +} + +/// Enforce prompt and total capacity without allowing Edge to silently clip generation. +inline void validate_capacity(int prompt_tokens, int input_limit, int capacity, + std::int64_t generated_tokens) { + if (prompt_tokens <= 0 || prompt_tokens > input_limit || generated_tokens <= 0 || + generated_tokens > static_cast(capacity) - prompt_tokens) + throw std::invalid_argument("Qwen3.8 Edge prompt and generation exceed bundle capacity"); +} + +} // namespace trtmc::qwen3_8::edge_llm diff --git a/families/qwen3_8/runtime/edge_llm/device_link.cu b/families/qwen3_8/runtime/edge_llm/device_link.cu new file mode 100644 index 0000000000..1ea4768d60 --- /dev/null +++ b/families/qwen3_8/runtime/edge_llm/device_link.cu @@ -0,0 +1,5 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +// Enable the final CUDA device-link step for Edge's static runtime dependencies. diff --git a/families/qwen3_8/runtime/edge_llm/request.h b/families/qwen3_8/runtime/edge_llm/request.h new file mode 100644 index 0000000000..5a8e8a2e75 --- /dev/null +++ b/families/qwen3_8/runtime/edge_llm/request.h @@ -0,0 +1,60 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include "families/qwen3_8/runtime/edge_llm/contract.h" + +#include + +namespace trtmc::qwen3_8::edge_llm { + +/// Match the source Jinja trim filter without locale-dependent ASCII-only trimming. +inline std::string source_user_content(std::string text) { + // Python str.strip whitespace, including C0 separators and Unicode spaces. + constexpr const char* whitespace[] = { + "\t", "\n", "\v", "\f", "\r", + "\x1c", "\x1d", "\x1e", "\x1f", " ", + "\xc2\x85", "\xc2\xa0", "\xe1\x9a\x80", "\xe2\x80\x80", "\xe2\x80\x81", + "\xe2\x80\x82", "\xe2\x80\x83", "\xe2\x80\x84", "\xe2\x80\x85", "\xe2\x80\x86", + "\xe2\x80\x87", "\xe2\x80\x88", "\xe2\x80\x89", "\xe2\x80\x8a", "\xe2\x80\xa8", + "\xe2\x80\xa9", "\xe2\x80\xaf", "\xe2\x81\x9f", "\xe3\x80\x80"}; + bool changed = true; + while (changed && !text.empty()) { + changed = false; + for (const std::string space : whitespace) { + if (text.compare(0, space.size(), space) == 0) { + text.erase(0, space.size()); + changed = true; + } + if (text.size() >= space.size() && + text.compare(text.size() - space.size(), space.size(), space) == 0) { + text.resize(text.size() - space.size()); + changed = true; + } + } + } + if (text.rfind("", 0) == 0 && text.size() >= 16 && + text.compare(text.size() - 16, 16, "") == 0) + throw std::invalid_argument("Qwen3.8 single-user prompt contains no user query"); + return text; +} + +/// Map Model Connect text arguments to the pinned Edge API; rejects unmapped controls. +inline trt_edgellm::rt::LLMGenerationRequest +make_request(const std::string& prompt, const TextGenerationConfig& config, int default_length) { + validate_generation(config); + trt_edgellm::rt::LLMGenerationRequest request{}; + request.requests.resize(1); + request.requests.front().messages.push_back({"user", {{"text", prompt}}}); + request.applyChatTemplate = config.use_chat_template; + request.enableThinking = config.enable_thinking; + request.temperature = config.temperature; + request.topK = config.top_k; + request.topP = config.top_p; + request.maxGenerateLength = config.max_new_tokens > 0 ? config.max_new_tokens : default_length; + return request; +} + +} // namespace trtmc::qwen3_8::edge_llm diff --git a/families/qwen3_8/runtime/plugin.cpp b/families/qwen3_8/runtime/plugin.cpp index 49a8b4c57f..114a0f4533 100644 --- a/families/qwen3_8/runtime/plugin.cpp +++ b/families/qwen3_8/runtime/plugin.cpp @@ -9,6 +9,9 @@ #include "families/qwen3_8/runtime/plugin_helpers.h" #include "families/qwen3_8/runtime/recurrent_state.h" #include "trtmc/runtime/family_factory.h" +#ifdef TRTMC_HAS_EDGE_LLM +#include "families/qwen3_8/runtime/edge_llm/adapter.h" +#endif #include #include @@ -119,6 +122,14 @@ std::string chat_template(const BundleReader& bundle) { } // namespace ITask* create(const FamilyContext& context) { + if (context.reader.find_section("edge_llm.json")) { +#ifdef TRTMC_HAS_EDGE_LLM + return edge_llm::create(context.reader); +#else + throw std::runtime_error("Qwen3.8 Edge bundle requires a runtime configured with " + "-DTRTMC_ENABLE_EDGELLM=ON; rebuild and install Model Connect"); +#endif + } const RuntimeConfig config = parse_runtime_config(context.reader); auto decoder = load_engine(context.backend, require_section(context.reader, "engine.plan"), "qwen3_8 decoder"); diff --git a/families/qwen3_8/tests/test_e2e.py b/families/qwen3_8/tests/test_e2e.py index cd5ad94989..5e284af876 100644 --- a/families/qwen3_8/tests/test_e2e.py +++ b/families/qwen3_8/tests/test_e2e.py @@ -133,7 +133,7 @@ def _thresholds(case_name: str) -> dict[str, float]: return thresholds -def _build_bundle(manifest: dict, model_dir: Path, bundle: Path) -> None: +def _build_bundle(manifest: dict, model_dir: Path, bundle: Path, *, execution=None) -> None: quantization = manifest.get("quantization") assert quantization is None or isinstance(quantization, str) fp32_layers = tuple(manifest.get("fp32_layers", ())) @@ -148,7 +148,8 @@ def _build_bundle(manifest: dict, model_dir: Path, bundle: Path) -> None: tensor_parallel_size=manifest["tensor_parallel_size"], quantization=quantization, fp32_layers=fp32_layers, - ) + ), + execution=execution, ) assert bundle.is_file() and bundle.stat().st_size > 0, bundle @@ -227,7 +228,7 @@ def _run_native( completed = subprocess.run( command, - check=True, + check=False, capture_output=True, text=True, timeout=600, @@ -235,6 +236,9 @@ def _run_native( ) record_evidence("commands", {"argv": getattr(completed, "args", None)}) record_evidence("native", {"stdout": getattr(completed, "stdout", None), "stderr": getattr(completed, "stderr", None)}) + (tmp_path / "native.stdout.log").write_text(completed.stdout, encoding="utf-8") + (tmp_path / "native.stderr.log").write_text(completed.stderr, encoding="utf-8") + completed.check_returncode() if tp_size == 1: return json.loads(completed.stdout) @@ -353,16 +357,102 @@ def _hf_reference( "bf16": torch.bfloat16, } assert reference_precision in dtypes, reference_precision - model = ( - AutoModelForCausalLM.from_pretrained( + if case.get("reference_decode_modelopt_mixed", False): + # Preserve the declared FP32 oracle; packed bytes are not floating weights. + # This does not emulate compiled activation or KV quantization. + from modelopt.torch.export.quant_utils import QUANTIZATION_FP8, from_quantized_weight + from modelopt.torch.quantization.qtensor import NVFP4QTensor + from safetensors.torch import load_file + from transformers import AutoConfig, GenerationConfig, Qwen3_5ForCausalLM + + assert not trust_remote_code and reference_precision == "fp32" + raw = json.loads((model_dir / "config.json").read_text()) + quant = json.loads((model_dir / "hf_quant_config.json").read_text())["quantization"] + embedded = raw["quantization_config"] + assert embedded["quant_method"] == "modelopt" + assert embedded["quant_algo"] == quant["quant_algo"] == "MIXED_PRECISION" + layers = quant["quantized_layers"] + assert layers and embedded["quantized_layers"] == layers + assert all( + policy["quant_algo"] == "FP8" + or (policy["quant_algo"] == "NVFP4" and policy["group_size"] == 16) + for policy in layers.values() + ) + state = {} + for shard in sorted(model_dir.glob("*.safetensors")): + tensors = load_file(str(shard), device="cpu") + assert not state.keys() & tensors.keys(), "duplicate checkpoint tensors" + state.update(tensors) + packed = {key for key, value in state.items() if value.dtype == torch.uint8} + fp8 = { + key for key, value in state.items() + if key.endswith(".weight") and value.dtype == torch.float8_e4m3fn + } + quantized = packed | fp8 + assert packed and fp8 and quantized == {key + ".weight" for key in layers} + assert all(layers[key.removesuffix(".weight")]["quant_algo"] == "NVFP4" for key in packed) + assert all(layers[key.removesuffix(".weight")]["quant_algo"] == "FP8" for key in fp8) + for key in packed: + weight = state[key] + assert key.endswith(".weight") and weight.ndim == 2, key + assert weight.shape[-1] % 8 == 0, key + prefix = key.removesuffix("weight") + scale = state[prefix + "weight_scale"] + double_scale = state[prefix + "weight_scale_2"] + shape = (weight.shape[0], weight.shape[1] * 2) + assert scale.dtype == torch.float8_e4m3fn + assert scale.shape == (shape[0], shape[1] // 16), key + assert torch.isfinite(scale.float()).all() and (scale.float() >= 0).all(), key + assert double_scale.numel() == 1 and torch.isfinite(double_scale).all(), key + assert (double_scale > 0).all(), key + state[key] = NVFP4QTensor(shape, torch.float32, weight).dequantize( + dtype=torch.float32, scale=scale, double_scale=double_scale, + block_sizes={-1: 16}, fast=False, + ) + assert state[key].shape == shape and torch.isfinite(state[key]).all(), key + for key in fp8: + weight = state[key] + scale = state[key.removesuffix("weight") + "weight_scale"] + assert weight.ndim == 2 and scale.numel() == 1, key + assert torch.isfinite(scale).all() and (scale > 0).all(), key + state[key] = from_quantized_weight( + weight, scale, QUANTIZATION_FP8, torch.float32, + ) + assert state[key].shape == weight.shape and torch.isfinite(state[key]).all(), key + for key in list(state): + if key.endswith((".weight_scale", ".weight_scale_2", ".input_scale")): + assert key.rsplit(".", 1)[0] + ".weight" in quantized, key + scale = state.pop(key) + assert torch.isfinite(scale.float()).all() and (scale.float() >= 0).all(), key + # Use the original official text-only class and checkpoint prefix conversion. + # HF owns its declared ignoring of unused visual/MTP keys; do not filter weights. + config = AutoConfig.from_pretrained( + model_dir, local_files_only=True, trust_remote_code=False, + ).get_text_config(decoder=True) + assert config.model_type == "qwen3_5_text" + assert not getattr(config, "quantization_config", None) + model, loading = Qwen3_5ForCausalLM.from_pretrained( + None, config=config, state_dict=state, dtype=torch.float32, + output_loading_info=True, + ) + assert all(not loading.get(key) for key in ( + "missing_keys", "unexpected_keys", "mismatched_keys", "error_msgs", + )), loading + if (model_dir / "generation_config.json").is_file(): + model.generation_config = GenerationConfig.from_pretrained( + model_dir, local_files_only=True, + ) + del state, tensors + else: + model = AutoModelForCausalLM.from_pretrained( model_dir, local_files_only=True, trust_remote_code=trust_remote_code, dtype=dtypes[reference_precision], ) - .eval() - .to("cuda") - ) + reference_device = case.get("reference_device", "cuda") + assert reference_device in {"cpu", "cuda"}, reference_device + model = model.eval().to(reference_device) inputs = _render_prompt(tokenizer, prompt, case).to(model.device) prompt_ids = inputs["input_ids"][0].tolist() if "expected_prompt_token_ids" in case: diff --git a/website/docs/features/model-families.md b/website/docs/features/model-families.md index f3292546e0..0ffd17827b 100644 --- a/website/docs/features/model-families.md +++ b/website/docs/features/model-families.md @@ -88,6 +88,22 @@ chat-template handling are documented in the These local qualifications are separate from the registered manifest inventory and do not imply that CI executes the paired cases. +### Qwen3.8 paired ONNX execution + +The Qwen3.8 family owns explicit mixed-NVFP4 target plus DSpark block7 execution +through the optional pinned native Edge-LLM SDK. The qualified profile uses +`RadixArk/Qwen3.8-27B-NVFP4` and `RadixArk/Qwen3.8-27B-DSpark`, text-only +FP16 execution with the source mixed NVFP4/FP8 metadata, TP1/batch1 on SM120. +Standalone builds retain the original native path; ordinary experimental Edge +offload and other platform routes are not enabled by this change. + +Provision the [native SDK](../user-guides/configure-runtime.md#optional-native-edge-llm-sdk), +then add `--execution-variant dspark --companion draft=/path/to/draft` to the build +CLI. See the [owning Qwen3.8 recipe](https://github.com/NVIDIA/TensorRT-Model-Connect/blob/main/families/qwen3_8/EDGE_LLM.md) +for exact revisions, capacities, sampling controls and quality evidence. +The local paired qualification is not a registered manifest case and does not +imply CI coverage of that pair or statistical sampling equivalence. + ## Runtime and validation The directory name is also the runtime DSO identity: From 50daf7d7cb8c081bffcbe7ce798301dc32d6d1b1 Mon Sep 17 00:00:00 2001 From: Joshua Calafato Date: Thu, 17 Sep 2026 00:03:10 +0000 Subject: [PATCH 02/11] style(qwen3.8): format paired runtime Apply the pinned clang-format22.1.8 wrapping required by Source quality. Full public source-quality checks pass, and the rebuilt runtime is byte-identical to the independently qualified binary. Signed-off-by: Joshua Calafato --- families/qwen3_8/runtime/edge_llm/adapter.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/families/qwen3_8/runtime/edge_llm/adapter.cpp b/families/qwen3_8/runtime/edge_llm/adapter.cpp index 6690cd07e2..a23687fdb2 100644 --- a/families/qwen3_8/runtime/edge_llm/adapter.cpp +++ b/families/qwen3_8/runtime/edge_llm/adapter.cpp @@ -166,8 +166,8 @@ class Stream { }; /// Delegate the complete DSpark block7 algorithm to the pinned runtime. -std::unique_ptr -make_runtime(const Artifacts& artifacts, cudaStream_t stream) { +std::unique_ptr make_runtime(const Artifacts& artifacts, + cudaStream_t stream) { trt_edgellm::rt::SpecDecodeDraftingConfig drafting{}; drafting.draftingTopK = 1; drafting.draftingStep = 1; @@ -178,8 +178,8 @@ make_runtime(const Artifacts& artifacts, cudaStream_t stream) { drafting.dsparkMinProposalLen = 1; drafting.dsparkMaxProposalLen = 0; return std::make_unique( - artifacts.engine(), "", std::unordered_map{}, drafting, - stream, trt_edgellm::rt::ContextCacheConfig{}, artifacts.checkpoint(), + artifacts.engine(), "", std::unordered_map{}, drafting, stream, + trt_edgellm::rt::ContextCacheConfig{}, artifacts.checkpoint(), (fs::path(artifacts.checkpoint()) / "draft").string()); } From dc33919088e056010a3557ce570871737f0f9d8e Mon Sep 17 00:00:00 2001 From: Joshua Calafato Date: Mon, 21 Sep 2026 17:12:13 +0000 Subject: [PATCH 03/11] fix(qwen3.8): clean successful Edge diagnostics Retain logs for failed preparation or publication only, and separate qualification labels from numeric values. Signed-off-by: Joshua Calafato --- families/qwen3_8/EDGE_LLM.md | 18 +++++++++--------- families/qwen3_8/dispatch.py | 1 + 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/families/qwen3_8/EDGE_LLM.md b/families/qwen3_8/EDGE_LLM.md index 1160f29d8e..1915d01ef6 100644 --- a/families/qwen3_8/EDGE_LLM.md +++ b/families/qwen3_8/EDGE_LLM.md @@ -10,7 +10,7 @@ Edge only for an explicit mixed-NVFP4 target plus DSpark companion. Provision the optional native SDK with `TRTMC_EDGELLM_ALL_KERNELS=ON` and `TRTMC_EDGELLM_ONNX=ON` using the [pinned package instructions](../../cmake/edgellm/README.md). The source is -GitHub Edge-LLM0.10.1 at `e8b29522938901f6df19ebeedd4b69bc8edbcd97`. +GitHub Edge-LLM 0.10.1 at `e8b29522938901f6df19ebeedd4b69bc8edbcd97`. Configure `CMAKE_PREFIX_PATH` for the installed package and compile the runtime with `TRTMC_ENABLE_EDGELLM=ON`. Cross compilation is unsupported. @@ -25,7 +25,7 @@ speculative engines, embedding/head sidecars and tokenizer assets are bundled; checkpoint weights are not duplicated in the bundle. The runtime calls the original Edge speculative inference constructor with -proposal block7, verify8, drafting topK1/step1 and DSpark scheduling disabled. +proposal block 7, verify 8, drafting topK 1/step 1 and DSpark scheduling disabled. It preserves supported sampling controls. Engine preparation errors warn and try native once with the same requested execution variant; native currently rejects the unmapped DSpark variant explicitly. No failure substitutes an @@ -37,16 +37,16 @@ ordinary base-only decoder. Inference errors propagate without fallback. `319f741cce68d7914884900c138a1fbb70a42f30`. - Draft: `RadixArk/Qwen3.8-27B-DSpark`, revision `b9a5dbdf03bc999c6c73c426b19c2d9041cea393`. -- Native SM120, CUDA13.3, TensorRT11.1.0.106, FP16 execution with source mixed - NVFP4/FP8 metadata retained, TP1/batch1, input/KV capacity1024. +- Native SM120, CUDA 13.3, TensorRT 11.1.0.106, FP16 execution with source mixed + NVFP4/FP8 metadata retained, TP 1/batch 1, input/KV capacity 1024. - Actual Model Connect paired build and public CLI inference passed. -- Independent greedy oracle: exact token match, NED0.0 against0.15. +- Independent greedy oracle: exact token match, NED 0.0 against 0.15. The saved CPU FP32 reference was reused after checkpoint-byte and reference function verification; it was not regenerated during this run. -- Original Edge `llm_basic` prompt and128-token sampled profile - (temperature1/topK50/topP1): ROUGE-1 **0.4246**, ROUGE-L **0.2458**, above +- Original Edge `llm_basic` prompt and 128-token sampled profile + (temperature 1/topK 50/topP 1): ROUGE-1 **0.4246**, ROUGE-L **0.2458**, above unchanged **0.25/0.20** gates. Chat enabled and thinking disabled. -- Publication regressions:11 existing family Python tests plus106 existing +- Publication regressions: 11 existing family Python tests plus 106 existing builder/architecture tests pass; both existing native C++ tests pass. These results qualify only the exact text profile, not standalone NVFP4, @@ -55,6 +55,6 @@ The local run reused existing owning E2E helpers with an explicit companion; this pair is not yet a registered pytest manifest case. A first inference attempt exposed incompatible development JSON headers sharing -Edge’s3.12.0 version label. Matching the exact pinned headers fixed the crash +Edge’s 3.12.0 version label. Matching the exact pinned headers fixed the crash without changing Edge or the engines. The generic SDK now checks header content rather than relying only on the version label. diff --git a/families/qwen3_8/dispatch.py b/families/qwen3_8/dispatch.py index d833bf71ac..e983051356 100644 --- a/families/qwen3_8/dispatch.py +++ b/families/qwen3_8/dispatch.py @@ -90,6 +90,7 @@ def build(request, writer, native, *, draft_dir: Path) -> None: # Edge preparation did not touch writer; publication cannot fallback. if adapter is not None: edge_llm.publish(request, writer, files, marker) + log_path.unlink() return log_path.unlink() # A platform non-match is not an Edge failure. try: From a3549301594856aa3f9953fb61e5181a9a8368b7 Mon Sep 17 00:00:00 2001 From: Joshua Calafato Date: Wed, 23 Sep 2026 04:52:35 +0000 Subject: [PATCH 04/11] refactor(qwen3_8): own Edge build selection Keep optional Edge dispatch, request options, adapters, and CMake wiring within family-owned edge_llm folders. Route explicit companions through the generic lazy CLI hook and the ordinary family build entrypoint; preserve native fallback semantics and quality gates. Signed-off-by: Joshua Calafato --- .../{EDGE_LLM.md => edge_llm/README.md} | 36 ++++- families/qwen3_8/edge_llm/__init__.py | 4 + .../{edge_llm.py => edge_llm/builder.py} | 0 families/qwen3_8/edge_llm/cli.py | 42 +++++ families/qwen3_8/edge_llm/config.py | 88 +++++++++++ families/qwen3_8/{ => edge_llm}/dispatch.py | 49 +++++- families/qwen3_8/model.py | 54 +------ families/qwen3_8/runtime/CMakeLists.txt | 23 +-- .../qwen3_8/runtime/edge_llm/Adapter.cmake | 28 ++++ families/qwen3_8/support.py | 5 +- families/qwen3_8/tests/test_support.py | 146 ++++++++++++++++++ website/docs/features/model-families.md | 2 +- 12 files changed, 402 insertions(+), 75 deletions(-) rename families/qwen3_8/{EDGE_LLM.md => edge_llm/README.md} (70%) create mode 100644 families/qwen3_8/edge_llm/__init__.py rename families/qwen3_8/{edge_llm.py => edge_llm/builder.py} (100%) create mode 100644 families/qwen3_8/edge_llm/cli.py create mode 100644 families/qwen3_8/edge_llm/config.py rename families/qwen3_8/{ => edge_llm}/dispatch.py (60%) create mode 100644 families/qwen3_8/runtime/edge_llm/Adapter.cmake diff --git a/families/qwen3_8/EDGE_LLM.md b/families/qwen3_8/edge_llm/README.md similarity index 70% rename from families/qwen3_8/EDGE_LLM.md rename to families/qwen3_8/edge_llm/README.md index 1915d01ef6..b65e065495 100644 --- a/families/qwen3_8/EDGE_LLM.md +++ b/families/qwen3_8/edge_llm/README.md @@ -9,14 +9,13 @@ Edge only for an explicit mixed-NVFP4 target plus DSpark companion. Provision the optional native SDK with `TRTMC_EDGELLM_ALL_KERNELS=ON` and `TRTMC_EDGELLM_ONNX=ON` using the -[pinned package instructions](../../cmake/edgellm/README.md). The source is +[pinned package instructions](../../../cmake/edge_llm/README.md). The source is GitHub Edge-LLM 0.10.1 at `e8b29522938901f6df19ebeedd4b69bc8edbcd97`. Configure `CMAKE_PREFIX_PATH` for the installed package and compile the runtime with `TRTMC_ENABLE_EDGELLM=ON`. Cross compilation is unsupported. The Python build API accepts `BuildExecutionInputs(variant="dspark", -checkpoints=(NamedCheckpoint("draft", draft_path),))` through its `execution` -argument. The CLI equivalent adds `--execution-variant dspark` and +checkpoints=(NamedCheckpoint("draft", draft_path),))` on a family-owned typed request. The CLI equivalent adds `--execution-variant dspark` and `--companion draft=/path/to/draft` to an ordinary build invocation. The family invokes the original Edge Python ONNX exporter and native `edgellm-onnx-build`; it does not alter source tensors or pad safetensors headers. @@ -58,3 +57,34 @@ A first inference attempt exposed incompatible development JSON headers sharing Edge’s 3.12.0 version label. Matching the exact pinned headers fixed the crash without changing Edge or the engines. The generic SDK now checks header content rather than relying only on the version label. + +## Family-owned build options + +The generic CLI loads this family's `edge_llm.cli` hook only after resolving +the model. The shared build API has no execution-mode or companion arguments. +All variant validation and builder selection remain in this family. + +```sh +trtmc build /path/to/target --family qwen3_8 --precision fp16 \ + --execution-variant dspark --companion draft=/path/to/draft \ + -o model.bundle +``` + +Put MODEL immediately after `build`. `trtmc build /path/to/target --help` +shows these family options using local metadata; remote-ID help does not download +a checkpoint. For Python callers, use this family's request extension: + +```python +from tensorrt_model_connect import build +from families.qwen3_8.edge_llm.config import ( + BuildExecutionInputs, NamedCheckpoint, with_execution, +) + +# request is an ordinary BuildRequest owned by this family; draft_path is a Path. +build(with_execution(request, BuildExecutionInputs( + "dspark", (NamedCheckpoint("draft", draft_path),), +))) +``` + +A failed explicit pair is never replaced by a base-only bundle. Previously +recorded full-model results above are historical, not fresh refactor-head E2Es. diff --git a/families/qwen3_8/edge_llm/__init__.py b/families/qwen3_8/edge_llm/__init__.py new file mode 100644 index 0000000000..df93d8cabe --- /dev/null +++ b/families/qwen3_8/edge_llm/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Family-owned optional complete-network Edge offload.""" diff --git a/families/qwen3_8/edge_llm.py b/families/qwen3_8/edge_llm/builder.py similarity index 100% rename from families/qwen3_8/edge_llm.py rename to families/qwen3_8/edge_llm/builder.py diff --git a/families/qwen3_8/edge_llm/cli.py b/families/qwen3_8/edge_llm/cli.py new file mode 100644 index 0000000000..011357a69f --- /dev/null +++ b/families/qwen3_8/edge_llm/cli.py @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Qwen38-owned CLI options for explicit paired Edge execution.""" + +import argparse +from pathlib import Path + +from .config import BuildExecutionInputs, NamedCheckpoint, with_execution + + +def add_build_arguments(parser: argparse.ArgumentParser) -> None: + """Register options only when the Qwen38 family has been resolved.""" + parser.add_argument("--execution-variant", choices=("dspark",), + help="Explicit Qwen38 paired execution mode") + parser.add_argument("--companion", action="append", default=[], metavar="ROLE=LOCAL_DIR", + help="Explicit local draft checkpoint; exactly one draft role is required") + + +def _execution_inputs(args: argparse.Namespace) -> BuildExecutionInputs | None: + """Parse only explicit local inputs; no variant list or model acquisition.""" + if args.command != "build": + return None + if args.execution_variant is None: + if args.companion: + raise ValueError("--companion requires --execution-variant") + return None + checkpoints = [] + for value in args.companion: + role, separator, directory = value.partition("=") + if not separator or not role or not directory: + raise ValueError("--companion must be ROLE=LOCAL_DIR") + if "://" in directory: + raise ValueError("--companion requires a local directory, not a URI") + checkpoints.append(NamedCheckpoint(role, Path(directory))) + return BuildExecutionInputs(args.execution_variant, tuple(checkpoints)) + + +def prepare_build_request(request, args): + """Attach a validated family-owned recipe before importing the GPU builder.""" + execution = _execution_inputs(args) + return request if execution is None else with_execution(request, execution) diff --git a/families/qwen3_8/edge_llm/config.py b/families/qwen3_8/edge_llm/config.py new file mode 100644 index 0000000000..9baf43d3b4 --- /dev/null +++ b/families/qwen3_8/edge_llm/config.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Explicit Qwen38 paired-build inputs; no shared execution-mode contract.""" + +from dataclasses import dataclass, fields +from pathlib import Path +import re + +from tensorrt_model_connect.build import BuildRequest + + +_ID = re.compile(r"[a-z][a-z0-9_]*\Z") + + +def _validate_id(field: str, value: object) -> str: + if not isinstance(value, str) or _ID.fullmatch(value) is None: + raise ValueError(f"{field} must be a lowercase identifier containing only letters, digits, and underscores") + return value + + +@dataclass(frozen=True) +class NamedCheckpoint: + """One explicitly named local checkpoint; the family owns role semantics.""" + + role: str + model_dir: Path + + def __post_init__(self) -> None: + _validate_id("checkpoint role", self.role) + if not isinstance(self.model_dir, Path): + raise TypeError("checkpoint model_dir must be a Path") + if not self.model_dir.is_dir(): + raise ValueError(f"checkpoint must be an existing local directory: {self.model_dir}") + + +@dataclass(frozen=True) +class BuildExecutionInputs: + """Optional family-owned execution variant and immutable local companions. + + Qwen38 validates these inputs without fetching or inferring companions. + """ + + variant: str + checkpoints: tuple[NamedCheckpoint, ...] = () + + def __post_init__(self) -> None: + _validate_id("execution variant", self.variant) + if not isinstance(self.checkpoints, tuple) or any( + not isinstance(checkpoint, NamedCheckpoint) for checkpoint in self.checkpoints + ): + raise TypeError("checkpoints must be a tuple of NamedCheckpoint values") + roles = [checkpoint.role for checkpoint in self.checkpoints] + if len(roles) != len(set(roles)): + raise ValueError("checkpoint roles must be unique") + self.validate_local() + + def validate_local(self) -> None: + """Recheck local availability before dispatch, without acquiring inputs.""" + for checkpoint in self.checkpoints: + if not checkpoint.model_dir.is_dir(): + raise ValueError( + f"checkpoint must be an existing local directory: {checkpoint.model_dir}" + ) + + +@dataclass(frozen=True) +class Qwen38BuildRequest(BuildRequest): + """Ordinary build inputs plus an explicitly requested Qwen38 execution recipe.""" + + execution: BuildExecutionInputs | None = None + + def __post_init__(self) -> None: + super().__post_init__() + if self.family != "qwen3_8": + raise ValueError("Qwen38BuildRequest requires the qwen3_8 family") + if self.execution is not None: + if not isinstance(self.execution, BuildExecutionInputs): + raise TypeError("execution must be BuildExecutionInputs") + self.execution.validate_local() + + +def with_execution(request: BuildRequest, execution: BuildExecutionInputs) -> Qwen38BuildRequest: + """Preserve ordinary request fields and callback identity.""" + return Qwen38BuildRequest( + **{field.name: getattr(request, field.name) for field in fields(BuildRequest)}, + execution=execution, + ) diff --git a/families/qwen3_8/dispatch.py b/families/qwen3_8/edge_llm/dispatch.py similarity index 60% rename from families/qwen3_8/dispatch.py rename to families/qwen3_8/edge_llm/dispatch.py index e983051356..f5ed09cca2 100644 --- a/families/qwen3_8/dispatch.py +++ b/families/qwen3_8/edge_llm/dispatch.py @@ -12,7 +12,7 @@ import tempfile import traceback -from . import edge_llm +from . import builder as edge_llm _LOG = logging.getLogger(__name__) @@ -99,3 +99,50 @@ def build(request, writer, native, *, draft_dir: Path) -> None: if failure is not None: raise error from failure raise + + +def build_paired(request, writer, execution) -> None: + """Build the retained mixed-NVFP4 Qwen3.8 / DSpark block7 pair.""" + execution.validate_local() + + if execution.variant != "dspark" or tuple(x.role for x in execution.checkpoints) != ("draft",): + raise ValueError("Qwen3.8 paired execution requires variant=dspark and one draft checkpoint") + draft_dir = execution.checkpoints[0].model_dir + raw = json.loads((request.model_dir / "config.json").read_text()) + draft = json.loads((draft_dir / "config.json").read_text()) + if not candidate(request, raw) or edge_llm.checkpoint_quantization(request.model_dir, raw) != "nvfp4": + raise ValueError("The retained Qwen3.8 DSpark pair requires a matching mixed-NVFP4 base") + base = raw.get("text_config", raw) + if not isinstance(draft, dict) or draft.get("architectures") != ["DSparkDraftModel"]: + raise ValueError("Expected a DSparkDraftModel companion") + for name in ("hidden_size", "vocab_size"): + if type(draft.get(name)) is not int or draft[name] != base.get(name): + raise ValueError(f"Qwen3.8 DSpark base and draft disagree on {name}") + if draft.get("num_target_layers") != base.get("num_hidden_layers"): + raise ValueError("Qwen3.8 DSpark target layer count differs from base") + config = draft.get("dspark_config") + if not isinstance(config, dict) or config.get("block_size", draft.get("block_size")) != 7: + raise ValueError("Qwen3.8 DSpark maps the upstream block7 / verify8 profile") + layers = config.get("target_layer_ids", draft.get("target_layer_ids")) + if (not isinstance(layers, list) or not layers + or any(type(i) is not int or not 0 <= i < base["num_hidden_layers"] for i in layers) + or len(set(layers)) != len(layers)): + raise ValueError("Invalid Qwen3.8 DSpark target layer IDs") + mask = config.get("mask_token_id", draft.get("mask_token_id")) + if type(mask) is not int or not 0 <= mask < base["vocab_size"]: + raise ValueError("Invalid Qwen3.8 DSpark mask token") + limit = request.max_sequence_length or min(base["max_position_embeddings"], 256) + capacity = draft.get("max_position_embeddings") + if type(capacity) is not int or not 8 < limit <= capacity: + raise ValueError("Requested context exceeds DSpark draft capacity or block minimum") + if draft.get("quantization_config") or any( + (draft_dir / name).exists() + for name in ("hf_quant_config.json", "quantize_config.json", "quant_config.json") + ): + raise ValueError("This Qwen3.8 DSpark profile requires unquantized draft weights") + + def native_pair(original_request, original_writer): + # A failure must never replace the requested pair with base-only decoding. + raise NotImplementedError("Native Qwen3.8 does not implement the requested DSpark variant") + + build(request, writer, native_pair, draft_dir=draft_dir) diff --git a/families/qwen3_8/model.py b/families/qwen3_8/model.py index 12f6fe7535..f551097db1 100644 --- a/families/qwen3_8/model.py +++ b/families/qwen3_8/model.py @@ -82,6 +82,13 @@ def _runtime_config(model_dir: Path, config: ModelConfig, model: Qwen38Model, ** def build(request, writer) -> None: """Build one Qwen3.8 hybrid text-generation bundle.""" + + from .edge_llm.config import Qwen38BuildRequest + from .edge_llm.dispatch import build_paired + + if isinstance(request, Qwen38BuildRequest) and request.execution is not None: + build_paired(request, writer, request.execution) + return if request.dynamic_kv_cache: raise NotImplementedError("qwen3_8 does not support dynamic_kv_cache") @@ -171,50 +178,3 @@ def build(request, writer) -> None: path = model_dir / filename if path.is_file(): writer.add_bytes(filename, path.read_bytes()) - - -def build_with_inputs(request, writer, execution) -> None: - """Build the retained mixed-NVFP4 Qwen3.8 / DSpark block7 pair.""" - from . import dispatch, edge_llm - - if execution.variant != "dspark" or tuple(x.role for x in execution.checkpoints) != ("draft",): - raise ValueError("Qwen3.8 paired execution requires variant=dspark and one draft checkpoint") - draft_dir = execution.checkpoints[0].model_dir - raw = json.loads((request.model_dir / "config.json").read_text()) - draft = json.loads((draft_dir / "config.json").read_text()) - if not dispatch.candidate(request, raw) or edge_llm.checkpoint_quantization(request.model_dir, raw) != "nvfp4": - raise ValueError("The retained Qwen3.8 DSpark pair requires a matching mixed-NVFP4 base") - base = raw.get("text_config", raw) - if not isinstance(draft, dict) or draft.get("architectures") != ["DSparkDraftModel"]: - raise ValueError("Expected a DSparkDraftModel companion") - for name in ("hidden_size", "vocab_size"): - if type(draft.get(name)) is not int or draft[name] != base.get(name): - raise ValueError(f"Qwen3.8 DSpark base and draft disagree on {name}") - if draft.get("num_target_layers") != base.get("num_hidden_layers"): - raise ValueError("Qwen3.8 DSpark target layer count differs from base") - config = draft.get("dspark_config") - if not isinstance(config, dict) or config.get("block_size", draft.get("block_size")) != 7: - raise ValueError("Qwen3.8 DSpark maps the upstream block7 / verify8 profile") - layers = config.get("target_layer_ids", draft.get("target_layer_ids")) - if (not isinstance(layers, list) or not layers - or any(type(i) is not int or not 0 <= i < base["num_hidden_layers"] for i in layers) - or len(set(layers)) != len(layers)): - raise ValueError("Invalid Qwen3.8 DSpark target layer IDs") - mask = config.get("mask_token_id", draft.get("mask_token_id")) - if type(mask) is not int or not 0 <= mask < base["vocab_size"]: - raise ValueError("Invalid Qwen3.8 DSpark mask token") - limit = request.max_sequence_length or min(base["max_position_embeddings"], 256) - capacity = draft.get("max_position_embeddings") - if type(capacity) is not int or not 8 < limit <= capacity: - raise ValueError("Requested context exceeds DSpark draft capacity or block minimum") - if draft.get("quantization_config") or any( - (draft_dir / name).exists() - for name in ("hf_quant_config.json", "quantize_config.json", "quant_config.json") - ): - raise ValueError("This Qwen3.8 DSpark profile requires unquantized draft weights") - - def native_pair(original_request, original_writer): - # A failure must never replace the requested pair with base-only decoding. - raise NotImplementedError("Native Qwen3.8 does not implement the requested DSpark variant") - - dispatch.build(request, writer, native_pair, draft_dir=draft_dir) diff --git a/families/qwen3_8/runtime/CMakeLists.txt b/families/qwen3_8/runtime/CMakeLists.txt index b74b007112..ac275f707c 100644 --- a/families/qwen3_8/runtime/CMakeLists.txt +++ b/families/qwen3_8/runtime/CMakeLists.txt @@ -75,25 +75,4 @@ if(TRTMC_BUILD_TESTS) ) endif() -# Complete-network offload is family-owned and absent from native-only builds. -if(TARGET EdgeLLM::Core) - target_sources(trtmc_model_qwen3_8 PRIVATE - edge_llm/adapter.cpp - edge_llm/device_link.cu - ) - target_compile_definitions(trtmc_model_qwen3_8 PRIVATE TRTMC_HAS_EDGE_LLM=1) - target_link_libraries(trtmc_model_qwen3_8 PRIVATE EdgeLLM::Core) - set_target_properties(trtmc_model_qwen3_8 PROPERTIES - CUDA_ARCHITECTURES "${EdgeLLM_CUDA_ARCHITECTURE}" - CUDA_SEPARABLE_COMPILATION ON - CUDA_RESOLVE_DEVICE_SYMBOLS ON - ) -endif() - -if(TARGET EdgeLLM::Core) - add_custom_command(TARGET trtmc_model_qwen3_8 POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different - $ $ - VERBATIM - ) -endif() +include(edge_llm/Adapter.cmake) diff --git a/families/qwen3_8/runtime/edge_llm/Adapter.cmake b/families/qwen3_8/runtime/edge_llm/Adapter.cmake new file mode 100644 index 0000000000..8b96763a68 --- /dev/null +++ b/families/qwen3_8/runtime/edge_llm/Adapter.cmake @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Complete-network offload is family-owned and absent from native-only builds. +if(TARGET EdgeLLM::Core) + if(NOT TARGET EdgeLLM::Plugin) + message(FATAL_ERROR "Qwen38 Edge adapter requires the complete EdgeLLM package (Core and Plugin)") + endif() + target_sources(trtmc_model_qwen3_8 PRIVATE + "${CMAKE_CURRENT_LIST_DIR}/adapter.cpp" + "${CMAKE_CURRENT_LIST_DIR}/device_link.cu" + ) + target_compile_definitions(trtmc_model_qwen3_8 PRIVATE TRTMC_HAS_EDGE_LLM=1) + target_link_libraries(trtmc_model_qwen3_8 PRIVATE EdgeLLM::Core) + set_target_properties(trtmc_model_qwen3_8 PROPERTIES + CUDA_ARCHITECTURES "${EdgeLLM_CUDA_ARCHITECTURE}" + CUDA_SEPARABLE_COMPILATION ON + CUDA_RESOLVE_DEVICE_SYMBOLS ON + ) +endif() + +if(TARGET EdgeLLM::Core) + add_custom_command(TARGET trtmc_model_qwen3_8 POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ $ + VERBATIM + ) +endif() diff --git a/families/qwen3_8/support.py b/families/qwen3_8/support.py index 6db925d744..ef68a7bb72 100644 --- a/families/qwen3_8/support.py +++ b/families/qwen3_8/support.py @@ -10,10 +10,13 @@ model_types=("qwen38", "qwen3.8", "qwen3_8"), tasks=("text_generation",), default_task="text_generation", + build_cli_module="edge_llm.cli", default_precision="bf16", ) _SUPPORT = FamilySupport( - tasks=("text_generation",), default_task="text_generation", default_precision="bf16") + tasks=("text_generation",), default_task="text_generation", default_precision="bf16", + build_cli_module="edge_llm.cli", +) def describe(metadata: ModelMetadata) -> FamilySupport | None: diff --git a/families/qwen3_8/tests/test_support.py b/families/qwen3_8/tests/test_support.py index af5d5169a4..c28ff96bb9 100644 --- a/families/qwen3_8/tests/test_support.py +++ b/families/qwen3_8/tests/test_support.py @@ -1,6 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import pytest + from tensorrt_model_connect.model_support import ModelMetadata, resolve_family @@ -16,3 +18,147 @@ def test_qwen38_marker_default_task_is_checkpoint_owned() -> None: ) assert family == "qwen3_8" assert support.default_task == "text_generation" + assert support.build_cli_module == "edge_llm.cli" + + +def _edge_cli_source(tmp_path): + import json + + source = tmp_path / "target" + source.mkdir() + (source / "config.json").write_text(json.dumps({ + "model_type": "qwen3_5", "text_config": {"output_gate_type": "sigmoid"}, + })) + draft = tmp_path / "draft" + draft.mkdir() + return source, draft + + +def test_edge_cli_uses_ordinary_family_build(tmp_path, monkeypatch): + from tensorrt_model_connect import build_cli + from families.qwen3_8.edge_llm import dispatch + from families.qwen3_8.edge_llm.config import Qwen38BuildRequest + + source, draft = _edge_cli_source(tmp_path) + output = tmp_path / "pair.bundle" + seen = [] + + def paired(request, writer, execution): + assert isinstance(request, Qwen38BuildRequest) + assert request.execution is execution + assert execution.variant == "dspark" + assert [(item.role, item.model_dir) for item in execution.checkpoints] == [("draft", draft)] + seen.append(request) + writer.set_header(family="qwen3_8", task=request.task, backend=request.backend) + writer.add_json("edge-test.json", {"variant": execution.variant}) + + monkeypatch.setattr(dispatch, "build_paired", paired) + assert build_cli.main([ + "build", str(source), "--precision", "fp16", "-o", str(output), + "--execution-variant", "dspark", "--companion", f"draft={draft}", + ]) == 0 + assert len(seen) == 1 + assert output.is_file() + + +@pytest.mark.parametrize("options", [ + ["--companion", "draft=/missing"], + ["--execution-variant", "dspark", "--companion", "missing_separator"], + ["--execution-variant", "dspark", "--companion", "=path"], + ["--execution-variant", "dspark", "--companion", "draft="], + ["--execution-variant", "dspark", "--companion", "draft=https://example.com/model"], +]) +def test_bad_edge_cli_inputs_fail_before_backend(tmp_path, monkeypatch, options): + import importlib + from tensorrt_model_connect import build_cli + + core = importlib.import_module("tensorrt_model_connect.build") + source, _ = _edge_cli_source(tmp_path) + monkeypatch.setattr(core, "_select_backend", lambda *_: pytest.fail("backend touched")) + monkeypatch.setattr(core, "BundleWriter", lambda *_: pytest.fail("writer created")) + with pytest.raises(ValueError): + build_cli.main(["build", str(source), "-o", str(tmp_path / "out"), *options]) + + +def test_edge_cli_help_is_family_owned(tmp_path, capsys): + from tensorrt_model_connect import build_cli + + source, _ = _edge_cli_source(tmp_path) + with pytest.raises(SystemExit) as caught: + build_cli.main(["build", str(source), "--help"]) + assert caught.value.code == 0 + help_text = capsys.readouterr().out + assert "--execution-variant {dspark}" in help_text + assert "--companion" in help_text + + +def test_edge_request_preserves_fields_and_family_owner(tmp_path): + import argparse + from families.qwen3_8.edge_llm import cli + from dataclasses import fields, replace + from tensorrt_model_connect.build import BuildRequest + from families.qwen3_8.edge_llm.config import ( + BuildExecutionInputs, NamedCheckpoint, with_execution, + ) + + source, draft = _edge_cli_source(tmp_path) + request = BuildRequest(source, tmp_path / "out", "qwen3_8", "text_generation", "fp16", + graph_transform=lambda layer: layer) + execution = BuildExecutionInputs("dspark", (NamedCheckpoint("draft", draft),)) + args = argparse.Namespace(command="build", execution_variant=None, companion=[]) + assert cli.prepare_build_request(request, args) is request + extended = with_execution(request, execution) + for field in fields(BuildRequest): + assert getattr(extended, field.name) is getattr(request, field.name) + with pytest.raises(ValueError, match="requires the qwen3_8 family"): + with_execution(replace(request, family="other"), execution) + with pytest.raises(ValueError, match="unique"): + BuildExecutionInputs("dspark", execution.checkpoints * 2) + + +@pytest.mark.parametrize("failure", [RuntimeError("paired build failed"), KeyboardInterrupt()]) +def test_edge_cli_failure_preserves_existing_bundle(tmp_path, monkeypatch, failure): + from tensorrt_model_connect import build_cli + from families.qwen3_8.edge_llm import dispatch + + source, draft = _edge_cli_source(tmp_path) + output = tmp_path / "pair.bundle" + output.write_bytes(b"previous publication") + + def fail(request, writer, execution): + writer.set_header(family="qwen3_8", task=request.task, backend=request.backend) + writer.add_json("edge-test.json", {"variant": execution.variant}) + raise failure + + monkeypatch.setattr(dispatch, "build_paired", fail) + with pytest.raises(type(failure)) as caught: + build_cli.main([ + "build", str(source), "-o", str(output), "--execution-variant", "dspark", + "--companion", f"draft={draft}", + ]) + assert caught.value is failure + assert output.read_bytes() == b"previous publication" + assert sorted(item.name for item in tmp_path.iterdir()) == ["draft", "pair.bundle", "target"] + + +def test_edge_pair_requires_draft_and_rechecks_local_inputs(tmp_path): + from tensorrt_model_connect.build import BuildRequest + from families.qwen3_8.edge_llm.config import BuildExecutionInputs, NamedCheckpoint + from families.qwen3_8.edge_llm.dispatch import build_paired + + source, draft = _edge_cli_source(tmp_path) + request = BuildRequest(source, tmp_path / "out", "qwen3_8", "text_generation", "fp16") + with pytest.raises(ValueError, match="paired execution requires"): + build_paired(request, None, BuildExecutionInputs("dspark")) + execution = BuildExecutionInputs("dspark", (NamedCheckpoint("draft", draft),)) + draft.rmdir() + with pytest.raises(ValueError, match="existing local directory"): + build_paired(request, None, execution) + + +@pytest.mark.parametrize("model_type", ["qwen38", "qwen3.8", "qwen3_8"]) +def test_qwen38_aliases_register_the_same_family_cli(model_type): + from families.qwen3_8.support import describe + + support = describe(ModelMetadata({"model_type": model_type}, {})) + assert support.build_cli_module == "edge_llm.cli" diff --git a/website/docs/features/model-families.md b/website/docs/features/model-families.md index 0ffd17827b..39f6587801 100644 --- a/website/docs/features/model-families.md +++ b/website/docs/features/model-families.md @@ -99,7 +99,7 @@ offload and other platform routes are not enabled by this change. Provision the [native SDK](../user-guides/configure-runtime.md#optional-native-edge-llm-sdk), then add `--execution-variant dspark --companion draft=/path/to/draft` to the build -CLI. See the [owning Qwen3.8 recipe](https://github.com/NVIDIA/TensorRT-Model-Connect/blob/main/families/qwen3_8/EDGE_LLM.md) +CLI. See the [owning Qwen3.8 recipe](https://github.com/NVIDIA/TensorRT-Model-Connect/blob/main/families/qwen3_8/edge_llm/README.md) for exact revisions, capacities, sampling controls and quality evidence. The local paired qualification is not a registered manifest case and does not imply CI coverage of that pair or statistical sampling equivalence. From 463087c3d0dcaee9f96aea9d625ff355adee0cdf Mon Sep 17 00:00:00 2001 From: Joshua Calafato Date: Wed, 23 Sep 2026 15:53:54 +0000 Subject: [PATCH 05/11] fix(qwen3_8): validate paired requests early Separate request errors from checkpoint compatibility, enforce the existing paired capacity before preparation, and stage next to the output. Extend existing contract regressions without changing quality gates. Signed-off-by: Joshua Calafato --- families/qwen3_8/edge_llm/README.md | 6 +- families/qwen3_8/edge_llm/dispatch.py | 34 ++++++++--- families/qwen3_8/tests/test_support.py | 85 ++++++++++++++++++++++++++ 3 files changed, 116 insertions(+), 9 deletions(-) diff --git a/families/qwen3_8/edge_llm/README.md b/families/qwen3_8/edge_llm/README.md index b65e065495..8b4ca5891f 100644 --- a/families/qwen3_8/edge_llm/README.md +++ b/families/qwen3_8/edge_llm/README.md @@ -70,7 +70,7 @@ trtmc build /path/to/target --family qwen3_8 --precision fp16 \ -o model.bundle ``` -Put MODEL immediately after `build`. `trtmc build /path/to/target --help` +Put MODEL before family-specific options; known core options may precede MODEL. `trtmc build /path/to/target --help` shows these family options using local metadata; remote-ID help does not download a checkpoint. For Python callers, use this family's request extension: @@ -88,3 +88,7 @@ build(with_execution(request, BuildExecutionInputs( A failed explicit pair is never replaced by a base-only bundle. Previously recorded full-model results above are historical, not fresh refactor-head E2Es. + +Request controls and the existing 9–1024 capacity range are checked before any +Edge preparation. The native precision default is not changed: this paired +profile explicitly requires FP16. Temporary staging uses the output filesystem. diff --git a/families/qwen3_8/edge_llm/dispatch.py b/families/qwen3_8/edge_llm/dispatch.py index f5ed09cca2..f74f100e14 100644 --- a/families/qwen3_8/edge_llm/dispatch.py +++ b/families/qwen3_8/edge_llm/dispatch.py @@ -22,6 +22,18 @@ } +def request_matches(request) -> bool: + """Validate mapped request controls independently of checkpoint metadata.""" + return ( + request.backend == "trt" and request.task == "text_generation" + and request.precision.lower() == "fp16" + and request.quantization in {None, "nvfp4"} + and request.max_batch_size == request.tensor_parallel_size == request.context_parallel_size == 1 + and not request.dynamic_kv_cache and not request.fp32_layers and request.graph_transform is None + and all(value is None for value in (request.image_height, request.image_width, request.video_num_frames)) + ) + + def candidate(request, raw: dict) -> bool: """Return whether this family's model/request contract can delegate to Edge.""" config = raw.get("text_config", raw) @@ -33,12 +45,7 @@ def candidate(request, raw: dict) -> bool: and config.get("linear_key_head_dim") == config.get("linear_value_head_dim") == 128 and not config.get("num_experts") and source_quantization == "nvfp4" - and request.backend == "trt" and request.task == "text_generation" - and request.precision.lower() == "fp16" - and request.quantization in {None, source_quantization} - and request.max_batch_size == request.tensor_parallel_size == request.context_parallel_size == 1 - and not request.dynamic_kv_cache and not request.fp32_layers and request.graph_transform is None - and all(value is None for value in (request.image_height, request.image_width, request.video_num_frames)) + and request_matches(request) ) @@ -73,7 +80,9 @@ def build(request, writer, native, *, draft_dir: Path) -> None: dir=request.output_path.parent) os.close(descriptor) log_path = Path(name) - with tempfile.TemporaryDirectory(prefix="trtmc-qwen3_8-edge-") as directory: + with tempfile.TemporaryDirectory( + prefix=f".{request.output_path.name}.edge-", dir=request.output_path.parent + ) as directory: try: target = edge_llm.local_target() key = (target["os"], target["arch"], target["sm"], request.precision.lower()) @@ -107,10 +116,17 @@ def build_paired(request, writer, execution) -> None: if execution.variant != "dspark" or tuple(x.role for x in execution.checkpoints) != ("draft",): raise ValueError("Qwen3.8 paired execution requires variant=dspark and one draft checkpoint") + if request.precision.lower() != "fp16": + raise ValueError("Qwen3.8 DSpark requires --precision fp16; the native default is unchanged") + if not request_matches(request): + raise ValueError( + "Qwen3.8 DSpark requires backend=trt, text_generation, batch/TP/CP=1, " + "quantization unset or nvfp4, and no dynamic KV, FP32 layers, graph or media overrides" + ) draft_dir = execution.checkpoints[0].model_dir raw = json.loads((request.model_dir / "config.json").read_text()) draft = json.loads((draft_dir / "config.json").read_text()) - if not candidate(request, raw) or edge_llm.checkpoint_quantization(request.model_dir, raw) != "nvfp4": + if not candidate(request, raw): raise ValueError("The retained Qwen3.8 DSpark pair requires a matching mixed-NVFP4 base") base = raw.get("text_config", raw) if not isinstance(draft, dict) or draft.get("architectures") != ["DSparkDraftModel"]: @@ -132,6 +148,8 @@ def build_paired(request, writer, execution) -> None: if type(mask) is not int or not 0 <= mask < base["vocab_size"]: raise ValueError("Invalid Qwen3.8 DSpark mask token") limit = request.max_sequence_length or min(base["max_position_embeddings"], 256) + if not 8 < limit <= 1024: + raise ValueError("Qwen3.8 DSpark requires max_sequence_length above 8 and at most 1024") capacity = draft.get("max_position_embeddings") if type(capacity) is not int or not 8 < limit <= capacity: raise ValueError("Requested context exceeds DSpark draft capacity or block minimum") diff --git a/families/qwen3_8/tests/test_support.py b/families/qwen3_8/tests/test_support.py index c28ff96bb9..8096e3bebe 100644 --- a/families/qwen3_8/tests/test_support.py +++ b/families/qwen3_8/tests/test_support.py @@ -162,3 +162,88 @@ def test_qwen38_aliases_register_the_same_family_cli(model_type): support = describe(ModelMetadata({"model_type": model_type}, {})) assert support.build_cli_module == "edge_llm.cli" + + +def _dspark_pair_request(tmp_path): + import json + from tensorrt_model_connect.build import BuildRequest + from families.qwen3_8.edge_llm.config import BuildExecutionInputs, NamedCheckpoint + + source, draft = _edge_cli_source(tmp_path) + layers = {"layer0": {"quant_algo": "FP8"}, "layer1": {"quant_algo": "NVFP4"}} + base = { + "model_type": "qwen3_5", "output_gate_type": "sigmoid", + "hidden_size": 16, "vocab_size": 32, "num_hidden_layers": 8, + "max_position_embeddings": 4096, "linear_key_head_dim": 128, + "linear_value_head_dim": 128, "quantization_config": { + "quant_method": "modelopt", "quant_algo": "MIXED_PRECISION", + "quantized_layers": layers, "kv_cache_scheme": {"type": "float"}, + }, + } + (source / "config.json").write_text(json.dumps(base)) + (source / "hf_quant_config.json").write_text(json.dumps({"quantization": { + "quant_algo": "MIXED_PRECISION", "quantized_layers": layers, + "kv_cache_quant_algo": "FP8", + }})) + (draft / "config.json").write_text(json.dumps({ + "architectures": ["DSparkDraftModel"], "hidden_size": 16, "vocab_size": 32, + "num_target_layers": 8, "max_position_embeddings": 4096, + "dspark_config": {"block_size": 7, "target_layer_ids": [1], "mask_token_id": 3}, + })) + return ( + BuildRequest(source, tmp_path / "out", "qwen3_8", "text_generation", "fp16"), + BuildExecutionInputs("dspark", (NamedCheckpoint("draft", draft),)), + ) + + +@pytest.mark.parametrize("overrides, message", [ + ({"precision": "bf16"}, "--precision fp16"), + ({"backend": "trt_rtx"}, "requires backend=trt"), + ({"max_batch_size": 2}, "batch/TP/CP=1"), + ({"tensor_parallel_size": 2}, "batch/TP/CP=1"), + ({"dynamic_kv_cache": True}, "no dynamic KV"), + ({"quantization": "fp8"}, "quantization unset or nvfp4"), + ({"max_sequence_length": 8}, "above 8 and at most 1024"), + ({"max_sequence_length": 1025}, "above 8 and at most 1024"), + ({"max_sequence_length": 2048}, "above 8 and at most 1024"), +]) +def test_dspark_request_errors_precede_adapter_work(tmp_path, monkeypatch, overrides, message): + from dataclasses import replace + from families.qwen3_8.edge_llm import dispatch + + request, execution = _dspark_pair_request(tmp_path) + monkeypatch.setattr(dispatch, "build", lambda *_args, **_kw: pytest.fail("adapter work started")) + with pytest.raises(ValueError, match=message) as caught: + dispatch.build_paired(replace(request, **overrides), None, execution) + assert caught.value.__cause__ is None + assert not list(tmp_path.glob(".out.edge-*")) + + +@pytest.mark.parametrize("limit", [9, 1024]) +def test_dspark_capacity_boundaries_keep_the_requested_pair(tmp_path, monkeypatch, limit): + from dataclasses import replace + from families.qwen3_8.edge_llm import dispatch + + request, execution = _dspark_pair_request(tmp_path) + request = replace(request, max_sequence_length=limit) + seen = [] + + def build(original, writer, native, *, draft_dir): + assert original is request and draft_dir == execution.checkpoints[0].model_dir + with pytest.raises(NotImplementedError, match="Native Qwen3.8"): + native(original, writer) + seen.append(original) + + monkeypatch.setattr(dispatch, "build", build) + dispatch.build_paired(request, None, execution) + assert seen == [request] + + +def test_dspark_checkpoint_error_is_distinct_from_request_error(tmp_path, monkeypatch): + from families.qwen3_8.edge_llm import dispatch + + request, execution = _dspark_pair_request(tmp_path) + (request.model_dir / "hf_quant_config.json").unlink() + monkeypatch.setattr(dispatch, "build", lambda *_args, **_kw: pytest.fail("adapter work started")) + with pytest.raises(ValueError, match="matching mixed-NVFP4 base"): + dispatch.build_paired(request, None, execution) From 79dfc64915be84ecffde4ab22bc71a40b07afc0b Mon Sep 17 00:00:00 2001 From: Joshua Calafato Date: Wed, 23 Sep 2026 17:12:37 +0000 Subject: [PATCH 06/11] fix(qwen3_8): explain unsupported DSpark platforms Signed-off-by: Joshua Calafato --- families/qwen3_8/edge_llm/dispatch.py | 5 ++++- families/qwen3_8/tests/test_support.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/families/qwen3_8/edge_llm/dispatch.py b/families/qwen3_8/edge_llm/dispatch.py index f74f100e14..aa3e2ed492 100644 --- a/families/qwen3_8/edge_llm/dispatch.py +++ b/families/qwen3_8/edge_llm/dispatch.py @@ -161,6 +161,9 @@ def build_paired(request, writer, execution) -> None: def native_pair(original_request, original_writer): # A failure must never replace the requested pair with base-only decoding. - raise NotImplementedError("Native Qwen3.8 does not implement the requested DSpark variant") + raise NotImplementedError( + "Native Qwen3.8 does not implement the requested DSpark variant; " + "the qualified Edge route requires Linux x86_64, SM120 and FP16" + ) build(request, writer, native_pair, draft_dir=draft_dir) diff --git a/families/qwen3_8/tests/test_support.py b/families/qwen3_8/tests/test_support.py index 8096e3bebe..331f63609b 100644 --- a/families/qwen3_8/tests/test_support.py +++ b/families/qwen3_8/tests/test_support.py @@ -247,3 +247,14 @@ def test_dspark_checkpoint_error_is_distinct_from_request_error(tmp_path, monkey monkeypatch.setattr(dispatch, "build", lambda *_args, **_kw: pytest.fail("adapter work started")) with pytest.raises(ValueError, match="matching mixed-NVFP4 base"): dispatch.build_paired(request, None, execution) + + +@pytest.mark.parametrize("arch, sm", [("x86_64", 90), ("aarch64", 120)]) +def test_dspark_unqualified_platform_names_required_route(tmp_path, monkeypatch, arch, sm): + from families.qwen3_8.edge_llm import builder, dispatch + + request, execution = _dspark_pair_request(tmp_path) + monkeypatch.setattr(builder, "local_target", lambda: {"os": "linux", "arch": arch, "sm": sm}) + with pytest.raises(NotImplementedError, match="Linux x86_64, SM120 and FP16"): + dispatch.build_paired(request, None, execution) + assert not list(tmp_path.glob(".out.edge-*")) From e81423bf3ab00c87a5149fb652de624e6e367453 Mon Sep 17 00:00:00 2001 From: Joshua Calafato Date: Wed, 23 Sep 2026 19:47:00 +0000 Subject: [PATCH 07/11] refactor(qwen3_8): use declared family CLI Reuse the existing family CLI protocol instead of extending the shared parser. Own the command description, request contract and build lifecycle; keep Edge companion semantics inside this family. Preserve legacy callers through strict conversion and keep numerical acceptance gates unchanged. Signed-off-by: Joshua Calafato --- families/qwen3_8/build_request.py | 94 ++++++++++++++++ families/qwen3_8/cli.json | 142 ++++++++++++++++++++++++ families/qwen3_8/cli.py | 61 ++++++++++ families/qwen3_8/edge_llm/README.md | 15 ++- families/qwen3_8/edge_llm/cli.py | 33 ++---- families/qwen3_8/edge_llm/config.py | 5 +- families/qwen3_8/model.py | 3 + families/qwen3_8/support.py | 2 - families/qwen3_8/tests/test_support.py | 85 +++++++++++--- website/docs/features/model-families.md | 5 + 10 files changed, 401 insertions(+), 44 deletions(-) create mode 100644 families/qwen3_8/build_request.py create mode 100644 families/qwen3_8/cli.json create mode 100644 families/qwen3_8/cli.py diff --git a/families/qwen3_8/build_request.py b/families/qwen3_8/build_request.py new file mode 100644 index 0000000000..698784cc54 --- /dev/null +++ b/families/qwen3_8/build_request.py @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""qwen3_8 build inputs and strict compatibility for existing Python callers.""" + +from __future__ import annotations + +from dataclasses import dataclass, fields +from pathlib import Path +import re +from typing import ClassVar + +from tensorrt_model_connect.graph_transform import GraphTransform + + +def _validate_id(field: str, value: str) -> None: + if not isinstance(value, str) or re.fullmatch(r"[a-z][a-z0-9_]*", value) is None: + raise ValueError(f"{field} must be a lowercase identifier") + + +@dataclass(frozen=True) +class BuildRequest: + """qwen3_8-owned inputs; unsupported legacy controls are read-only defaults.""" + + model_dir: Path + output_path: Path + family: str + task: str + precision: str + backend: str = "trt" + max_sequence_length: int | None = None + image_height: ClassVar[int | None] = None + image_width: ClassVar[int | None] = None + video_num_frames: ClassVar[int | None] = None + max_batch_size: ClassVar[int] = 1 + tensor_parallel_size: int = 1 + context_parallel_size: ClassVar[int] = 1 + quantization: str | None = None + fp32_layers: tuple[int, ...] = () + dynamic_kv_cache: ClassVar[bool] = False + verbose: bool = False + graph_transform: GraphTransform | None = None + + def __post_init__(self) -> None: + if not self.precision: + raise ValueError("precision must be non-empty") + _validate_id("family", self.family) + _validate_id("task", self.task) + if self.backend not in {"trt", "trt_rtx"}: + raise ValueError("backend must be 'trt' or 'trt_rtx'") + if self.max_sequence_length is not None and self.max_sequence_length < 1: + raise ValueError("max_sequence_length must be positive") + for field in ("image_height", "image_width", "video_num_frames"): + value = getattr(self, field) + if value is not None and value < 1: + raise ValueError(f"{field} must be positive") + if self.max_batch_size < 1: + raise ValueError("max_batch_size must be positive") + if self.tensor_parallel_size < 1: + raise ValueError("tensor_parallel_size must be positive") + if self.context_parallel_size < 1: + raise ValueError("context_parallel_size must be positive") + if self.quantization is not None and not self.quantization: + raise ValueError("quantization must be non-empty when provided") + if any(layer < 0 for layer in self.fp32_layers): + raise ValueError("fp32_layers must contain non-negative indices") + if not isinstance(self.dynamic_kv_cache, bool): + raise ValueError("dynamic_kv_cache must be a bool") + if self.graph_transform is not None and not callable(self.graph_transform): + raise ValueError("graph_transform must be callable when provided") + + +def coerce_request(request: object) -> BuildRequest: + """Reject unsupported/unknown legacy inputs before converting to owner fields.""" + if isinstance(request, BuildRequest): + return request + unsupported = { + "image_height": None, + "image_width": None, + "video_num_frames": None, + "max_batch_size": 1, + "context_parallel_size": 1, + "dynamic_kv_cache": False, + } + for name, default in unsupported.items(): + value = getattr(request, name, default) + if name == "quantization" and value == "none": + continue + if value != default: + raise NotImplementedError(f"qwen3_8 does not support {name}") + names = {field.name for field in fields(BuildRequest)} + if unknown := set(vars(request)) - names - set(unsupported): + raise ValueError(f"unknown qwen3_8 build inputs: {sorted(unknown)}") + return BuildRequest(**{name: getattr(request, name) for name in names}) diff --git a/families/qwen3_8/cli.json b/families/qwen3_8/cli.json new file mode 100644 index 0000000000..01fa56b9f7 --- /dev/null +++ b/families/qwen3_8/cli.json @@ -0,0 +1,142 @@ +{ + "version": 1, + "commands": [ + { + "name": "build", + "help": "Build one qwen3_8 TensorRT bundle", + "executor": "python", + "handler": "cli:build", + "arguments": [ + { + "name": "model", + "type": "string", + "help": "Hugging Face model ID or local snapshot" + }, + { + "name": "output", + "flags": [ + "-o", + "--output" + ], + "type": "path", + "required": true + }, + { + "name": "revision", + "flags": [ + "--revision" + ], + "type": "string" + }, + { + "name": "task", + "flags": [ + "--task" + ], + "type": "string", + "choices": [ + "text_generation" + ], + "default": "text_generation" + }, + { + "name": "precision", + "flags": [ + "--precision" + ], + "type": "string", + "choices": [ + "fp32", + "fp16", + "bf16" + ], + "default": "bf16" + }, + { + "name": "backend", + "flags": [ + "--backend" + ], + "type": "string", + "choices": [ + "trt", + "trt_rtx" + ], + "default": "trt" + }, + { + "name": "max_sequence_length", + "flags": [ + "--max-sequence-length" + ], + "type": "int" + }, + { + "name": "tensor_parallel_size", + "flags": [ + "--tensor-parallel-size" + ], + "type": "int", + "choices": [ + 1, + 2, + 4, + 8 + ], + "default": 1 + }, + { + "name": "verbose", + "flags": [ + "--verbose" + ], + "type": "bool", + "action": "store_true", + "default": false + }, + { + "name": "fp32_layers", + "flags": [ + "--fp32-layer" + ], + "type": "int", + "action": "append", + "default": [] + }, + { + "name": "quantization", + "flags": [ + "--quantization" + ], + "type": "string", + "choices": [ + "none", + "nvfp4", + "fp8" + ] + }, + { + "name": "execution_variant", + "flags": [ + "--execution-variant" + ], + "type": "string", + "choices": [ + "dspark" + ], + "help": "Explicit paired execution mode; no automatic draft discovery" + }, + { + "name": "companion", + "flags": [ + "--companion" + ], + "type": "string", + "action": "append", + "default": [], + "help": "Local companion checkpoint as ROLE=LOCAL_DIR" + } + ] + } + ] +} diff --git a/families/qwen3_8/cli.py b/families/qwen3_8/cli.py new file mode 100644 index 0000000000..a0199d3703 --- /dev/null +++ b/families/qwen3_8/cli.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""qwen3_8-owned build command and typed inputs; importing this module is CPU-only.""" + +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path + +from tensorrt_model_connect.build import select_backend +from tensorrt_model_connect.bundle_writer import BundleWriter +from tensorrt_model_connect.graph_transform import graph_transform +from tensorrt_model_connect.model_support import load_model_metadata, resolve_family, resolve_model + +from .build_request import BuildRequest + +from .edge_llm.cli import execution_inputs +from .edge_llm.config import with_execution + + +def build_bundle(request: BuildRequest, output: Path) -> None: + """Build and publish through the owning model, preserving atomic failure.""" + request = replace(request, output_path=output) + select_backend(request.backend) + from .model import build as build_model + + writer = BundleWriter(output) + try: + with graph_transform(request.graph_transform): + build_model(request, writer) + writer.finish() + except BaseException: + writer.abort() + raise + +def build( + *, model: str, output: Path, revision: str | None = None, + task: str = "text_generation", precision: str = "bf16", backend: str = "trt", + max_sequence_length: int | None = None, tensor_parallel_size: int = 1, + verbose: bool = False, + fp32_layers: list[int] | tuple[int, ...] = (), + quantization: str | None = None, + execution_variant: str | None = None, companion: list[str] | tuple[str, ...] = (), +) -> int: + """Run the declared owner command; help never imports this handler.""" + execution = execution_inputs(execution_variant, companion) + model_dir = resolve_model(model, revision) + resolve_family(load_model_metadata(model_dir), "qwen3_8") + request = BuildRequest( + model_dir=model_dir, output_path=output, family="qwen3_8", + task=task, precision=precision, backend=backend, + max_sequence_length=max_sequence_length, tensor_parallel_size=tensor_parallel_size, + verbose=verbose, + fp32_layers=tuple(fp32_layers), + quantization=quantization, + ) + if execution is not None: + request = with_execution(request, execution) + build_bundle(request, output) + return 0 diff --git a/families/qwen3_8/edge_llm/README.md b/families/qwen3_8/edge_llm/README.md index 8b4ca5891f..3fffd82d6c 100644 --- a/families/qwen3_8/edge_llm/README.md +++ b/families/qwen3_8/edge_llm/README.md @@ -60,17 +60,18 @@ rather than relying only on the version label. ## Family-owned build options -The generic CLI loads this family's `edge_llm.cli` hook only after resolving -the model. The shared build API has no execution-mode or companion arguments. +The existing family CLI reads this owner's cli.json and invokes cli.py. +Edge-specific inputs and selection remain in edge_llm/; the shared parser, +CLI protocol and build API gain no new options or hooks. All variant validation and builder selection remain in this family. ```sh -trtmc build /path/to/target --family qwen3_8 --precision fp16 \ +trtmc qwen3_8 build /path/to/target --precision fp16 \ --execution-variant dspark --companion draft=/path/to/draft \ -o model.bundle ``` -Put MODEL before family-specific options; known core options may precede MODEL. `trtmc build /path/to/target --help` +Options may precede or follow MODEL. `trtmc qwen3_8 build /path/to/target --help` shows these family options using local metadata; remote-ID help does not download a checkpoint. For Python callers, use this family's request extension: @@ -92,3 +93,9 @@ recorded full-model results above are historical, not fresh refactor-head E2Es. Request controls and the existing 9–1024 capacity range are checked before any Edge preparation. The native precision default is not changed: this paired profile explicitly requires FP16. Temporary staging uses the output filesystem. + +## Declared build command + +This family uses the existing cli.json protocol introduced in #1310. The family\nowns its declaration, typed inputs and Python handler. The handler adapts those\ninputs to the unchanged builder API, preserving native/Edge dispatch and bundle\npublication. The legacy flat build command remains available for its existing\nordinary options; new family options use `trtmc qwen3_8 build`. +Help is offline and does not need a local checkpoint. No shared parser hook or +family registry entry is added. diff --git a/families/qwen3_8/edge_llm/cli.py b/families/qwen3_8/edge_llm/cli.py index 011357a69f..b9f80bae10 100644 --- a/families/qwen3_8/edge_llm/cli.py +++ b/families/qwen3_8/edge_llm/cli.py @@ -3,40 +3,27 @@ """Qwen38-owned CLI options for explicit paired Edge execution.""" -import argparse from pathlib import Path -from .config import BuildExecutionInputs, NamedCheckpoint, with_execution +from .config import BuildExecutionInputs, NamedCheckpoint -def add_build_arguments(parser: argparse.ArgumentParser) -> None: - """Register options only when the Qwen38 family has been resolved.""" - parser.add_argument("--execution-variant", choices=("dspark",), - help="Explicit Qwen38 paired execution mode") - parser.add_argument("--companion", action="append", default=[], metavar="ROLE=LOCAL_DIR", - help="Explicit local draft checkpoint; exactly one draft role is required") - - -def _execution_inputs(args: argparse.Namespace) -> BuildExecutionInputs | None: +def execution_inputs( + execution_variant: str | None, companion: list[str] | tuple[str, ...] = (), +) -> BuildExecutionInputs | None: """Parse only explicit local inputs; no variant list or model acquisition.""" - if args.command != "build": - return None - if args.execution_variant is None: - if args.companion: + if execution_variant is None: + if companion: raise ValueError("--companion requires --execution-variant") return None + if execution_variant not in ['dspark']: + raise ValueError("unsupported qwen3_8 execution variant") checkpoints = [] - for value in args.companion: + for value in companion: role, separator, directory = value.partition("=") if not separator or not role or not directory: raise ValueError("--companion must be ROLE=LOCAL_DIR") if "://" in directory: raise ValueError("--companion requires a local directory, not a URI") checkpoints.append(NamedCheckpoint(role, Path(directory))) - return BuildExecutionInputs(args.execution_variant, tuple(checkpoints)) - - -def prepare_build_request(request, args): - """Attach a validated family-owned recipe before importing the GPU builder.""" - execution = _execution_inputs(args) - return request if execution is None else with_execution(request, execution) + return BuildExecutionInputs(execution_variant, tuple(checkpoints)) diff --git a/families/qwen3_8/edge_llm/config.py b/families/qwen3_8/edge_llm/config.py index 9baf43d3b4..44a4ba17da 100644 --- a/families/qwen3_8/edge_llm/config.py +++ b/families/qwen3_8/edge_llm/config.py @@ -7,7 +7,7 @@ from pathlib import Path import re -from tensorrt_model_connect.build import BuildRequest +from ..build_request import BuildRequest, coerce_request _ID = re.compile(r"[a-z][a-z0-9_]*\Z") @@ -81,7 +81,8 @@ def __post_init__(self) -> None: def with_execution(request: BuildRequest, execution: BuildExecutionInputs) -> Qwen38BuildRequest: - """Preserve ordinary request fields and callback identity.""" + """Preserve supported ordinary request fields and callback identity.""" + request = coerce_request(request) return Qwen38BuildRequest( **{field.name: getattr(request, field.name) for field in fields(BuildRequest)}, execution=execution, diff --git a/families/qwen3_8/model.py b/families/qwen3_8/model.py index f551097db1..1c7bb2558c 100644 --- a/families/qwen3_8/model.py +++ b/families/qwen3_8/model.py @@ -82,6 +82,9 @@ def _runtime_config(model_dir: Path, config: ModelConfig, model: Qwen38Model, ** def build(request, writer) -> None: """Build one Qwen3.8 hybrid text-generation bundle.""" + from .build_request import coerce_request + + request = coerce_request(request) from .edge_llm.config import Qwen38BuildRequest from .edge_llm.dispatch import build_paired diff --git a/families/qwen3_8/support.py b/families/qwen3_8/support.py index ef68a7bb72..a38cce3a96 100644 --- a/families/qwen3_8/support.py +++ b/families/qwen3_8/support.py @@ -10,12 +10,10 @@ model_types=("qwen38", "qwen3.8", "qwen3_8"), tasks=("text_generation",), default_task="text_generation", - build_cli_module="edge_llm.cli", default_precision="bf16", ) _SUPPORT = FamilySupport( tasks=("text_generation",), default_task="text_generation", default_precision="bf16", - build_cli_module="edge_llm.cli", ) diff --git a/families/qwen3_8/tests/test_support.py b/families/qwen3_8/tests/test_support.py index 331f63609b..b1c98b8b70 100644 --- a/families/qwen3_8/tests/test_support.py +++ b/families/qwen3_8/tests/test_support.py @@ -18,7 +18,8 @@ def test_qwen38_marker_default_task_is_checkpoint_owned() -> None: ) assert family == "qwen3_8" assert support.default_task == "text_generation" - assert support.build_cli_module == "edge_llm.cli" + from tensorrt_model_connect.family_cli import load_family_cli + assert load_family_cli("qwen3_8")["commands"][0]["handler"] == "cli:build" def _edge_cli_source(tmp_path): @@ -35,7 +36,7 @@ def _edge_cli_source(tmp_path): def test_edge_cli_uses_ordinary_family_build(tmp_path, monkeypatch): - from tensorrt_model_connect import build_cli + from tensorrt_model_connect import family_cli as build_cli from families.qwen3_8.edge_llm import dispatch from families.qwen3_8.edge_llm.config import Qwen38BuildRequest @@ -53,7 +54,7 @@ def paired(request, writer, execution): writer.add_json("edge-test.json", {"variant": execution.variant}) monkeypatch.setattr(dispatch, "build_paired", paired) - assert build_cli.main([ + assert build_cli.main(["qwen3_8", "build", str(source), "--precision", "fp16", "-o", str(output), "--execution-variant", "dspark", "--companion", f"draft={draft}", ]) == 0 @@ -70,22 +71,22 @@ def paired(request, writer, execution): ]) def test_bad_edge_cli_inputs_fail_before_backend(tmp_path, monkeypatch, options): import importlib - from tensorrt_model_connect import build_cli + from tensorrt_model_connect import family_cli as build_cli core = importlib.import_module("tensorrt_model_connect.build") source, _ = _edge_cli_source(tmp_path) monkeypatch.setattr(core, "_select_backend", lambda *_: pytest.fail("backend touched")) monkeypatch.setattr(core, "BundleWriter", lambda *_: pytest.fail("writer created")) with pytest.raises(ValueError): - build_cli.main(["build", str(source), "-o", str(tmp_path / "out"), *options]) + build_cli.main(["qwen3_8", "build", str(source), "-o", str(tmp_path / "out"), *options]) def test_edge_cli_help_is_family_owned(tmp_path, capsys): - from tensorrt_model_connect import build_cli + from tensorrt_model_connect import family_cli as build_cli source, _ = _edge_cli_source(tmp_path) with pytest.raises(SystemExit) as caught: - build_cli.main(["build", str(source), "--help"]) + build_cli.main(["qwen3_8", "build", str(source), "--help"]) assert caught.value.code == 0 help_text = capsys.readouterr().out assert "--execution-variant {dspark}" in help_text @@ -93,7 +94,6 @@ def test_edge_cli_help_is_family_owned(tmp_path, capsys): def test_edge_request_preserves_fields_and_family_owner(tmp_path): - import argparse from families.qwen3_8.edge_llm import cli from dataclasses import fields, replace from tensorrt_model_connect.build import BuildRequest @@ -105,8 +105,7 @@ def test_edge_request_preserves_fields_and_family_owner(tmp_path): request = BuildRequest(source, tmp_path / "out", "qwen3_8", "text_generation", "fp16", graph_transform=lambda layer: layer) execution = BuildExecutionInputs("dspark", (NamedCheckpoint("draft", draft),)) - args = argparse.Namespace(command="build", execution_variant=None, companion=[]) - assert cli.prepare_build_request(request, args) is request + assert cli.execution_inputs(None) is None extended = with_execution(request, execution) for field in fields(BuildRequest): assert getattr(extended, field.name) is getattr(request, field.name) @@ -118,7 +117,7 @@ def test_edge_request_preserves_fields_and_family_owner(tmp_path): @pytest.mark.parametrize("failure", [RuntimeError("paired build failed"), KeyboardInterrupt()]) def test_edge_cli_failure_preserves_existing_bundle(tmp_path, monkeypatch, failure): - from tensorrt_model_connect import build_cli + from tensorrt_model_connect import family_cli as build_cli from families.qwen3_8.edge_llm import dispatch source, draft = _edge_cli_source(tmp_path) @@ -132,7 +131,7 @@ def fail(request, writer, execution): monkeypatch.setattr(dispatch, "build_paired", fail) with pytest.raises(type(failure)) as caught: - build_cli.main([ + build_cli.main(["qwen3_8", "build", str(source), "-o", str(output), "--execution-variant", "dspark", "--companion", f"draft={draft}", ]) @@ -161,7 +160,9 @@ def test_qwen38_aliases_register_the_same_family_cli(model_type): from families.qwen3_8.support import describe support = describe(ModelMetadata({"model_type": model_type}, {})) - assert support.build_cli_module == "edge_llm.cli" + assert support is not None + from tensorrt_model_connect.family_cli import load_family_cli + assert load_family_cli("qwen3_8")["commands"][0]["handler"] == "cli:build" def _dspark_pair_request(tmp_path): @@ -258,3 +259,61 @@ def test_dspark_unqualified_platform_names_required_route(tmp_path, monkeypatch, with pytest.raises(NotImplementedError, match="Linux x86_64, SM120 and FP16"): dispatch.build_paired(request, None, execution) assert not list(tmp_path.glob(".out.edge-*")) + + +@pytest.mark.parametrize("options", [[], ["--precision", "fp16", "--max-sequence-length", "64"]]) +def test_declared_build_matches_legacy_request(tmp_path, monkeypatch, options): + """Owner command preserves ordinary request defaults and explicit controls.""" + import json + from families.qwen3_8 import cli as owner + from tensorrt_model_connect import build_cli, family_cli + + source = tmp_path / "checkpoint" + source.mkdir() + (source / "config.json").write_text(json.dumps({"model_type": "qwen3_8"})) + output = tmp_path / "model.bundle" + captured = [] + monkeypatch.setattr(owner, "build_bundle", lambda request, output: captured.append(request)) + monkeypatch.setattr(build_cli, "build", captured.append) + args = [str(source), "-o", str(output), *options] + assert family_cli.main(["qwen3_8", "build", *args]) == 0 + assert build_cli.main(["build", *args, "--family", "qwen3_8"]) == 0 + assert len(captured) == 2 + from dataclasses import fields + assert isinstance(captured[0], owner.BuildRequest) + for field in fields(captured[1]): + assert getattr(captured[0], field.name) == getattr(captured[1], field.name) + from dataclasses import replace + from families.qwen3_8.build_request import coerce_request + assert coerce_request(captured[1]) == captured[0] + with pytest.raises(NotImplementedError, match="image_height"): + coerce_request(replace(captured[1], image_height=32)) + from types import SimpleNamespace + with pytest.raises(ValueError, match="unknown"): + coerce_request(SimpleNamespace(**vars(captured[1]), unexpected_option=True)) + assert captured[0].family == "qwen3_8" + assert captured[0].task == "text_generation" + assert captured[0].precision == ("fp16" if options else "bf16") + assert not output.exists() + + +def test_declared_help_is_offline_and_dependency_free(): + """Actual child-process help needs neither a checkpoint nor GPU imports.""" + import subprocess + import sys + + code = """ +import sys +from tensorrt_model_connect.family_cli import main +try: + main(["qwen3_8", "build", "--help"]) +except SystemExit as error: + assert error.code == 0 +else: + raise AssertionError("help did not exit") +assert "families.qwen3_8.cli" not in sys.modules +assert "tensorrt" not in sys.modules +assert "huggingface_hub" not in sys.modules +""" + result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True) + assert "trtmc qwen3_8 build" in result.stdout diff --git a/website/docs/features/model-families.md b/website/docs/features/model-families.md index 39f6587801..2ce1c238d7 100644 --- a/website/docs/features/model-families.md +++ b/website/docs/features/model-families.md @@ -90,6 +90,11 @@ and do not imply that CI executes the paired cases. ### Qwen3.8 paired ONNX execution +Use `trtmc qwen3_8 build MODEL -o model.bundle` with the owning +family\u0027s options. `trtmc qwen3_8 build --help` works offline without +a checkpoint or GPU imports. This uses the existing +[family CLI protocol](../extend/family-cli.md), not an extension to the shared parser. + The Qwen3.8 family owns explicit mixed-NVFP4 target plus DSpark block7 execution through the optional pinned native Edge-LLM SDK. The qualified profile uses `RadixArk/Qwen3.8-27B-NVFP4` and `RadixArk/Qwen3.8-27B-DSpark`, text-only From 5b91f57730e4cb1c7ccca1cbf4325d62e0552a4e Mon Sep 17 00:00:00 2001 From: Joshua Calafato Date: Wed, 23 Sep 2026 21:57:44 +0000 Subject: [PATCH 08/11] fix(qwen3_8): address Edge review findings Signed-off-by: Joshua Calafato --- families/qwen3_8/edge_llm/README.md | 6 +++++- families/qwen3_8/edge_llm/dispatch.py | 3 +++ website/docs/features/model-families.md | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/families/qwen3_8/edge_llm/README.md b/families/qwen3_8/edge_llm/README.md index 3fffd82d6c..88ea5bb957 100644 --- a/families/qwen3_8/edge_llm/README.md +++ b/families/qwen3_8/edge_llm/README.md @@ -96,6 +96,10 @@ profile explicitly requires FP16. Temporary staging uses the output filesystem. ## Declared build command -This family uses the existing cli.json protocol introduced in #1310. The family\nowns its declaration, typed inputs and Python handler. The handler adapts those\ninputs to the unchanged builder API, preserving native/Edge dispatch and bundle\npublication. The legacy flat build command remains available for its existing\nordinary options; new family options use `trtmc qwen3_8 build`. +This family uses the existing cli.json protocol introduced in #1310. The family +owns its declaration, typed inputs and Python handler. The handler adapts those +inputs to the unchanged builder API, preserving native/Edge dispatch and bundle +publication. The legacy flat build command remains available for its existing +ordinary options; new family options use `trtmc qwen3_8 build`. Help is offline and does not need a local checkpoint. No shared parser hook or family registry entry is added. diff --git a/families/qwen3_8/edge_llm/dispatch.py b/families/qwen3_8/edge_llm/dispatch.py index aa3e2ed492..ba744a3e16 100644 --- a/families/qwen3_8/edge_llm/dispatch.py +++ b/families/qwen3_8/edge_llm/dispatch.py @@ -95,6 +95,9 @@ def build(request, writer, native, *, draft_dir: Path) -> None: traceback.print_exception(error, file=log) _LOG.warning("qwen3_8 Edge build failed: %s. Diagnostics: %s. " "Retrying native once with the unchanged request.", error, log_path, exc_info=True) + except BaseException: + log_path.unlink(missing_ok=True) + raise else: # Edge preparation did not touch writer; publication cannot fallback. if adapter is not None: diff --git a/website/docs/features/model-families.md b/website/docs/features/model-families.md index 2ce1c238d7..ae49254cc2 100644 --- a/website/docs/features/model-families.md +++ b/website/docs/features/model-families.md @@ -91,7 +91,7 @@ and do not imply that CI executes the paired cases. ### Qwen3.8 paired ONNX execution Use `trtmc qwen3_8 build MODEL -o model.bundle` with the owning -family\u0027s options. `trtmc qwen3_8 build --help` works offline without +family's options. `trtmc qwen3_8 build --help` works offline without a checkpoint or GPU imports. This uses the existing [family CLI protocol](../extend/family-cli.md), not an extension to the shared parser. From bd2d2474aeb1b1fae8d5b788249db31265652903 Mon Sep 17 00:00:00 2001 From: Joshua Calafato Date: Thu, 24 Sep 2026 06:05:18 +0000 Subject: [PATCH 09/11] fix(tests): honor offline checkpoint loading Pass the Hub offline setting explicitly to snapshot_download. Pinned revisions can otherwise request uncached tree metadata in Hub 1.32 even when the checkpoint was staged before entering the offline runner. Keep network isolation, pinned revisions, and numerical quality gates unchanged. This repairs existing smoke tests, not model support scope. Signed-off-by: Joshua Calafato --- families/qwen3_8/tests/test_e2e.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/families/qwen3_8/tests/test_e2e.py b/families/qwen3_8/tests/test_e2e.py index 5e284af876..a146e79ebb 100644 --- a/families/qwen3_8/tests/test_e2e.py +++ b/families/qwen3_8/tests/test_e2e.py @@ -98,12 +98,13 @@ def _required_environment(tp_size: int): def _checkpoint(manifest: dict) -> Path: - from huggingface_hub import snapshot_download + from huggingface_hub import constants, snapshot_download path = Path( snapshot_download( repo_id=manifest["hf_id"], revision=manifest.get("hf_revision"), + local_files_only=constants.HF_HUB_OFFLINE, ) ) assert (path / "config.json").is_file(), path From 2380542ee7078847bd933b21c264efbb6d2fcc02 Mon Sep 17 00:00:00 2001 From: Joshua Calafato Date: Thu, 1 Oct 2026 01:48:37 +0000 Subject: [PATCH 10/11] fix(qwen3_8): use family build API in E2E The existing E2E helper still passed execution to the removed shared build API, failing both native and paired test construction before inference. Route it through the declared family build handler and extend the existing native and paired CLI regressions to exercise that helper. Model quality gates are unchanged. Signed-off-by: Joshua Calafato --- families/qwen3_8/tests/test_e2e.py | 26 ++++++++++++------------- families/qwen3_8/tests/test_support.py | 27 ++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/families/qwen3_8/tests/test_e2e.py b/families/qwen3_8/tests/test_e2e.py index a146e79ebb..7ea4915010 100644 --- a/families/qwen3_8/tests/test_e2e.py +++ b/families/qwen3_8/tests/test_e2e.py @@ -17,7 +17,7 @@ import pytest -from tensorrt_model_connect import BuildRequest, build +from families.qwen3_8.cli import build _TEST_DIR = Path(__file__).resolve().parent @@ -139,18 +139,18 @@ def _build_bundle(manifest: dict, model_dir: Path, bundle: Path, *, execution=No assert quantization is None or isinstance(quantization, str) fp32_layers = tuple(manifest.get("fp32_layers", ())) build( - BuildRequest( - model_dir=model_dir, - output_path=bundle, - family=_FAMILY, - task="text_generation", - precision=manifest["precision"], - max_sequence_length=manifest["max_sequence_length"], - tensor_parallel_size=manifest["tensor_parallel_size"], - quantization=quantization, - fp32_layers=fp32_layers, - ), - execution=execution, + model=str(model_dir), + output=bundle, + task="text_generation", + precision=manifest["precision"], + max_sequence_length=manifest["max_sequence_length"], + tensor_parallel_size=manifest["tensor_parallel_size"], + quantization=quantization, + fp32_layers=fp32_layers, + execution_variant=execution.variant if execution is not None else None, + companion=tuple( + f"{checkpoint.role}={checkpoint.model_dir}" for checkpoint in execution.checkpoints + ) if execution is not None else (), ) assert bundle.is_file() and bundle.stat().st_size > 0, bundle diff --git a/families/qwen3_8/tests/test_support.py b/families/qwen3_8/tests/test_support.py index b1c98b8b70..656e2d8b7d 100644 --- a/families/qwen3_8/tests/test_support.py +++ b/families/qwen3_8/tests/test_support.py @@ -61,6 +61,18 @@ def paired(request, writer, execution): assert len(seen) == 1 assert output.is_file() + from families.qwen3_8.tests.test_e2e import _build_bundle + from families.qwen3_8.edge_llm.config import BuildExecutionInputs, NamedCheckpoint + + _build_bundle( + {"precision": "fp16", "max_sequence_length": 64, "tensor_parallel_size": 1}, + source, output, + execution=BuildExecutionInputs("dspark", (NamedCheckpoint("draft", draft),)), + ) + assert len(seen) == 2 + assert seen[-1].max_sequence_length == 64 + assert output.is_file() + @pytest.mark.parametrize("options", [ ["--companion", "draft=/missing"], @@ -296,6 +308,21 @@ def test_declared_build_matches_legacy_request(tmp_path, monkeypatch, options): assert captured[0].precision == ("fp16" if options else "bf16") assert not output.exists() + from families.qwen3_8.tests.test_e2e import _build_bundle + + def capture_bundle(request, destination): + captured.append(request) + destination.write_bytes(b"test bundle") + + monkeypatch.setattr(owner, "build_bundle", capture_bundle) + _build_bundle( + {"precision": captured[0].precision, + "max_sequence_length": captured[0].max_sequence_length, + "tensor_parallel_size": captured[0].tensor_parallel_size}, + source, output, + ) + assert captured[-1] == captured[0] + def test_declared_help_is_offline_and_dependency_free(): """Actual child-process help needs neither a checkpoint nor GPU imports.""" From 0fe374e92fc19379c734df5eb0a719a025170687 Mon Sep 17 00:00:00 2001 From: Joshua Calafato Date: Fri, 2 Oct 2026 17:03:25 +0000 Subject: [PATCH 11/11] fix(qwen3_8): align paired defaults and request lifetime Use FP16 only when paired CLI precision is omitted while preserving ordinary BF16 and explicit controls. Drain request work before returning from Edge and exercise family-owned bindings in the existing regressions. Signed-off-by: Joshua Calafato --- families/qwen3_8/cli.json | 2 +- families/qwen3_8/cli.py | 4 +++- families/qwen3_8/edge_llm/README.md | 6 ++++-- families/qwen3_8/runtime/edge_llm/adapter.cpp | 6 ++++++ families/qwen3_8/tests/test_support.py | 19 +++++++++++-------- 5 files changed, 25 insertions(+), 12 deletions(-) diff --git a/families/qwen3_8/cli.json b/families/qwen3_8/cli.json index 01fa56b9f7..77f79702fc 100644 --- a/families/qwen3_8/cli.json +++ b/families/qwen3_8/cli.json @@ -50,7 +50,7 @@ "fp16", "bf16" ], - "default": "bf16" + "help": "Compute precision (default: fp16 for paired execution, bf16 otherwise)" }, { "name": "backend", diff --git a/families/qwen3_8/cli.py b/families/qwen3_8/cli.py index a0199d3703..db920cd573 100644 --- a/families/qwen3_8/cli.py +++ b/families/qwen3_8/cli.py @@ -36,7 +36,7 @@ def build_bundle(request: BuildRequest, output: Path) -> None: def build( *, model: str, output: Path, revision: str | None = None, - task: str = "text_generation", precision: str = "bf16", backend: str = "trt", + task: str = "text_generation", precision: str | None = None, backend: str = "trt", max_sequence_length: int | None = None, tensor_parallel_size: int = 1, verbose: bool = False, fp32_layers: list[int] | tuple[int, ...] = (), @@ -45,6 +45,8 @@ def build( ) -> int: """Run the declared owner command; help never imports this handler.""" execution = execution_inputs(execution_variant, companion) + if precision is None: + precision = "fp16" if execution is not None else "bf16" model_dir = resolve_model(model, revision) resolve_family(load_model_metadata(model_dir), "qwen3_8") request = BuildRequest( diff --git a/families/qwen3_8/edge_llm/README.md b/families/qwen3_8/edge_llm/README.md index 88ea5bb957..ec20c14e81 100644 --- a/families/qwen3_8/edge_llm/README.md +++ b/families/qwen3_8/edge_llm/README.md @@ -91,8 +91,10 @@ A failed explicit pair is never replaced by a base-only bundle. Previously recorded full-model results above are historical, not fresh refactor-head E2Es. Request controls and the existing 9–1024 capacity range are checked before any -Edge preparation. The native precision default is not changed: this paired -profile explicitly requires FP16. Temporary staging uses the output filesystem. +Edge preparation. When CLI precision is omitted, paired execution defaults to +FP16 while ordinary native builds retain BF16. Explicit precision values are +unchanged; this paired profile accepts only FP16. Temporary staging uses the +output filesystem. ## Declared build command diff --git a/families/qwen3_8/runtime/edge_llm/adapter.cpp b/families/qwen3_8/runtime/edge_llm/adapter.cpp index a23687fdb2..4c294c1008 100644 --- a/families/qwen3_8/runtime/edge_llm/adapter.cpp +++ b/families/qwen3_8/runtime/edge_llm/adapter.cpp @@ -221,6 +221,12 @@ class EdgeTask final : public ITextGeneration { throw std::runtime_error("Qwen3.8 Edge returned invalid prompt counts"); validate_capacity(counts.front(), input_limit_, capacity_, request.maxGenerateLength); trt_edgellm::rt::LLMGenerationResponse response{}; + // Complete queued work before response/request storage is destroyed, + // including exception paths in a persistent task. + struct Drain { + cudaStream_t stream; + ~Drain() { cudaStreamSynchronize(stream); } + } drain{stream_.get()}; if (!runtime_->handleRequest(request, response, stream_.get()) || response.outputIds.size() != 1 || response.outputTexts.size() != 1 || response.outputIds.front().empty() || diff --git a/families/qwen3_8/tests/test_support.py b/families/qwen3_8/tests/test_support.py index 656e2d8b7d..619197e79a 100644 --- a/families/qwen3_8/tests/test_support.py +++ b/families/qwen3_8/tests/test_support.py @@ -35,7 +35,8 @@ def _edge_cli_source(tmp_path): return source, draft -def test_edge_cli_uses_ordinary_family_build(tmp_path, monkeypatch): +@pytest.mark.parametrize("precision", [None, "fp16", "fp32", "bf16"]) +def test_edge_cli_uses_ordinary_family_build(tmp_path, monkeypatch, precision): from tensorrt_model_connect import family_cli as build_cli from families.qwen3_8.edge_llm import dispatch from families.qwen3_8.edge_llm.config import Qwen38BuildRequest @@ -54,11 +55,13 @@ def paired(request, writer, execution): writer.add_json("edge-test.json", {"variant": execution.variant}) monkeypatch.setattr(dispatch, "build_paired", paired) + options = ["--precision", precision] if precision else [] assert build_cli.main(["qwen3_8", - "build", str(source), "--precision", "fp16", "-o", str(output), + "build", str(source), *options, "-o", str(output), "--execution-variant", "dspark", "--companion", f"draft={draft}", ]) == 0 assert len(seen) == 1 + assert seen[0].precision == (precision or "fp16") assert output.is_file() from families.qwen3_8.tests.test_e2e import _build_bundle @@ -70,6 +73,7 @@ def paired(request, writer, execution): execution=BuildExecutionInputs("dspark", (NamedCheckpoint("draft", draft),)), ) assert len(seen) == 2 + assert seen[-1].precision == "fp16" assert seen[-1].max_sequence_length == 64 assert output.is_file() @@ -82,13 +86,11 @@ def paired(request, writer, execution): ["--execution-variant", "dspark", "--companion", "draft=https://example.com/model"], ]) def test_bad_edge_cli_inputs_fail_before_backend(tmp_path, monkeypatch, options): - import importlib from tensorrt_model_connect import family_cli as build_cli - - core = importlib.import_module("tensorrt_model_connect.build") + from families.qwen3_8 import cli as owner source, _ = _edge_cli_source(tmp_path) - monkeypatch.setattr(core, "_select_backend", lambda *_: pytest.fail("backend touched")) - monkeypatch.setattr(core, "BundleWriter", lambda *_: pytest.fail("writer created")) + monkeypatch.setattr(owner, "select_backend", lambda *_: pytest.fail("backend touched")) + monkeypatch.setattr(owner, "BundleWriter", lambda *_: pytest.fail("writer created")) with pytest.raises(ValueError): build_cli.main(["qwen3_8", "build", str(source), "-o", str(tmp_path / "out"), *options]) @@ -108,7 +110,7 @@ def test_edge_cli_help_is_family_owned(tmp_path, capsys): def test_edge_request_preserves_fields_and_family_owner(tmp_path): from families.qwen3_8.edge_llm import cli from dataclasses import fields, replace - from tensorrt_model_connect.build import BuildRequest + from families.qwen3_8.build_request import BuildRequest, coerce_request from families.qwen3_8.edge_llm.config import ( BuildExecutionInputs, NamedCheckpoint, with_execution, ) @@ -116,6 +118,7 @@ def test_edge_request_preserves_fields_and_family_owner(tmp_path): source, draft = _edge_cli_source(tmp_path) request = BuildRequest(source, tmp_path / "out", "qwen3_8", "text_generation", "fp16", graph_transform=lambda layer: layer) + assert coerce_request(request) is request execution = BuildExecutionInputs("dspark", (NamedCheckpoint("draft", draft),)) assert cli.execution_inputs(None) is None extended = with_execution(request, execution)