diff --git a/tests/composer/test_chat_template.py b/tests/composer/test_chat_template.py index 8aededf..9ffe599 100644 --- a/tests/composer/test_chat_template.py +++ b/tests/composer/test_chat_template.py @@ -27,9 +27,13 @@ from types import SimpleNamespace from unittest.mock import patch +import pytest from jinja2 import Environment -from granite_switch.composer.tokenizer_setup import configure_chat_template +from granite_switch.composer.tokenizer_setup import ( + configure_audio_chat_template, + configure_chat_template, +) _PATCH_TARGET = "granite_switch.composer.tokenizer_setup._decode_alora_invocation_text" @@ -531,3 +535,102 @@ def test_mixed_adapters_from_adapter_config(self): assert "<|answerability|>" not in result_none assert "<|context_relevance|>" not in result_none assert "<|summarization|>" not in result_none + + +# ════════════════════════════════════════════════════════════════════ +# Audio: <|audio|> marker preservation, and coexistence with adapters +# ════════════════════════════════════════════════════════════════════ + + +def _audio_messages(text="transcribe this", audio_type="audio"): + """A user turn whose content is a parts list carrying an audio clip.""" + return [ + { + "role": "user", + "content": [ + {"type": "text", "text": text}, + {"type": audio_type, audio_type: "clip-placeholder"}, + ], + } + ] + + +class TestAudioChatTemplate: + """configure_audio_chat_template emits the <|audio|> marker for audio parts.""" + + def test_audio_part_emits_marker(self): + tokenizer = _make_tokenizer() + configure_audio_chat_template(tokenizer) + result = _render( + tokenizer, messages=_audio_messages(), add_generation_prompt=True + ) + assert "<|audio|>" in result + assert "transcribe this" in result + + def test_marker_dropped_without_injection(self): + # Regression canary: the un-injected base template drops the audio part. + tokenizer = _make_tokenizer() + result = _render( + tokenizer, messages=_audio_messages(), add_generation_prompt=True + ) + assert "<|audio|>" not in result + assert "transcribe this" in result + + def test_input_audio_and_audio_url_types_emit_marker(self): + for audio_type in ("input_audio", "audio_url"): + tokenizer = _make_tokenizer() + configure_audio_chat_template(tokenizer) + result = _render( + tokenizer, + messages=_audio_messages(audio_type=audio_type), + add_generation_prompt=True, + ) + assert "<|audio|>" in result, f"marker missing for type={audio_type!r}" + + def test_custom_marker_string(self): + tokenizer = _make_tokenizer() + configure_audio_chat_template(tokenizer, marker="<|snd|>") + result = _render( + tokenizer, messages=_audio_messages(), add_generation_prompt=True + ) + assert "<|snd|>" in result + + def test_missing_anchor_raises(self): + tokenizer = SimpleNamespace(chat_template="{{ messages }}") + with pytest.raises(ValueError, match="content-part loop"): + configure_audio_chat_template(tokenizer) + + def test_none_template_is_noop(self): + tokenizer = SimpleNamespace(chat_template=None) + configure_audio_chat_template(tokenizer) + assert tokenizer.chat_template is None + + +class TestAudioAndAdapterInjectionsCompose: + """Adapter and audio injections applied in sequence leave both intact.""" + + def test_lora_prefix_and_audio_marker_coexist(self): + tokenizer = _make_tokenizer() + configure_chat_template(tokenizer, [("/path/a", "ctx_rel", "lora")]) + configure_audio_chat_template(tokenizer) + + result = _render( + tokenizer, + messages=_audio_messages(), + add_generation_prompt=True, + adapter_name="ctx_rel", + ) + assert result.startswith("<|ctx_rel|>"), result[:80] + assert "<|audio|>" in result + assert "transcribe this" in result + + def test_no_adapter_still_emits_audio_marker(self): + tokenizer = _make_tokenizer() + configure_chat_template(tokenizer, [("/path/a", "ctx_rel", "lora")]) + configure_audio_chat_template(tokenizer) + + result = _render( + tokenizer, messages=_audio_messages(), add_generation_prompt=True + ) + assert "<|audio|>" in result + assert "<|ctx_rel|>" not in result diff --git a/tests/integration/fixtures/eiffel_tower_paris.wav b/tests/integration/fixtures/eiffel_tower_paris.wav new file mode 100644 index 0000000..d88341d Binary files /dev/null and b/tests/integration/fixtures/eiffel_tower_paris.wav differ diff --git a/tests/integration/test_adapter_routing_audio_enabled.py b/tests/integration/test_adapter_routing_audio_enabled.py new file mode 100644 index 0000000..131a01f --- /dev/null +++ b/tests/integration/test_adapter_routing_audio_enabled.py @@ -0,0 +1,166 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Adapters + audio (issue #47): every default adapter routes to its own index +on an audio-enabled checkpoint. + +Composes the default adapter set (RAG + Core + Guardian) with --enable-audio and +sweeps every adapter control token that resolved, asserting the switch maps +``adapter_token_ids[i]`` to index ``i + 1`` and leaves pre-control positions on +the base (``0``) — i.e. enabling audio does not perturb the token->index map. +For granite-4.1-3b that is 12 adapters: context_relevance ships no 4.1-3b flavor, +so 12 of the 13 defined adapters resolve. Complements the single-adapter sweep in +test_switch_e2e_compose and the serve-time answerability check in +test_answerability_over_audio. + +Markers: slow + requires_model + gpu (opt-in via -m). +""" + +import importlib.util +import json +import os + +import pytest + +pytestmark = [pytest.mark.slow, pytest.mark.requires_model, pytest.mark.gpu] + +if importlib.util.find_spec("granite_switch.hf") is None: + pytest.skip("requires the HF backend ([hf] extra)", allow_module_level=True) + + +# The default adapter set is the union of the three granitelib libraries (the +# "12 adapters"): RAG + Core + Guardian. +_DEFAULT_ADAPTER_LIBRARIES = [ + "ibm-granite/granitelib-rag-r1.0", + "ibm-granite/granitelib-core-r1.0", + "ibm-granite/granitelib-guardian-r1.0", +] +_DEFAULT_BASE_MODEL_PAIRS = [ + ("ibm-granite/granite-4.1-3b", _DEFAULT_ADAPTER_LIBRARIES), +] + + +def _load_experimental_pairs(): + raw = os.environ.get("GRANITE_SWITCH_EXPERIMENTAL_MODEL_PAIRS", "") + if not raw: + return [] + try: + entries = json.loads(raw) + except json.JSONDecodeError as e: + raise ValueError( + f"GRANITE_SWITCH_EXPERIMENTAL_MODEL_PAIRS is not valid JSON: {e}\n" + f'Expected format: \'[{{"base":"/path","adapter":"/path"}}, ...]\'' + ) + # adapter may be a single library or a list of libraries. + pairs = [] + for p in entries: + adapters = p["adapter"] + pairs.append( + (p["base"], adapters if isinstance(adapters, list) else [adapters]) + ) + return pairs + + +BASE_MODEL_PAIRS = _DEFAULT_BASE_MODEL_PAIRS + _load_experimental_pairs() + +COMPOSE_TIMEOUT_S = 1800 +_SEQ_LEN = 8 +_CTRL_POS = 1 +_FILLER_TOKENS = [791, 5679, 2766, 279, 893, 389, 813, 1450] + + +@pytest.fixture( + scope="module", + params=BASE_MODEL_PAIRS, + ids=lambda p: p[0].rsplit("/", 1)[-1], +) +def audio_switch_model(request, tmp_path_factory): + import subprocess + import sys + + import torch + + from granite_switch.hf import GraniteSwitchForCausalLM + + base_model, adapter_libraries = request.param + save_dir = tmp_path_factory.mktemp(base_model.rsplit("/", 1)[-1]) / "model" + + cmd = [ + sys.executable, + "-m", + "granite_switch.composer.compose_granite_switch", + "--base-model", + base_model, + "--adapters", + *adapter_libraries, + "--enable-audio", + "--output", + str(save_dir), + ] + result = subprocess.run( + cmd, capture_output=True, text=True, timeout=COMPOSE_TIMEOUT_S + ) + if result.returncode != 0: + raise RuntimeError( + f"compose failed for base={base_model} adapters={adapter_libraries}\n" + f"--- STDOUT ---\n{result.stdout}\n--- STDERR ---\n{result.stderr}" + ) + + # ignore_mismatched_sizes: the <|audio|> token bumps vocab_size one past the + # switch's control_to_substitute_lut buffer, but that buffer is rebuilt from + # config in SingleSwitch.__init__, so the freshly-built one is kept intact. + model = ( + GraniteSwitchForCausalLM.from_pretrained( + str(save_dir), dtype=torch.bfloat16, ignore_mismatched_sizes=True + ) + .eval() + .cuda() + ) + return {"base_model": base_model, "model": model, "config": model.config} + + +def _adapter_indices_after_forward(model, control_token_id): + import torch + + seq = [_FILLER_TOKENS[i % len(_FILLER_TOKENS)] for i in range(_SEQ_LEN)] + seq[_CTRL_POS] = control_token_id + with torch.no_grad(): + model(input_ids=torch.tensor([seq], device="cuda")) + return model.model._last_adapter_indices[0] + + +def test_all_adapters_route_with_audio_enabled(audio_switch_model): + """Each adapter control token routes to its own index on an audio checkpoint.""" + config = audio_switch_model["config"] + base_model = audio_switch_model["base_model"] + + assert getattr(config, "asr_enabled", False) is True, ( + f"checkpoint is not audio-enabled (base_model={base_model})" + ) + + token_ids = list(getattr(config, "adapter_token_ids", None) or []) + names = list(getattr(config, "adapter_names", None) or []) + assert token_ids, f"composed checkpoint has no adapters (base_model={base_model})" + print(f"\n sweeping {len(token_ids)} adapters (base_model={base_model})") + + failures = [] + for i, token_id in enumerate(token_ids): + expected = i + 1 # adapter_token_ids[i] activates adapter i+1; 0 = base + name = names[i] if i < len(names) else f"adapter_{expected}" + ai = _adapter_indices_after_forward(audio_switch_model["model"], token_id) + + ok = bool((ai[:_CTRL_POS] == 0).all()) and bool( + (ai[_CTRL_POS:] == expected).all() + ) + print( + f" [{'ok' if ok else 'FAIL'}] idx {expected:>2} {name!r}: {ai.tolist()}" + ) + if not ok: + failures.append((expected, name, token_id, ai.tolist())) + + assert not failures, ( + f"{len(failures)} adapter(s) mis-routed with audio enabled " + f"(base_model={base_model}); expected pre-control=0, post-control=index:\n" + + "\n".join( + f" idx {idx} {name!r} (token {tok}): {indices}" + for idx, name, tok, indices in failures + ) + ) diff --git a/tests/integration/test_answerability_over_audio.py b/tests/integration/test_answerability_over_audio.py new file mode 100644 index 0000000..e59f11b --- /dev/null +++ b/tests/integration/test_answerability_over_audio.py @@ -0,0 +1,208 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Adapters + audio: the RAG answerability adapter judging a query against a +document delivered as audio (issue #47). + +Covers the "adapters + audio" box. Where test_audio_serving_smoke.py only proves +an adapter control token doesn't crash the audio path, this checks the switch +reaches the *correct* verdict when the RAG document arrives as speech. + +Document-as-audio: ``documents=[{"text": "<|audio|>"}]`` — the Granite template +renders each document with ``doc | tojson``, so the marker lands in the +```` block where the ASR processor splices the transcript. The +``<|answerability|>`` control token is inserted by the same template (aLoRA +fallback path). Ground truth is one committed speech clip (fixtures/, generated +with SpeechT5 — MIT) driving both classes; the questions key on content that +transcribes cleanly, so this tests the decision, not transcription accuracy +(WER). + +Markers: slow + requires_model + gpu (opt-in via -m). +""" + +import importlib.util +import json +import os +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.slow, pytest.mark.requires_model, pytest.mark.gpu] + +if importlib.util.find_spec("vllm") is None: + pytest.skip("requires vLLM installed", allow_module_level=True) + + +_DEFAULT_BASE_MODEL_PAIRS = [ + ("ibm-granite/granite-4.1-3b", "ibm-granite/granitelib-rag-r1.0"), +] + + +def _load_experimental_pairs(): + """Extra (base, adapter) pairs from GRANITE_SWITCH_EXPERIMENTAL_MODEL_PAIRS. + + JSON array of {"base": str, "adapter": str}; mirrors the other E2E files. + """ + raw = os.environ.get("GRANITE_SWITCH_EXPERIMENTAL_MODEL_PAIRS", "") + if not raw: + return [] + try: + entries = json.loads(raw) + except json.JSONDecodeError as e: + raise ValueError( + f"GRANITE_SWITCH_EXPERIMENTAL_MODEL_PAIRS is not valid JSON: {e}\n" + f'Expected format: \'[{{"base":"/path","adapter":"/path"}}, ...]\'' + ) + return [(p["base"], p["adapter"]) for p in entries] + + +BASE_MODEL_PAIRS = _DEFAULT_BASE_MODEL_PAIRS + _load_experimental_pairs() + +COMPOSE_TIMEOUT_S = 1800 # matches the other E2E compose fixtures +_AUDIO_MARKER = "<|audio|>" + +# Committed speech clip; spoken sentence is "The Eiffel Tower is located in the +# city of Paris in France." The location words transcribe cleanly (the proper +# noun does not, hence the questions key on the location, not the name). +_AUDIO_FIXTURE = Path(__file__).parent / "fixtures" / "eiffel_tower_paris.wav" +_ANSWERABILITY_CASES = [ + ("In which city is the tower located?", "answerable"), + ("What is the boiling point of water?", "unanswerable"), +] + + +@pytest.fixture( + scope="module", + params=BASE_MODEL_PAIRS, + ids=lambda p: p[0].rsplit("/", 1)[-1], +) +def audio_rag_checkpoint(request, tmp_path_factory): + """Compose one audio-enabled RAG checkpoint per (base, adapter) pair.""" + import subprocess + import sys + + base_model, adapter_library = request.param + save_dir = tmp_path_factory.mktemp(base_model.rsplit("/", 1)[-1]) / "model" + + cmd = [ + sys.executable, + "-m", + "granite_switch.composer.compose_granite_switch", + "--base-model", + base_model, + "--adapters", + adapter_library, + "--enable-audio", + "--output", + str(save_dir), + ] + result = subprocess.run( + cmd, capture_output=True, text=True, timeout=COMPOSE_TIMEOUT_S + ) + if result.returncode != 0: + raise RuntimeError( + f"compose failed for base={base_model} adapter={adapter_library}\n" + f"--- STDOUT ---\n{result.stdout}\n--- STDERR ---\n{result.stderr}" + ) + return {"base_model": base_model, "save_dir": save_dir} + + +@pytest.fixture(scope="module") +def served(audio_rag_checkpoint): + """Boot vLLM once for the checkpoint and share it across the cases.""" + import gc + + import torch + + os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn") + from vllm import LLM + + llm = LLM( + model=str(audio_rag_checkpoint["save_dir"]), + dtype="bfloat16", + gpu_memory_utilization=0.7, + enforce_eager=True, + ) + try: + yield { + "llm": llm, + "config": llm.llm_engine.model_config.hf_config, + "tokenizer": llm.get_tokenizer(), + } + finally: + del llm + gc.collect() + torch.cuda.empty_cache() + + +def _answerability_adapter_name(config): + # Discovery derives the name from the library layout — match rather than + # hard-code, and skip loudly if this checkpoint has no answerability adapter. + names = list(getattr(config, "adapter_names", None) or []) + for name in names: + if "answerab" in name.lower(): + return name + pytest.skip(f"no answerability adapter in composed checkpoint (adapters={names})") + + +@pytest.fixture(scope="module") +def speech_clip(): + """The committed 16 kHz mono speech clip as (waveform, sample_rate).""" + import soundfile as sf + + waveform, sr = sf.read(_AUDIO_FIXTURE, dtype="float32") + if waveform.ndim > 1: + waveform = waveform.mean(axis=1) + return {"waveform": waveform, "sr": sr} + + +def _build_answerability_prompt(tokenizer, adapter_name, question): + # documents=[{"text": "<|audio|>"}] places one audio marker in the template's + # block; adapter_name arms the answerability control-token insert. + return tokenizer.apply_chat_template( + [{"role": "user", "content": question}], + documents=[{"text": _AUDIO_MARKER}], + add_generation_prompt=True, + adapter_name=adapter_name, + tokenize=False, + ) + + +def _parse_label(text): + # Adapter emits the bare enum ("answerable"/"unanswerable"), maybe quoted. + # Match the longer label first so "answerable" doesn't shadow "unanswerable". + norm = text.strip().strip('"').strip().lower() + if "unanswerable" in norm: + return "unanswerable" + if "answerable" in norm: + return "answerable" + return norm # unrecognized — surfaced by the assertion + + +@pytest.mark.parametrize( + "question,expected", + _ANSWERABILITY_CASES, + ids=[c[1] for c in _ANSWERABILITY_CASES], +) +def test_answerability_over_audio_document(served, speech_clip, question, expected): + """Answerability adapter reaches the correct verdict on an audio document.""" + from vllm import SamplingParams + + adapter_name = _answerability_adapter_name(served["config"]) + prompt = _build_answerability_prompt(served["tokenizer"], adapter_name, question) + + outputs = served["llm"].generate( + { + "prompt": prompt, + "multi_modal_data": { + "audio": [(speech_clip["waveform"], speech_clip["sr"])] + }, + }, + SamplingParams(max_tokens=8, temperature=0.0), + ) + + assert len(outputs) == 1 + raw = outputs[0].outputs[0].text + label = _parse_label(raw) + assert label == expected, ( + f"answerability mismatch for {question!r}: got {label!r} " + f"(raw={raw!r}), expected {expected!r}" + ) diff --git a/tests/integration/test_audio_serving_smoke.py b/tests/integration/test_audio_serving_smoke.py new file mode 100644 index 0000000..b1eb3a6 --- /dev/null +++ b/tests/integration/test_audio_serving_smoke.py @@ -0,0 +1,228 @@ +# SPDX-License-Identifier: Apache-2.0 +"""End-to-end vLLM serving smoke for the audio cascade (issue #47). + +Boots a real composed, audio-enabled GraniteSwitch checkpoint under vLLM and +drives the three request shapes the #47 checklist calls out as the serving +smoke — **text + adapter + audio x1/x2/x3** — through one live engine. This is +the path the low-level tests deliberately bypass (CUDA graphs, torch.compile, +the ASR processor running inside vLLM's EngineCore subprocess, the multi-clip +splice, and generation), so it is the only test that proves those pieces +compose at serve time. + +Scope is a *smoke*: each request must complete and return well-formed output, +and multi-clip requests must be accepted up to the checkpoint's declared clip +ceiling. It intentionally does NOT assert transcript content — transcription +quality (WER) and adapter-routing correctness are separate #47 boxes covered by +the eval harness and `test_switch_e2e_compose.py` respectively. Synthetic audio +keeps the test asset-free; a silent/tonal clip exercises the serving plumbing +without shipping a speech fixture. + +Model construction goes through the compose CLI (CLAUDE.md gotcha #5): no test +hand-assembles a config. Audio is enabled with `--enable-audio`, which defaults +the ASR front-end to the small built-in model (distil-whisper/distil-small.en) +to keep CI cost down. + +Markers: @pytest.mark.slow + @pytest.mark.requires_model + @pytest.mark.gpu. +CI must opt in explicitly: `pytest -m "slow and requires_model and gpu"`. +""" + +import importlib.util +import json +import os + +import pytest + +pytestmark = [pytest.mark.slow, pytest.mark.requires_model, pytest.mark.gpu] + +if importlib.util.find_spec("vllm") is None: + pytest.skip("requires vLLM installed", allow_module_level=True) + + +# ---------------------------------------------------------------------------- +# Base-model / adapter-library pairs — kept in lockstep with +# tests/integration/test_switch_e2e_compose.py so the two E2E files exercise +# the same model matrix. A fast CI profile can pin one pair via -k or the +# experimental env var below. +# ---------------------------------------------------------------------------- + +_DEFAULT_BASE_MODEL_PAIRS = [ + ("ibm-granite/granite-4.0-micro", "ibm-granite/granitelib-core-r1.0"), + ("ibm-granite/granite-4.1-3b", "ibm-granite/granitelib-core-r1.0"), +] + + +def _load_experimental_pairs(): + """Local/experimental pairings from GRANITE_SWITCH_EXPERIMENTAL_MODEL_PAIRS. + + JSON array of {"base": str, "adapter": str}; base/adapter may be HF ids or + local paths. The mechanism is committed; the values are not. Mirrors + test_switch_e2e_compose.py so both E2E files share one extension knob. + """ + raw = os.environ.get("GRANITE_SWITCH_EXPERIMENTAL_MODEL_PAIRS", "") + if not raw: + return [] + try: + entries = json.loads(raw) + except json.JSONDecodeError as e: + raise ValueError( + f"GRANITE_SWITCH_EXPERIMENTAL_MODEL_PAIRS is not valid JSON: {e}\n" + f'Expected format: \'[{{"base":"/path","adapter":"/path"}}, ...]\'' + ) + return [(p["base"], p["adapter"]) for p in entries] + + +BASE_MODEL_PAIRS = _DEFAULT_BASE_MODEL_PAIRS + _load_experimental_pairs() + +COMPOSE_TIMEOUT_S = 1800 # 30 min — matches test_switch_e2e_compose.py +_TARGET_SR = 16_000 + + +@pytest.fixture( + scope="module", + params=BASE_MODEL_PAIRS, + ids=lambda p: p[0].rsplit("/", 1)[-1], +) +def audio_checkpoint(request, tmp_path_factory): + """Compose one audio-enabled checkpoint per (base, adapter) pair. + + Delegates to the compose CLI (same as test_switch_e2e_compose.py) with + `--enable-audio`, then returns the save dir. Module scope amortizes the + download-dominated first run across every smoke case for the pair. + """ + import subprocess + import sys + + base_model, adapter_library = request.param + save_dir = tmp_path_factory.mktemp(base_model.rsplit("/", 1)[-1]) / "model" + + cmd = [ + sys.executable, + "-m", + "granite_switch.composer.compose_granite_switch", + "--base-model", + base_model, + "--adapters", + adapter_library, + "--enable-audio", + "--output", + str(save_dir), + ] + result = subprocess.run( + cmd, capture_output=True, text=True, timeout=COMPOSE_TIMEOUT_S + ) + if result.returncode != 0: + raise RuntimeError( + f"compose failed for base={base_model} adapter={adapter_library}\n" + f"--- STDOUT ---\n{result.stdout}\n--- STDERR ---\n{result.stderr}" + ) + return {"base_model": base_model, "save_dir": save_dir} + + +@pytest.fixture(scope="module") +def served(audio_checkpoint): + """Boot vLLM once for the checkpoint and share it across smoke cases. + + Tokenizer init stays ON (unlike the argmax-equivalence test): the ASR + processor needs a tokenizer to encode both the prompt and the transcript, + and the adapter case needs it to tokenize text around the control token. + """ + import gc + + import torch + + os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn") + from vllm import LLM + + llm = LLM( + model=str(audio_checkpoint["save_dir"]), + dtype="bfloat16", + gpu_memory_utilization=0.7, + enforce_eager=True, # smoke: skip CUDA-graph capture for a faster boot + ) + try: + yield {"llm": llm, "config": llm.llm_engine.model_config.hf_config} + finally: + del llm + gc.collect() + torch.cuda.empty_cache() + + +def _tone(seconds: float = 1.0, freq: float = 440.0): + """Deterministic mono 16 kHz waveform (no speech fixture needed).""" + import numpy as np + + t = np.arange(int(seconds * _TARGET_SR), dtype=np.float32) / _TARGET_SR + return (0.1 * np.sin(2 * np.pi * freq * t)).astype(np.float32) + + +def _one_completion(outputs): + """Assert a single RequestOutput carrying at least one generated token.""" + assert len(outputs) == 1 + completion = outputs[0].outputs[0] + assert len(completion.token_ids) >= 1 + return completion + + +def test_text_only_serving(served): + """Baseline: a plain text request serves normally (backward-compat).""" + from vllm import SamplingParams + + outputs = served["llm"].generate( + "The capital of France is", + SamplingParams(max_tokens=8, temperature=0.0), + ) + _one_completion(outputs) + + +def test_adapter_control_token_serving(served): + """An adapter control token routes through the switch under serving. + + Correctness of the routing is `test_switch_e2e_compose.py`'s job; here we + only prove the switch path runs end-to-end in the live engine without + crashing and still generates. + """ + from vllm import SamplingParams + from vllm.inputs import TokensPrompt + + config = served["config"] + if not getattr(config, "adapter_token_ids", None): + pytest.skip("composed checkpoint has no adapters") + + tokenizer = served["llm"].get_tokenizer() + text_ids = tokenizer.encode("Summarize the document.", add_special_tokens=False) + # LORA control tokens sit at the sequence start (CLAUDE.md gotcha #3). + prompt = TokensPrompt(prompt_token_ids=[config.adapter_token_ids[0], *text_ids]) + + outputs = served["llm"].generate( + prompt, SamplingParams(max_tokens=8, temperature=0.0) + ) + _one_completion(outputs) + + +@pytest.mark.parametrize("num_clips", [1, 2, 3]) +def test_audio_clip_serving(served, num_clips): + """Audio x1/x2/x3: N markers + N clips transcribe, splice, and generate. + + Exercises the multi-clip long-audio serving path through the ASR processor + running inside vLLM's engine subprocess. Content is not asserted (WER is a + separate #47 box); the bar is a completed request with well-formed output. + """ + from vllm import SamplingParams + + ceiling = int(getattr(served["config"], "asr_max_audio_clips", 32) or 32) + if num_clips > ceiling: + pytest.skip(f"checkpoint clip ceiling {ceiling} < {num_clips}") + + marker = "<|audio|>" + prompt = { + "prompt": marker * num_clips + " What was said?", + "multi_modal_data": { + "audio": [ + (_tone(freq=220.0 * (i + 1)), _TARGET_SR) for i in range(num_clips) + ] + }, + } + outputs = served["llm"].generate( + prompt, SamplingParams(max_tokens=8, temperature=0.0) + ) + _one_completion(outputs) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index cdc65a3..f25e2a7 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -158,3 +158,33 @@ def test_asr_kwargs_round_trip(self, tmp_path): assert loaded.asr_model_id == "openai/whisper-large-v3" assert loaded.asr_pipeline_kwargs == {"chunk_length_s": 15, "batch_size": 4} assert loaded.asr_generate_kwargs == {"language": "de", "task": "transcribe"} + + def test_adapters_and_audio_coexist_round_trip(self, tmp_path): + """Adapters and the audio cascade coexist and survive save→load.""" + cfg = GraniteSwitchConfig( + **_valid_kwargs(num_adapters=3), + asr_enabled=True, + asr_model_id="distil-whisper/distil-small.en", + asr_max_audio_clips=4, + ) + assert cfg.num_adapters == 3 + assert cfg.adapter_token_ids == [500, 501, 502] + assert cfg.asr_enabled is True + + cfg.save_pretrained(tmp_path) + loaded = GraniteSwitchConfig.from_pretrained(tmp_path) + assert loaded.num_adapters == 3 + assert loaded.adapter_token_ids == [500, 501, 502] + assert loaded.adapter_names == ["adapter_0", "adapter_1", "adapter_2"] + assert loaded.adapter_ranks == [8, 8, 8] + assert loaded.adapter_substitute_token_ids == [1, 1, 1] + assert loaded.asr_enabled is True + assert loaded.asr_model_id == "distil-whisper/distil-small.en" + assert loaded.asr_max_audio_clips == 4 + + def test_adapter_token_ids_do_not_collide_with_reserved_audio_row(self, tmp_path): + """Enabling audio does not perturb adapter token ids.""" + before = GraniteSwitchConfig(**_valid_kwargs(num_adapters=2)) + after = GraniteSwitchConfig(**_valid_kwargs(num_adapters=2), asr_enabled=True) + assert before.adapter_token_ids == after.adapter_token_ids + assert before.adapter_substitute_token_ids == after.adapter_substitute_token_ids diff --git a/tests/vllm/test_audio_processor.py b/tests/vllm/test_audio_processor.py index 5a69136..2d81f3d 100644 --- a/tests/vllm/test_audio_processor.py +++ b/tests/vllm/test_audio_processor.py @@ -315,3 +315,135 @@ def test_self_chunks_and_chunk_params_forwarded(self, monkeypatch): assert capture["self_chunks"] is False assert capture["chunk_length_s"] == 20.0 assert capture["chunk_overlap_s"] == 3.0 + + +def _make_processor_transcribing(info, monkeypatch, text, *, ids_for): + """Processor whose transcriber returns a fixed ``text``; ``ids_for`` maps it to ids.""" + info.get_tokenizer = lambda: SimpleNamespace( + encode=lambda t, add_special_tokens=False: ids_for(t) + ) + proc = object.__new__(GraniteSwitchASRMultiModalProcessor) + proc.info = info + + class FakeTranscriber: + def transcribe( + self, + audio, + sampling_rate=None, + generate_kwargs=None, + self_chunks=True, + chunk_length_s=30.0, + chunk_overlap_s=5.0, + ): + return text + + monkeypatch.setattr( + proc_mod, + "get_transcriber", + lambda model_id=None, device="cpu", pipeline_kwargs=None: FakeTranscriber(), + ) + return proc + + +class TestEmptyAndSilentClip: + """A silent clip transcribes to "" -> zero transcript tokens, marker elided.""" + + def _ids_for(self, text): + return [] if text == "" else [1, 2, 3] + + def test_empty_transcript_yields_zero_length_item(self, monkeypatch): + import torch + + info = _make_info(asr_enabled=True, asr_model_id="w") + proc = _make_processor_transcribing( + info, monkeypatch, "", ids_for=self._ids_for + ) + + bf = proc._call_hf_processor( + prompt="<|audio|>", + mm_data={"audios": [np.zeros(1600, dtype=np.float32)]}, + mm_kwargs={}, + tok_kwargs={}, + ) + assert bf["audio_num_tokens"].tolist() == [0] + assert len(bf["audio_token_ids"]) == 0 + assert bf["audio_token_ids"].dtype == torch.long + + def test_empty_clip_replacement_is_empty(self, monkeypatch): + info = _make_info(asr_enabled=True, asr_model_id="w") + proc = _make_processor_transcribing( + info, monkeypatch, "", ids_for=self._ids_for + ) + + class _Kwargs: + def __init__(self, data): + self._data = data + + def get_data(self): + return self._data + + import torch + + out = _Kwargs( + { + "audio_num_tokens": torch.tensor([0], dtype=torch.long), + "audio_token_ids": torch.zeros(0, dtype=torch.long), + } + ) + updates = proc._get_prompt_updates(None, {}, out) + assert len(updates) == 1 + assert updates[0].replacement(0) == [] + + def test_mixed_empty_and_nonempty_clips(self, monkeypatch): + info = _make_info(asr_enabled=True, asr_max_audio_clips=4, asr_model_id="w") + texts = iter(["", "words"]) + info.get_tokenizer = lambda: SimpleNamespace( + encode=lambda t, add_special_tokens=False: self._ids_for(t) + ) + proc = object.__new__(GraniteSwitchASRMultiModalProcessor) + proc.info = info + + class FakeTranscriber: + def transcribe(self, audio, **kw): + return next(texts) + + monkeypatch.setattr( + proc_mod, + "get_transcriber", + lambda model_id=None, device="cpu", pipeline_kwargs=None: FakeTranscriber(), + ) + + bf = proc._call_hf_processor( + prompt="<|audio|> <|audio|>", + mm_data={ + "audios": [ + np.zeros(1600, dtype=np.float32), + np.zeros(1600, dtype=np.float32), + ] + }, + mm_kwargs={}, + tok_kwargs={}, + ) + assert bf["audio_num_tokens"].tolist() == [0, 3] + assert len(bf["audio_token_ids"]) == 3 + + +class TestClipCeiling: + """get_supported_mm_limits publishes asr_max_audio_clips as the clip ceiling.""" + + def test_ceiling_equals_configured_max(self): + assert _make_info(asr_enabled=True).get_supported_mm_limits() == {"audio": 32} + assert _make_info( + asr_enabled=True, asr_max_audio_clips=1 + ).get_supported_mm_limits() == {"audio": 1} + assert _make_info( + asr_enabled=True, asr_max_audio_clips=8 + ).get_supported_mm_limits() == {"audio": 8} + + def test_per_item_bound_never_exceeds_context(self): + info = _make_info(asr_enabled=True, asr_max_audio_clips=32) + seq_len = 4096 + for count in (1, 2, 8, 32): + bound = info.get_mm_max_tokens_per_item(seq_len, {"audio": count})["audio"] + assert bound == seq_len // count + assert bound <= seq_len