diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp index c5f816e96268..174f2243135c 100644 --- a/src/models/qwen35.cpp +++ b/src/models/qwen35.cpp @@ -1,5 +1,6 @@ #include "models.h" #include "llama-memory-recurrent.h" +#include "llama-impl.h" void llama_model_qwen35::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); @@ -634,6 +635,20 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd; tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens); + + // a Hadamard-latent embedding table (prism.hadamard.inverse_weight_names) + // stores rotated rows; restore the primal basis right after the lookup, + // the same way llm_graph_context::build_inp_embd() does, so an MTP head + // that shares the trunk's token_embd reads standard-basis embeddings + if (hadamard_inverses) { + const auto it = hadamard_inverses->find(tok_embd_w); + if (it != hadamard_inverses->end()) { + tok_embd = llama_mul_mat_hadamard(ctx0, tok_embd, it->second.rot); + if (it->second.signs) { + tok_embd = ggml_mul(ctx0, tok_embd, it->second.signs); + } + } + } } else { tok_embd = inp->embd; }