Skip to content

Qwen35 tensor split (created with deepseek V4.1 flash) #268

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Yvi71 wants to merge 7 commits into from
Closed

Qwen35 tensor split (created with deepseek V4.1 flash) #268

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
72792b0
tensor-split debug: divisibility preference, DBG prints, whole-dim se…
Sep 23, 2026
984b9a5
tensor-split: explicit nk-mapping, FFN mirror (PQ2_0), signs split, r…
Sep 23, 2026
64ec876
tensor-split: output-mirror, bin_bcast rule, seg-dump; PPL: out-CPU 1…
Sep 23, 2026
e916733
tensor-split: split FFN weights + matching hadamard signs split; mirr…
Sep 23, 2026
d429f6f
tensor-split: full SSM+attention split (qwen35). Fix ssm_out hadamard…
Sep 24, 2026
0563456
tensor-split: split PQ2_0 output head (mirror was unnecessary; PPL 1.…
Sep 24, 2026
dadcb4f
cleanup: remove debug SEG/DBG_SEG/DBG_GENERIC/DBG_CHAIN/DBG_SPLIT_FAI…
Sep 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 81 additions & 8 deletions ggml/src/ggml-backend-meta.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -555,6 +555,25 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(
return src_ss[0]; // GGML_OP_ADD_ID
}
GGML_ASSERT(tensor->src[2] == nullptr || src_ss[2].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED);
// Broadcast: one side mirrored (full copy on each device), other side split,
// same shape -> result follows the split side (each device combines local shards).
// Covers Hadamard signs/scales against split activations. Only fires where the
// old code asserted, so no behavior change for loading models.
for (int ab = 0; ab < 2; ab++) {
const int ia = ab, ib = 1 - ab;
if (src_ss[ia].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED &&
src_ss[ib].axis >= 0 && src_ss[ib].axis < GGML_MAX_DIMS) {
bool same_shape = true;
for (int dim = 0; dim < GGML_MAX_DIMS; dim++) {
const int64_t nd = tensor->src[ib]->ne[dim];
const int64_t nm = tensor->src[ia]->ne[dim];
if (nd != nm && nm != 1) { same_shape = false; break; }
}
if (same_shape) {
return src_ss[ib];
}
}
}
return handle_generic(src_ss, /*scalar_only =*/ false);
};

Expand All @@ -581,8 +600,6 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(
if (src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_1 && src_ss[1].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED) {
ggml_backend_meta_split_state ret = src_ss[0];
ret.axis = GGML_BACKEND_SPLIT_AXIS_0;
ret.nr[0] = 1;
ret.n_segments = 1;
return ret;
}
if (src_ss[1].axis == GGML_BACKEND_SPLIT_AXIS_1 && src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED) {
Expand Down Expand Up @@ -613,6 +630,14 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(
case GGML_BACKEND_SPLIT_AXIS_1:
case GGML_BACKEND_SPLIT_AXIS_2:
case GGML_BACKEND_SPLIT_AXIS_3: {
// Explicit Hadamard-permute mapping (qwen35 hybrid): a [128,16,3,...] view of an
// axis-0-split 6144-dim splits on nk (axis 1, 8+8). The generic cumulative rule
// would pick the rep dim (size 3, unshardable). FWHT keeps full 128, reps stay whole.
if (src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_0 &&
tensor->src[0] && tensor->src[0]->ne[0] == 6144 &&
tensor->ne[0] == 128 && tensor->ne[1] == 16 && tensor->ne[2] == 3) {
return {GGML_BACKEND_SPLIT_AXIS_1, {0}, {1}, 1};
}
int64_t base_ne_in = 1;
for (int dim = 0; dim <= src_ss[0].axis; dim++) {
base_ne_in *= tensor->src[0]->ne[dim];
Expand All @@ -636,15 +661,28 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(
}
}
// Reshape outputs use one segment; split-state propagation merges source segments.
// Prefer a dim that actually divides across devices (e.g. skip rep=3 tilings);
// first cumulative match kept as fallback = legacy behavior for all working models.
int64_t base_ne_out = 1;
int fallback_dim = -1;
uint32_t fallback_nr = 1;
for (int dim = 0; dim < GGML_MAX_DIMS; dim++) {
base_ne_out *= tensor->ne[dim];
if (base_ne_out % base_ne_in == 0) {
return {ggml_backend_meta_split_axis(dim), {0}, {uint32_t(base_ne_out/base_ne_in)}, 1};
if (fallback_dim < 0) {
fallback_dim = dim;
fallback_nr = uint32_t(base_ne_out/base_ne_in);
}
if (n_bufs > 0 && tensor->ne[dim] % (int64_t) n_bufs == 0) {
return {ggml_backend_meta_split_axis(dim), {0}, {uint32_t(base_ne_out/base_ne_in)}, 1};
}
}
if (base_ne_out > base_ne_in) {
GGML_ASSERT(src_ss[0].n_segments == 1);
GGML_ASSERT(src_ss[0].nr[0] == 1);
if (fallback_dim >= 0) {
return {ggml_backend_meta_split_axis(fallback_dim), {0}, {fallback_nr}, 1};
}
return {ggml_backend_meta_split_axis(dim), {0}, {1}, 1};
}
}
Expand Down Expand Up @@ -801,10 +839,14 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(
auto handle_ssm_conv = [&](const std::vector<ggml_backend_meta_split_state> & src_ss) -> ggml_backend_meta_split_state {
if (src_ss[0].axis == src_ss[1].axis) {
if (src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_0) {
return {GGML_BACKEND_SPLIT_AXIS_1, {0}, {1}, 1};
ggml_backend_meta_split_state ret = src_ss[0];
ret.axis = GGML_BACKEND_SPLIT_AXIS_1;
return ret;
}
if (src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_1) {
return {GGML_BACKEND_SPLIT_AXIS_0, {0}, {1}, 1};
ggml_backend_meta_split_state ret = src_ss[0];
ret.axis = GGML_BACKEND_SPLIT_AXIS_0;
return ret;
}
}
return handle_generic(src_ss, /*scalar_only =*/ false);
Expand All @@ -824,7 +866,25 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(
// state shape is [S_v, S_v, H_v, n_seqs] (s0 only); the heads dim is its own axis 2,
// so a head-aligned split on the input cache lands on axis 2 here.
GGML_ASSERT(src_ss[5].axis == GGML_BACKEND_SPLIT_AXIS_2 || src_ss[5].axis == GGML_BACKEND_SPLIT_AXIS_1 || src_ss[5].axis == GGML_BACKEND_SPLIT_AXIS_0);
return {GGML_BACKEND_SPLIT_AXIS_0, {0}, {1}, 1};
// qwen35 grouped-V: the GDN output is laid out as head_ratio blocks of n_k heads
// (device-local V groups). Preserve that grouping so the downstream ssm_out
// (segmented {key_dim, head_ratio}) matches its activation layout.
{
const int64_t gdn_head_dim = tensor->src[0]->ne[0];
const int64_t gdn_n_k_heads = tensor->src[0]->ne[1];
const size_t gdn_n_bufs = ggml_backend_meta_buffer_n_bufs(tensor->buffer);
ggml_backend_meta_split_state gdn_ret = {GGML_BACKEND_SPLIT_AXIS_0, {0}, {1}, 1};
if (gdn_head_dim > 0 && gdn_n_k_heads > 0 && tensor->ne[0] % (gdn_head_dim * gdn_n_k_heads) == 0) {
const int64_t gdn_ratio = tensor->ne[0] / (gdn_head_dim * gdn_n_k_heads);
if (gdn_ratio > 1 && tensor->ne[0] % (int64_t)(gdn_n_bufs * gdn_ratio) == 0) {
gdn_ret.nr[0] = (uint32_t) gdn_ratio;
for (size_t j = 0; j < gdn_n_bufs; j++) {
gdn_ret.ne[j] = tensor->ne[0] / (int64_t)(gdn_n_bufs * gdn_ratio);
}
}
}
return gdn_ret;
}
};

auto calculate_split_state = [&]() -> ggml_backend_meta_split_state {
Expand Down Expand Up @@ -1063,7 +1123,8 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(
split_state = {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, {1}, 1};
} break;
}
if (split_state.axis >= 0 && split_state.axis < GGML_MAX_DIMS) {
const bool split_ne_precomputed = split_state.nr[0] > 1 && split_state.ne[0] > 0;
if (split_state.axis >= 0 && split_state.axis < GGML_MAX_DIMS && !split_ne_precomputed) {
bool first_src_split_by_axis = true;
const size_t n_bufs = ggml_backend_meta_buffer_n_bufs(tensor->buffer);

Expand All @@ -1072,7 +1133,19 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(
continue;
}
if (first_src_split_by_axis) {
const bool seg_copy = src_ss[i].n_segments > 1 &&
tensor->ne[split_state.axis] == tensor->src[i]->ne[src_ss[i].axis];
if (seg_copy) {
split_state.n_segments = src_ss[i].n_segments;
for (size_t s = 0; s < src_ss[i].n_segments; s++) {
split_state.nr[s] = src_ss[i].nr[s];
for (size_t j = 0; j < n_bufs; j++) {
split_state.ne[s*n_bufs + j] = src_ss[i].ne[s*n_bufs + j];
}
}
}
for (size_t j = 0; j < n_bufs; j++) {
if (seg_copy) break;
// Take over ratio from src:
for (size_t s = 0; s < src_ss[i].n_segments; s++) {
split_state.ne[s*n_bufs + j] = 0;
Expand All @@ -1083,7 +1156,7 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(
split_state.ne[j] *= tensor->ne[split_state.axis];
if (split_state.ne[j] != 0 || tensor->src[i]->ne[src_ss[i].axis] != 0) {
const int64_t div = tensor->src[i]->ne[src_ss[i].axis] * split_state.nr[0];
GGML_ASSERT(split_state.ne[j] % div == 0);
GGML_ASSERT(div == 0 || split_state.ne[j] % div == 0);
split_state.ne[j] /= div;
}
}
Expand Down
81 changes: 79 additions & 2 deletions src/llama-model.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -492,6 +492,56 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
}
}

// QWEN35 linear block: mirror SSM weights (replicated compute needs no sharding;
// residual stream is mirrored, verified by working Q4 tensor-split).
if (false && ud->model->arch == LLM_ARCH_QWEN35) {
if (std::regex_match(tensor_name, pattern_attn_gate_weight) ||
std::regex_match(tensor_name, pattern_ssm_out_weight) ||
std::regex_match(tensor_name, pattern_qkv_weight) ||
std::regex_match(tensor_name, pattern_ssm_conv1d) ||
std::regex_match(tensor_name, pattern_ssm_dt) ||
std::regex_match(tensor_name, pattern_ssm_a) ||
std::regex_match(tensor_name, pattern_ssm_alpha) ||
std::regex_match(tensor_name, pattern_ssm_beta) ||
std::regex_match(tensor_name, pattern_ssm_beta_alpha) ||
std::regex_match(tensor_name, pattern_r_cache) ||
std::regex_match(tensor_name, pattern_s_cache)) {
return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);
}
}
// Bonsai (hadamard-folded PQ2_0): 1-D sign vectors feed the hadamard input
// transform and must carry the SAME split state as the activation they
// multiply. signs.6144 is consumed by BOTH the attention-output activation
// and the (mirrored) SSM output; one vector cannot serve a split and a
// mirrored consumer at once. Keep the small attention block mirrored and
// split the large FFN instead.
if (false && !ud->model->hadamard_sign_data.empty() &&
(std::regex_match(tensor_name, pattern_q_weight) ||
std::regex_match(tensor_name, pattern_kv_weight) ||
std::regex_match(tensor_name, pattern_q_bias) ||
std::regex_match(tensor_name, pattern_kv_bias) ||
std::regex_match(tensor_name, pattern_qk_norm) ||
std::regex_match(tensor_name, pattern_attn_out_weight) ||
std::regex_match(tensor_name, pattern_attn_out_bias) ||
std::regex_match(tensor_name, pattern_kv_cache))) {
return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);
}
// FFN is split (hadamard input is 1024-block aligned; signs.17408 splits with it).
// Hadamard sign vectors: a 1-D signs vector must carry the SAME split state
// as the activation it multiplies (the elementwise mul indexes from the
// device-local offset). The FFN input activation (ffn_swiglu, width 17408)
// is split axis-0, so its signs split too; every other width is consumed by
// mirrored activations and stays mirrored.
{
static const std::string signs_prefix = "prism.hadamard.signs.";
if (tensor_name.compare(0, signs_prefix.size(), signs_prefix) == 0) {
const long sign_width = std::stol(tensor_name.substr(signs_prefix.size()));
if (sign_width == 17408 || sign_width == 6144) {
return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0);
}
return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);
}
}
// standard attention
if (std::regex_match(tensor_name, pattern_q_weight) || std::regex_match(tensor_name, pattern_kv_weight)) {
return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight", "ssm_out.weight");
Expand Down Expand Up @@ -605,9 +655,15 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
GGML_ASSERT(tensor->ne[axis] == 2*key_dim + value_dim);
return {{key_dim, 2 + head_ratio}};
}
if (std::regex_match(tensor_name, pattern_attn_gate_weight) || std::regex_match(tensor_name, pattern_ssm_out_weight)) {
if (std::regex_match(tensor_name, pattern_attn_gate_weight)) {
return {{key_dim, head_ratio}};
}
if (std::regex_match(tensor_name, pattern_ssm_out_weight)) {
// ssm_out consumes the hadamard-regrouped (perm_rep) activation, whose
// device-local layout is a contiguous axis-0 half; keep it contiguous
// so the weight rows match the activation elements per device.
return {{tensor->ne[axis], 1}};
}
if (std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_ssm_a) ||
std::regex_match(tensor_name, pattern_ssm_alpha) || std::regex_match(tensor_name, pattern_ssm_beta)) {
return {{n_k_heads, head_ratio}};
Expand Down Expand Up @@ -749,14 +805,23 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
}

// FFN
// Hadamard sign vectors: use the same 1024-block granularity as the FFN
// hadamard input so the elementwise-mul split state equals ffn_swiglu's.
{
static const std::string signs_pfx = "prism.hadamard.signs.";
if (tensor_name.compare(0, signs_pfx.size(), signs_pfx) == 0) {
return {std::lcm(blck_size, 1024)};
}
}
if (std::regex_match(tensor_name, pattern_ffn_up_weight) || std::regex_match(tensor_name, pattern_ffn_up_bias) ||
std::regex_match(tensor_name, pattern_ffn_gate_weight) || std::regex_match(tensor_name, pattern_ffn_gate_bias) ||
std::regex_match(tensor_name, pattern_ffn_gate_up_weight) ||
std::regex_match(tensor_name, pattern_ffn_down_weight) ||
std::regex_match(tensor_name, pattern_ffn_up_shexp_weight) ||
std::regex_match(tensor_name, pattern_ffn_gate_shexp_weight) ||
std::regex_match(tensor_name, pattern_ffn_down_shexp_weight)) {
const int64_t blck_size_perf = std::lcm(blck_size, 128);
const int64_t blck_size_perf = (tensor->type == GGML_TYPE_PQ2_0 || tensor->type == GGML_TYPE_PTQ1_0)
? std::lcm(blck_size, 1024) : std::lcm(blck_size, 128);
GGML_ASSERT(segments.size() == 1);
return {blck_size_perf};
}
Expand All @@ -769,6 +834,18 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
ggml_backend_meta_split_state split_state;
memset(&split_state, 0, sizeof(split_state));
tensor_config tc = get_tensor_config();
// Hadamard FFN: force rotation 0 so every layer's ffn_swiglu split and the
// shared 1-D signs vector receive identical device boundaries (rotation would
// make odd/even layers disagree, which a single shared vector cannot satisfy).
if (!ud->model->hadamard_sign_data.empty()) {
static const std::string signs_pfx_rot = "prism.hadamard.signs.";
if (std::regex_match(tensor_name, pattern_ffn_up_weight) ||
std::regex_match(tensor_name, pattern_ffn_gate_weight) ||
std::regex_match(tensor_name, pattern_ffn_down_weight) ||
tensor_name.compare(0, signs_pfx_rot.size(), signs_pfx_rot) == 0) {
tc.rotation = 0;
}
}
split_state.axis = tc.axis;
if (split_state.axis >= 0 && split_state.axis < GGML_MAX_DIMS) {
const int64_t blck_size = ggml_blck_size(tc.tensor_axis_0->type);
Expand Down