From 6d1d921ddd97e715a98d499738e967e8ff85215e Mon Sep 17 00:00:00 2001 From: lenny76 Date: Wed, 23 Sep 2026 11:21:05 +0200 Subject: [PATCH] ggml-cpu : convert each mul_mat src1 row on a single thread The src1 conversion to vec_dot_type split every row into nth block ranges. With one row (token generation) every thread writes a few blocks, cache lines at the range borders are written by two cores (often on different sockets), and all later vec_dot reads of the row stay slow. On a 2-socket Xeon Gold 6262 with 48 threads a PTQ1_0 row dot took ~12000 cycles instead of ~2700. Split the conversion over flattened (i11, i12, i13) rows instead, so each row is written by one thread. This is also what the repack path already does. The converted data is the same. Co-Authored-By: Claude Opus 5.5 --- ggml/src/ggml-cpu/ggml-cpu.c | 43 ++++++++++++------------------------ 1 file changed, 14 insertions(+), 29 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index f63c87b03ca0..7debad319e74 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -1362,7 +1362,6 @@ UseGgmlGemm1:; if (src1->type != vec_dot_type) { char * wdata = params->wdata; - const size_t nbw0 = ggml_type_size(vec_dot_type); const size_t nbw1 = ggml_row_size(vec_dot_type, ne10); const size_t nbw2 = nbw1*ne11; const size_t nbw3 = nbw2*ne12; @@ -1372,36 +1371,22 @@ UseGgmlGemm1:; GGML_ASSERT(src1->type == GGML_TYPE_F32 || (src1->type == GGML_TYPE_F16 && vec_dot_type == GGML_TYPE_F32)); - #if 0 - for (int64_t i13 = 0; i13 < ne13; ++i13) { - for (int64_t i12 = 0; i12 < ne12; ++i12) { - for (int64_t i11 = ith; i11 < ne11; i11 += nth) { - from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11), - (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1), - ne10); - } - } - } - #else - for (int64_t i13 = 0; i13 < ne13; ++i13) { - for (int64_t i12 = 0; i12 < ne12; ++i12) { - for (int64_t i11 = 0; i11 < ne11; ++i11) { - size_t bs = ggml_blck_size(vec_dot_type); - int64_t ne10_block_start = (ith * ne10/bs) / nth; - int64_t ne10_block_end = ((ith + 1) * ne10/bs) / nth; - const char * src1_block = (const char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + ne10_block_start*bs*nb10; - char * dst_block = wdata + i13*nbw3 + i12*nbw2 + i11*nbw1 + ne10_block_start*nbw0; - const int64_t n_block = (ne10_block_end - ne10_block_start) * bs; - - if (src1->type == GGML_TYPE_F32) { - from_float((const float *) src1_block, dst_block, n_block); - } else { - ggml_cpu_fp16_to_fp32((const ggml_fp16_t *) src1_block, (float *) dst_block, n_block); - } - } + // one thread per src1 row: splitting a row across threads makes several cores write the same + // cache lines, and every later vec_dot read of the row gets slow (10x on a 2-socket Xeon) + const int64_t nr1 = ne11*ne12*ne13; + for (int64_t ir = ith; ir < nr1; ir += nth) { + const int64_t i13 = ir/(ne12*ne11); + const int64_t i12 = (ir - i13*ne12*ne11)/ne11; + const int64_t i11 = ir - i13*ne12*ne11 - i12*ne11; + const char * src1_row = (const char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11; + char * dst_row = wdata + i13*nbw3 + i12*nbw2 + i11*nbw1; + + if (src1->type == GGML_TYPE_F32) { + from_float((const float *) src1_row, dst_row, ne10); + } else { + ggml_cpu_fp16_to_fp32((const ggml_fp16_t *) src1_row, (float *) dst_row, ne10); } } - #endif } if (ith == 0) {