From 837cae19a46edab1db07b13c64ccb2144bfeeccb Mon Sep 17 00:00:00 2001 From: Scott J Guyton Date: Thu, 24 Sep 2026 08:16:05 -0700 Subject: [PATCH 1/4] arm: add NEON vec_dot for PTQ1_0 and Q8_K-path PQ2_0 Same gap on ARM as x86 had before the SSE work: PTQ1_0 had no NEON implementation at all (arch-fallback.h aliased it straight to the generic C loop), and the existing NEON PQ2_0 kernel targets Q8_0 activations, which is no longer what the traits table dispatches now that PQ2_0 uses Q8_K - so it was dead code for normal inference, same issue bri-prism found on x86 in PR #248. Add: ggml_vec_dot_pq2_0_q8_K - same 2-bit codec and vqtbl1q_u8 replication trick as the existing Q8_0 kernel, but four sub-block dots accumulate in int32 and the block scale is applied once, matching the Q8_K activation layout (one float scale per 256 elements instead of one fp16 scale per 32). ggml_vec_dot_ptq1_0_q8_0 - base-3 trit decode: for packed byte v the next trit is floor(3*v/256) and v carries forward as (3*v) & 0xFF, done 16 lanes at a time in uint16 so 3*v (max 765) never overflows and needs no division. Same three stages as the scalar reference (16/16/8 qs groups, then qh), reduced with ggml_vdotq_s32 (sdot) on the 8-lane groups and vmull_s8 + vpadalq_s32 on the tail 8-lane groups. Correctness: exact vec_dot check (ported from PR #248, same test) passes 0 failures for both formats on a real Cortex-A76/A55 board (Orange Pi 5 Plus, RK3588). Also validated with a private property-based harness (theft, ~124k random inputs per format) and an exhaustive edge sweep (every weight byte 0-255 x activation extremes) before this PR - not included here, same reasoning as the x86 PR. Found one real bug before it ever ran: an earlier version fed the wrong vector width into vpadalq_s16 (vpaddl_s8+vmovl_s16 produces int32x4_t, not the int16x8_t the intrinsic expects), which failed to compile on ARM. Fixed to vmull_s8 (real int16x8_t product, no overflow since |trit| <= 1 and |activation| <= 128) feeding vpadalq_s16 directly. Performance, Orange Pi 5 Plus (RK3588, 4x Cortex-A76 + 4x Cortex-A55, asimddp), isolated A/B (same binary, only libggml-cpu.so swapped, alternated, two pairs): Ternary Bonsai 1.7B PQ2_0 pp128 tg32 scalar 4.50/4.53 3.63/3.66 NEON 23.19/23.21 17.54/15.89 ~5.1x pp, ~4.6x tg - the largest speedup measured across any backend for this format family so far. End-to-end verification: greedy (temp=0) completion on the 8B PQ2_0 model is byte-identical between this NEON build and an x86 SSE build (PR #248), same prompt, same seed. This exercises the full inference pipeline, not just the kernel in isolation. The 27B PTQ1_0 model does NOT match byte-for-byte between x86 and ARM at temp=0, diverging after ~28 tokens on an otherwise-identical prefix. This is not a bug in this kernel: the same divergence reproduces with the generic scalar C path on ARM (i.e. with no NEON code in the execution path at all), so it predates and is independent of this PR. It is the well-documented cross-architecture floating-point non-associativity issue in SIMD reduction (different horizontal-sum order between AVX2 and NEON, plus FMA/reassociation differences) - the same class of divergence llama-cpp-et (github.com/anomly-labs/llama-cpp-et) exists to eliminate, at a stated ~3x throughput cost for their exact/deterministic profile. Not worth chasing here: it would erase most of the margin ternary quantization exists to create, and same-architecture runs are already reproducible (confirmed: re-running the x86 build gives identical output; the ARM scalar and ARM NEON builds agree with each other, just not with x86). --- ggml/src/ggml-cpu/arch-fallback.h | 4 - ggml/src/ggml-cpu/arch/arm/quants.c | 144 ++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h index e116558224b4..cb093eb59d4b 100644 --- a/ggml/src/ggml-cpu/arch-fallback.h +++ b/ggml/src/ggml-cpu/arch-fallback.h @@ -82,10 +82,6 @@ #define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0 #define ggml_gemm_pq2_0_4x8_q8_0_generic ggml_gemm_pq2_0_4x8_q8_0 #elif defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64) -// PTQ1_0 currently has only the generic vec_dot; alias it here until a SIMD version lands -#define ggml_vec_dot_ptq1_0_q8_0_generic ggml_vec_dot_ptq1_0_q8_0 -// PQ2_0 x Q8_K has only the generic vec_dot outside x86; alias it until a SIMD version lands -#define ggml_vec_dot_pq2_0_q8_K_generic ggml_vec_dot_pq2_0_q8_K // repack.cpp #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 diff --git a/ggml/src/ggml-cpu/arch/arm/quants.c b/ggml/src/ggml-cpu/arch/arm/quants.c index c93fc7b4fd3e..c098a331be75 100644 --- a/ggml/src/ggml-cpu/arch/arm/quants.c +++ b/ggml/src/ggml-cpu/arch/arm/quants.c @@ -365,6 +365,150 @@ void ggml_vec_dot_pq2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo *s = sumf; } +void ggml_vec_dot_pq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_pq2_0 * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + const int nb = n / QK_PQ2_0; + + float sumf = 0.0f; + +#if defined(__ARM_NEON) + // Same 2-bit codec as the Q8_0 path above: byte b of a 32-weight sub-block holds + // weights 4b..4b+3, LSB pair first, so replicating each byte four times and shifting + // by {0,2,4,6} lands code 4b+j at lane 4b+j. With one activation scale per 256 the + // four sub-block dots accumulate in int32 and the scale is applied once per block, + // instead of a float multiply-add per sub-block. + static const uint8_t tbl_idx_lo[16] = {0,0,0,0, 1,1,1,1, 2,2,2,2, 3,3,3,3}; + static const uint8_t tbl_idx_hi[16] = {4,4,4,4, 5,5,5,5, 6,6,6,6, 7,7,7,7}; + static const int8_t shift_vals[16] = {0,-2,-4,-6, 0,-2,-4,-6, 0,-2,-4,-6, 0,-2,-4,-6}; + + const uint8x16_t idx_lo = vld1q_u8(tbl_idx_lo); + const uint8x16_t idx_hi = vld1q_u8(tbl_idx_hi); + const int8x16_t shifts = vld1q_s8(shift_vals); + const uint8x16_t mask2 = vdupq_n_u8(0x03); + const int8x16_t one = vdupq_n_s8(1); + + for (int i = 0; i < nb; i++) { + const block_q8_K * GGML_RESTRICT yb = &y[i >> 1]; + const int8_t * GGML_RESTRICT q8 = yb->qs + 128 * (i & 1); + + int32x4_t acc = vdupq_n_s32(0); + + for (int k = 0; k < 4; k++) { + const uint8x8_t raw = vld1_u8(&x[i].qs[8 * k]); + const uint8x16_t raw16 = vcombine_u8(raw, raw); + + const int8x16_t qv0 = vsubq_s8( + vreinterpretq_s8_u8(vandq_u8(vshlq_u8(vqtbl1q_u8(raw16, idx_lo), shifts), mask2)), one); + const int8x16_t qv1 = vsubq_s8( + vreinterpretq_s8_u8(vandq_u8(vshlq_u8(vqtbl1q_u8(raw16, idx_hi), shifts), mask2)), one); + + const int8x16_t y0 = vld1q_s8(q8 + 32 * k); + const int8x16_t y1 = vld1q_s8(q8 + 32 * k + 16); + + acc = ggml_vdotq_s32(acc, qv0, y0); + acc = ggml_vdotq_s32(acc, qv1, y1); + } + + sumf += (GGML_CPU_FP16_TO_FP32(x[i].d) * yb->d) * (float) vaddvq_s32(acc); + } +#else + ggml_vec_dot_pq2_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + return; +#endif + + *s = sumf; +} + +void ggml_vec_dot_ptq1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_PTQ1_0 == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_ptq1_0 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + const int nb = n / QK_PTQ1_0; + + float sumf = 0.0f; + +#if defined(__ARM_NEON) + // Base-3 trit decode, matching dequantize_row_ptq1_0(): for each packed byte v the + // next trit is floor(3*v/256) and the byte carries forward as (3*v) & 0xFF. Doing that + // 16 bytes at a time in uint16 lanes keeps 3*v exact (max 765) and needs no division. + // qs holds 24 bytes x 5 trits = 120 weights, qh 2 bytes x 4 trits = 8 more, in the + // stage order 16/8 that the reference walks. + for (int i = 0; i < nb; i++) { + const block_q8_0 * GGML_RESTRICT yb = &y[i * 4]; + + int32x4_t acc[4] = { vdupq_n_s32(0), vdupq_n_s32(0), vdupq_n_s32(0), vdupq_n_s32(0) }; + + // stage 1: qs[0..15], 5 trits each -> weights 0..79 + uint16x8_t lo = vmovl_u8(vld1_u8(x[i].qs)); + uint16x8_t hi = vmovl_u8(vld1_u8(x[i].qs + 8)); + for (int t = 0; t < 5; t++) { + const uint16x8_t l3 = vmulq_n_u16(lo, 3); + const uint16x8_t h3 = vmulq_n_u16(hi, 3); + const int8x16_t q = vsubq_s8( + vreinterpretq_s8_u8(vcombine_u8(vshrn_n_u16(l3, 8), vshrn_n_u16(h3, 8))), + vdupq_n_s8(1)); + lo = vandq_u16(l3, vdupq_n_u16(0xFF)); + hi = vandq_u16(h3, vdupq_n_u16(0xFF)); + + const int o = t * 16; + acc[o >> 5] = ggml_vdotq_s32(acc[o >> 5], q, vld1q_s8(yb[o >> 5].qs + (o & 31))); + } + + // stage 2: qs[16..23], 5 trits each -> weights 80..119 + uint16x8_t t8 = vmovl_u8(vld1_u8(x[i].qs + 16)); + for (int t = 0; t < 5; t++) { + const uint16x8_t p3 = vmulq_n_u16(t8, 3); + const int8x8_t q = vsub_s8(vreinterpret_s8_u8(vshrn_n_u16(p3, 8)), vdup_n_s8(1)); + t8 = vandq_u16(p3, vdupq_n_u16(0xFF)); + + const int o = 80 + t * 8; + const int8x8_t yv = vld1_s8(yb[o >> 5].qs + (o & 31)); + acc[o >> 5] = vpadalq_s16(acc[o >> 5], vmull_s8(q, yv)); + } + + // stage 3: qh[0..1], 4 trits each -> weights 120..127 + uint16_t qh2; + memcpy(&qh2, x[i].qh, sizeof(qh2)); + uint16x8_t hv = vmovl_u8(vreinterpret_u8_u16(vdup_n_u16(qh2))); + { + static const uint16_t pw[8] = {1, 1, 3, 3, 9, 9, 27, 27}; + hv = vandq_u16(vmulq_u16(hv, vld1q_u16(pw)), vdupq_n_u16(0xFF)); + } + { + const uint16x8_t p3 = vmulq_n_u16(hv, 3); + const int8x8_t q = vsub_s8(vreinterpret_s8_u8(vshrn_n_u16(p3, 8)), vdup_n_s8(1)); + const int8x8_t yv = vld1_s8(yb[3].qs + 24); + acc[3] = vpadalq_s16(acc[3], vmull_s8(q, yv)); + } + + float sumi = 0.0f; + for (int k = 0; k < 4; k++) { + sumi += GGML_CPU_FP16_TO_FP32(yb[k].d) * (float) vaddvq_s32(acc[k]); + } + sumf += GGML_CPU_FP16_TO_FP32(x[i].d) * sumi; + } +#else + ggml_vec_dot_ptq1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + return; +#endif + + *s = sumf; +} + void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { const int qk = QK8_0; const int nb = n / qk; From 308f4b648c910b9d53cbfda2e6ef476af06c53bf Mon Sep 17 00:00:00 2001 From: Scott J Guyton Date: Thu, 24 Sep 2026 08:16:13 -0700 Subject: [PATCH 2/4] test-quantize-fns: exact vec_dot check for PTQ1_0 and PQ2_0 Same test as PR #248 (x86), applicable here unchanged since the property being checked - packed SIMD vec_dot bit-for-bit equal to dequantize-then-scalar-dot, with power-of-two scales so the comparison is exact - has nothing architecture-specific about it. Kept as a separate commit for the same reason it was split there: the test and the kernel are independently reviewable, and this one is the correctness gate that would have caught the earlier NEON compile bug in a debug build if it had been silently wrong instead of a hard compile error. Passes on ARM (Cortex-A76/A55) with 0 failures for both formats. --- tests/test-quantize-fns.cpp | 80 +++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/tests/test-quantize-fns.cpp b/tests/test-quantize-fns.cpp index 4f77fb9701c7..4172f316ef38 100644 --- a/tests/test-quantize-fns.cpp +++ b/tests/test-quantize-fns.cpp @@ -2,6 +2,7 @@ #include "ggml.h" #include "ggml-cpu.h" +#include "../ggml/src/ggml-quants.h" #undef NDEBUG #include @@ -202,6 +203,84 @@ static int test_vec_dot_q(bool verbose) { return num_failed; } +static int test_vec_dot_ternary(bool verbose) { + int num_failed = 0; + for (ggml_type type : {GGML_TYPE_PQ2_0, GGML_TYPE_PTQ1_0}) { + const auto * traits = ggml_get_type_traits(type); + const auto * cpu = ggml_get_type_traits_cpu(type); + // PQ2_0 dots against Q8_K (one float scale per 256), PTQ1_0 against Q8_0 (one fp16 + // scale per 32), so the activation side is built per format. PQ2_0 needs whole Q8_K + // blocks, i.e. an even number of 128-weight blocks. + const bool q8k = type == GGML_TYPE_PQ2_0; + const ggml_type ytype = q8k ? GGML_TYPE_Q8_K : GGML_TYPE_Q8_0; + for (int nb : q8k ? std::vector{2, 4} : std::vector{1, 3}) { + const int n = nb * 128; + std::vector pq(nb); + std::vector ptq(nb); + std::vector q8(nb * 4); + std::vector q8k_blocks(nb / 2 + 1); + std::vector x(n), y(n); + const void * weights = type == GGML_TYPE_PQ2_0 ? (const void *) pq.data() : (const void *) ptq.data(); + for (int pattern = 0; pattern < 256; ++pattern) { + for (int i = 0; i < nb; ++i) { + pq[i].d = ptq[i].d = ggml_fp32_to_fp16(0.25f * (i + 1)); + for (size_t j = 0; j < sizeof(pq[i].qs); ++j) { + pq[i].qs[j] = (uint8_t) (pattern + 17*j + i); + } + for (size_t j = 0; j < sizeof(ptq[i].qs); ++j) { + ptq[i].qs[j] = (uint8_t) (pattern + 17*j + i); + } + for (size_t j = 0; j < sizeof(ptq[i].qh); ++j) { + ptq[i].qh[j] = (uint8_t) (pattern + 37*j + i); + } + } + for (int i = 0; i < nb * 4; ++i) { + q8[i].d = ggml_fp32_to_fp16(0.125f * (i % 4 + 1)); + for (int j = 0; j < QK8_0; ++j) { + q8[i].qs[j] = (int8_t) ((pattern + 13*j + i) % 256 - 128); + } + } + for (size_t i = 0; i < q8k_blocks.size(); ++i) { + q8k_blocks[i].d = 0.125f * (i % 4 + 1); + for (int j = 0; j < QK_K; ++j) { + q8k_blocks[i].qs[j] = (int8_t) ((pattern + 13*j + i) % 256 - 128); + } + // bsums is unused by the PQ2_0 dot but keep it consistent. + for (int j = 0; j < QK_K/16; ++j) { + int16_t s = 0; + for (int t = 0; t < 16; ++t) s += q8k_blocks[i].qs[j*16 + t]; + q8k_blocks[i].bsums[j] = s; + } + } + const void * acts = q8k ? (const void *) q8k_blocks.data() : (const void *) q8.data(); + traits->to_float(weights, x.data(), n); + if (q8k) { + // Q8_K is an activation-only type and has no to_float, so expand it here. + for (int j = 0; j < n; ++j) { + const block_q8_K & b = q8k_blocks[j / QK_K]; + y[j] = b.d * (float) b.qs[j % QK_K]; + } + } else { + ggml_get_type_traits(ytype)->to_float(acts, y.data(), n); + } + const float ref = dot_product(x.data(), y.data(), n); + float result = INFINITY; + cpu->vec_dot(n, &result, 0, weights, 0, acts, 0, 1); + // Power-of-two scales keep this comparison exact. + const bool failed = result != ref; + num_failed += failed; + if (failed) { + printf("%5s packed dot nb=%d pattern=%d: FAILED (ref=%f got=%f)\n", ggml_type_name(type), nb, pattern, ref, result); + } + } + } + } + if (num_failed || verbose) { + printf("ternary packed dot products: %s (%d failures)\n", RESULT_STR[num_failed != 0], num_failed); + } + return num_failed; +} + int main(int argc, char * argv[]) { bool verbose = false; @@ -223,6 +302,7 @@ int main(int argc, char * argv[]) { num_failed += test_vec_dot_f32(verbose); num_failed += test_vec_dot_q(verbose); + num_failed += test_vec_dot_ternary(verbose); if (num_failed || verbose) { printf("%d tests failed\n", num_failed); From 22e70a2d00698021ac4597e11eb576e0900e1220 Mon Sep 17 00:00:00 2001 From: Scott J Guyton Date: Thu, 24 Sep 2026 11:56:25 -0700 Subject: [PATCH 3/4] arm: fix ARMv7 build by using the ggml_vqtbl1q_u8 wrapper Raw vqtbl1q_u8 is AArch64-only, but both new kernels are guarded by __ARM_NEON, which 32-bit ARMv7 NEON also defines. ggml-cpu-impl.h already has ggml_vqtbl1q_u8 with a 32-bit fallback for exactly this, and it's what other kernels in this same file use (e.g. the unrelated ggml_vec_dot_q2_0_q8_0). Switch the two new call sites, plus the two pre-existing ones in the (currently dead, per this PR's own commit message) ggml_vec_dot_pq2_0_q8_0 that had the same mistake already. Found by bri-prism cross-compiling with clang -fsyntax-only for armv7a-w64-mingw32; I don't have ARMv7 hardware or a cross-toolchain here to independently re-verify the fix compiles there (no sudo to install one), so this is the same pattern-match the reviewer already identified, not an independent confirmation. What IS re-verified, on the real Cortex-A76/A55 board this PR was tested on: ggml_vqtbl1q_u8 is a plain #define to vqtbl1q_u8 on __aarch64__ (ggml-cpu-impl.h), so this is a no-op there by construction. test-quantize-fns still passes 0 failures for both formats after the change, confirming nothing broke on the platform I can actually test. --- ggml/src/ggml-cpu/arch/arm/quants.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/arm/quants.c b/ggml/src/ggml-cpu/arch/arm/quants.c index c098a331be75..b36d24d51b50 100644 --- a/ggml/src/ggml-cpu/arch/arm/quants.c +++ b/ggml/src/ggml-cpu/arch/arm/quants.c @@ -336,12 +336,12 @@ void ggml_vec_dot_pq2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo const uint8x8_t raw = vld1_u8(&x[i].qs[k * 8]); const uint8x16_t raw16 = vcombine_u8(raw, raw); - uint8x16_t bytes0 = vqtbl1q_u8(raw16, idx_lo); + uint8x16_t bytes0 = ggml_vqtbl1q_u8(raw16, idx_lo); int8x16_t qv0 = vsubq_s8( vreinterpretq_s8_u8(vandq_u8(vshlq_u8(bytes0, shifts), mask2)), one); - uint8x16_t bytes1 = vqtbl1q_u8(raw16, idx_hi); + uint8x16_t bytes1 = ggml_vqtbl1q_u8(raw16, idx_hi); int8x16_t qv1 = vsubq_s8( vreinterpretq_s8_u8(vandq_u8(vshlq_u8(bytes1, shifts), mask2)), one); @@ -406,9 +406,9 @@ void ggml_vec_dot_pq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo const uint8x16_t raw16 = vcombine_u8(raw, raw); const int8x16_t qv0 = vsubq_s8( - vreinterpretq_s8_u8(vandq_u8(vshlq_u8(vqtbl1q_u8(raw16, idx_lo), shifts), mask2)), one); + vreinterpretq_s8_u8(vandq_u8(vshlq_u8(ggml_vqtbl1q_u8(raw16, idx_lo), shifts), mask2)), one); const int8x16_t qv1 = vsubq_s8( - vreinterpretq_s8_u8(vandq_u8(vshlq_u8(vqtbl1q_u8(raw16, idx_hi), shifts), mask2)), one); + vreinterpretq_s8_u8(vandq_u8(vshlq_u8(ggml_vqtbl1q_u8(raw16, idx_hi), shifts), mask2)), one); const int8x16_t y0 = vld1q_s8(q8 + 32 * k); const int8x16_t y1 = vld1q_s8(q8 + 32 * k + 16); From 643a8ade27187c590391d476bc1c6cfe51d4305a Mon Sep 17 00:00:00 2001 From: Scott J Guyton Date: Sun, 27 Sep 2026 18:41:19 -0700 Subject: [PATCH 4/4] arm: drop the PTQ1_0 NEON kernel, fall back to generic Two independent real-hardware reports (bri-prism on Apple M5 Pro, karusrus on a Snapdragon 7 Gen 4 phone) found this kernel slower than the plain scalar loop it was meant to replace - the u16-lane widening used to decode trits costs more than it saves. karusrus also posted a working replacement (8-bit lane decode, i8mm/SMMLA 2x2 tiling) that measures ~1.9x/1.2x over generic on their hardware: https://github.com/karusrus/llama.cpp/tree/arm-neon-ptq1_0 I don't have i8mm-capable ARM hardware to independently verify that kernel's fast path, so rather than merge an alternative I can't test, this drops the regressive kernel here and aliases PTQ1_0 back to generic on ARM (matching the existing x86 fallback pattern in arch-fallback.h before #248 added a real SIMD kernel there). PQ2_0's NEON kernels are unaffected and keep their validated ~5.1x/4.6x speedup. Verified on Orange Pi 5 Plus (RK3588, Cortex-A76/A55, no i8mm): test-quantize-fns passes 0 failures for both formats. Real 27B PTQ1_0 throughput on this fallback: pp64 0.33 t/s, tg16 0.27 t/s (generic loop, 5 threads) - down from this PR's earlier NEON numbers, which were never a real improvement to begin with. Claude-Session: https://claude.ai/code/session_016pBjsH3XSFLbnzoDyzkcUU --- ggml/src/ggml-cpu/arch-fallback.h | 4 ++ ggml/src/ggml-cpu/arch/arm/quants.c | 82 ----------------------------- 2 files changed, 4 insertions(+), 82 deletions(-) diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h index cb093eb59d4b..d7059b64601a 100644 --- a/ggml/src/ggml-cpu/arch-fallback.h +++ b/ggml/src/ggml-cpu/arch-fallback.h @@ -82,6 +82,10 @@ #define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0 #define ggml_gemm_pq2_0_4x8_q8_0_generic ggml_gemm_pq2_0_4x8_q8_0 #elif defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64) +// PTQ1_0's NEON vec_dot is unverified on real ARM hardware and measured slower +// than the generic loop on two independent devices (Apple M5 Pro, Snapdragon 7 +// Gen 4); alias it back to generic until a validated replacement lands +#define ggml_vec_dot_ptq1_0_q8_0_generic ggml_vec_dot_ptq1_0_q8_0 // repack.cpp #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 diff --git a/ggml/src/ggml-cpu/arch/arm/quants.c b/ggml/src/ggml-cpu/arch/arm/quants.c index b36d24d51b50..bea782fdb57e 100644 --- a/ggml/src/ggml-cpu/arch/arm/quants.c +++ b/ggml/src/ggml-cpu/arch/arm/quants.c @@ -427,88 +427,6 @@ void ggml_vec_dot_pq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo *s = sumf; } -void ggml_vec_dot_ptq1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { - assert(n % QK_PTQ1_0 == 0); - assert(nrc == 1); - UNUSED(nrc); - UNUSED(bx); - UNUSED(by); - UNUSED(bs); - - const block_ptq1_0 * GGML_RESTRICT x = vx; - const block_q8_0 * GGML_RESTRICT y = vy; - const int nb = n / QK_PTQ1_0; - - float sumf = 0.0f; - -#if defined(__ARM_NEON) - // Base-3 trit decode, matching dequantize_row_ptq1_0(): for each packed byte v the - // next trit is floor(3*v/256) and the byte carries forward as (3*v) & 0xFF. Doing that - // 16 bytes at a time in uint16 lanes keeps 3*v exact (max 765) and needs no division. - // qs holds 24 bytes x 5 trits = 120 weights, qh 2 bytes x 4 trits = 8 more, in the - // stage order 16/8 that the reference walks. - for (int i = 0; i < nb; i++) { - const block_q8_0 * GGML_RESTRICT yb = &y[i * 4]; - - int32x4_t acc[4] = { vdupq_n_s32(0), vdupq_n_s32(0), vdupq_n_s32(0), vdupq_n_s32(0) }; - - // stage 1: qs[0..15], 5 trits each -> weights 0..79 - uint16x8_t lo = vmovl_u8(vld1_u8(x[i].qs)); - uint16x8_t hi = vmovl_u8(vld1_u8(x[i].qs + 8)); - for (int t = 0; t < 5; t++) { - const uint16x8_t l3 = vmulq_n_u16(lo, 3); - const uint16x8_t h3 = vmulq_n_u16(hi, 3); - const int8x16_t q = vsubq_s8( - vreinterpretq_s8_u8(vcombine_u8(vshrn_n_u16(l3, 8), vshrn_n_u16(h3, 8))), - vdupq_n_s8(1)); - lo = vandq_u16(l3, vdupq_n_u16(0xFF)); - hi = vandq_u16(h3, vdupq_n_u16(0xFF)); - - const int o = t * 16; - acc[o >> 5] = ggml_vdotq_s32(acc[o >> 5], q, vld1q_s8(yb[o >> 5].qs + (o & 31))); - } - - // stage 2: qs[16..23], 5 trits each -> weights 80..119 - uint16x8_t t8 = vmovl_u8(vld1_u8(x[i].qs + 16)); - for (int t = 0; t < 5; t++) { - const uint16x8_t p3 = vmulq_n_u16(t8, 3); - const int8x8_t q = vsub_s8(vreinterpret_s8_u8(vshrn_n_u16(p3, 8)), vdup_n_s8(1)); - t8 = vandq_u16(p3, vdupq_n_u16(0xFF)); - - const int o = 80 + t * 8; - const int8x8_t yv = vld1_s8(yb[o >> 5].qs + (o & 31)); - acc[o >> 5] = vpadalq_s16(acc[o >> 5], vmull_s8(q, yv)); - } - - // stage 3: qh[0..1], 4 trits each -> weights 120..127 - uint16_t qh2; - memcpy(&qh2, x[i].qh, sizeof(qh2)); - uint16x8_t hv = vmovl_u8(vreinterpret_u8_u16(vdup_n_u16(qh2))); - { - static const uint16_t pw[8] = {1, 1, 3, 3, 9, 9, 27, 27}; - hv = vandq_u16(vmulq_u16(hv, vld1q_u16(pw)), vdupq_n_u16(0xFF)); - } - { - const uint16x8_t p3 = vmulq_n_u16(hv, 3); - const int8x8_t q = vsub_s8(vreinterpret_s8_u8(vshrn_n_u16(p3, 8)), vdup_n_s8(1)); - const int8x8_t yv = vld1_s8(yb[3].qs + 24); - acc[3] = vpadalq_s16(acc[3], vmull_s8(q, yv)); - } - - float sumi = 0.0f; - for (int k = 0; k < 4; k++) { - sumi += GGML_CPU_FP16_TO_FP32(yb[k].d) * (float) vaddvq_s32(acc[k]); - } - sumf += GGML_CPU_FP16_TO_FP32(x[i].d) * sumi; - } -#else - ggml_vec_dot_ptq1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); - return; -#endif - - *s = sumf; -} - void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { const int qk = QK8_0; const int nb = n / qk;