From df81cbd7627634387820163ee6675e3e95b01a18 Mon Sep 17 00:00:00 2001 From: Scott J Guyton Date: Tue, 22 Sep 2026 23:43:58 -0700 Subject: [PATCH 1/3] x86: add SSE2/SSSE3 vec_dot for PTQ1_0 and PQ2_0 PTQ1_0 had no x86 SIMD path at all - arch-fallback.h aliased it to the generic C implementation, so every x86 CPU decoded base-3 trits one weight at a time. PQ2_0 had an x86 kernel, but it is gated on VNNI (dpbusd), so everything below Ice Lake / Zen 4 fell through to the scalar loop. Add a 128-bit path for both, in two tiers: SSE2 - trit decode by multiply-and-shift, sign-extended _mm_madd_epi16 for the dot. Works on any x86-64. SSSE3 - _mm_maddubs_epi16 for the unsigned x signed multiply-add, which halves the ALU work in the inner loop. The ternary offset (codes are 0..2, values -1..1) is folded into the accumulator as a separate maddubs against ones rather than subtracted per element, which avoids negating -128 activations. Also add GGML_SSSE3 as a build variant so the dispatcher can score it, matching how the other x86 feature tiers are handled. Measured on an i5-12600K with AVX disabled at compile time, 8 threads. Isolated A/B: same binary, only libggml-cpu.so swapped, runs alternated to cancel background load, two pairs per configuration. Ternary Bonsai 1.7B PQ2_0 pp128 tg32 scalar 15.33/15.39 9.49/9.62 SSE2 42.91/43.23 25.20/25.35 SSSE3 59.49/58.98 32.07/31.42 Ternary Bonsai 2 27B PTQ1_0 pp64 tg16 scalar 0.71/0.72 0.60/0.62 SSSE3 2.66/2.66 2.11/2.10 3.3x tg on PQ2_0 and 3.5x on PTQ1_0 over scalar. Also verified on a dual Xeon E5645 (Westmere, SSE4.2 ceiling, no AVX): builds clean and runs both formats, which is the hardware tier this is mainly for. --- docs/build.md | 23 ++++ ggml/CMakeLists.txt | 1 + ggml/src/CMakeLists.txt | 3 +- ggml/src/ggml-cpu/CMakeLists.txt | 10 ++ ggml/src/ggml-cpu/arch-fallback.h | 2 - ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp | 4 + ggml/src/ggml-cpu/arch/x86/quants.c | 147 +++++++++++++++++++++++ ggml/src/ggml-cpu/ggml-cpu-impl.h | 4 +- 8 files changed, 189 insertions(+), 5 deletions(-) diff --git a/docs/build.md b/docs/build.md index ed48e7a05ec4..6b58575eeb80 100644 --- a/docs/build.md +++ b/docs/build.md @@ -92,6 +92,29 @@ cmake --build build --config Release - **Fedora / RHEL / Rocky / Alma:** `sudo dnf install openssl-devel` - **Arch / Manjaro:** `sudo pacman -S openssl` +### x86 CPU instruction sets + +To build for x86-64 CPUs with SSSE3 and no AVX requirement, use a fresh build directory: + +```bash +cmake -B build-ssse3 -DGGML_NATIVE=OFF -DGGML_SSSE3=ON \ + -DGGML_SSE42=OFF -DGGML_AVX=OFF -DGGML_AVX2=OFF \ + -DGGML_FMA=OFF -DGGML_F16C=OFF -DGGML_BMI2=OFF +cmake --build build-ssse3 --config Release -j 8 +``` + +SSSE3 is a separate extension from SSE3. Set `GGML_SSSE3=OFF` in this configuration for the x86-64 SSE2 baseline. Avoid `-march=native` in custom compiler flags when targeting older CPUs. + +For automatic selection at startup, build the CPU variants as shared backends: + +```bash +cmake -B build-auto -DGGML_NATIVE=OFF -DBUILD_SHARED_LIBS=ON \ + -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON +cmake --build build-auto --config Release -j 8 +``` + +On x86-64, this includes SSE2, SSSE3, SSE4.2, and AVX-family variants. Keep the backend libraries with the executable when distributing the build; the loader selects the highest-ranked variant supported by the CPU. + ## BLAS Build Building the program with BLAS support may lead to some performance improvements in prompt processing using batch sizes higher than 32 (the default is 512). Using BLAS doesn't affect the generation performance. There are currently several different BLAS implementations available for build and use: diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index aa1bef9484de..817b362deed3 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -151,6 +151,7 @@ message(DEBUG "INS_ENB : ${INS_ENB}") option(GGML_CPU_HBM "ggml: use memkind for CPU HBM" OFF) option(GGML_CPU_REPACK "ggml: use runtime weight conversion of Q4_0 to Q4_X_X" ON) option(GGML_CPU_KLEIDIAI "ggml: use KleidiAI optimized kernels if applicable" OFF) +option(GGML_SSSE3 "ggml: enable SSSE3" OFF) option(GGML_SSE42 "ggml: enable SSE 4.2" ${INS_ENB}) option(GGML_AVX "ggml: enable AVX" ${INS_ENB}) option(GGML_AVX_VNNI "ggml: enable AVX-VNNI" OFF) diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txt index 96535b49fa84..9fef45dfdb7f 100644 --- a/ggml/src/CMakeLists.txt +++ b/ggml/src/CMakeLists.txt @@ -441,7 +441,7 @@ function(ggml_add_cpu_backend_variant tag_name) # other: OPENMP LLAMAFILE CPU_HBM if (GGML_SYSTEM_ARCH STREQUAL "x86") foreach (feat NATIVE - SSE42 + SSSE3 SSE42 AVX AVX2 BMI2 AVX_VNNI FMA F16C AVX512 AVX512_VBMI AVX512_VNNI AVX512_BF16 AMX_TILE AMX_INT8 AMX_BF16) @@ -490,6 +490,7 @@ if (GGML_CPU_ALL_VARIANTS) endif() if (GGML_SYSTEM_ARCH STREQUAL "x86") ggml_add_cpu_backend_variant(x64) + ggml_add_cpu_backend_variant(ssse3 SSSE3) ggml_add_cpu_backend_variant(sse42 SSE42) ggml_add_cpu_backend_variant(sandybridge SSE42 AVX) if (NOT MSVC) diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index e16ac996a4a9..a13247f334e4 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -294,6 +294,12 @@ function(ggml_add_cpu_backend_variant_impl tag_name) list(APPEND ARCH_FLAGS /arch:SSE4.2) list(APPEND ARCH_DEFINITIONS GGML_SSE42) endif() + if (GGML_SSSE3) + list(APPEND ARCH_DEFINITIONS GGML_SSSE3) + if (CMAKE_C_COMPILER_ID STREQUAL "Clang") + list(APPEND ARCH_FLAGS -mssse3) + endif() + endif() if (GGML_AVX_VNNI) list(APPEND ARCH_DEFINITIONS __AVXVNNI__ GGML_AVX_VNNI) endif() @@ -305,6 +311,10 @@ function(ggml_add_cpu_backend_variant_impl tag_name) if (GGML_NATIVE) list(APPEND ARCH_FLAGS -march=native) else () + if (GGML_SSSE3) + list(APPEND ARCH_FLAGS -mssse3) + list(APPEND ARCH_DEFINITIONS GGML_SSSE3) + endif() if (GGML_SSE42) list(APPEND ARCH_FLAGS -msse4.2) list(APPEND ARCH_DEFINITIONS GGML_SSE42) diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h index e116558224b4..de3592160294 100644 --- a/ggml/src/ggml-cpu/arch-fallback.h +++ b/ggml/src/ggml-cpu/arch-fallback.h @@ -98,8 +98,6 @@ #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K #define ggml_gemm_pq2_0_4x8_q8_0_generic ggml_gemm_pq2_0_4x8_q8_0 #elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64) -// PTQ1_0 currently has only the generic vec_dot; alias it here until a SIMD version lands -#define ggml_vec_dot_ptq1_0_q8_0_generic ggml_vec_dot_ptq1_0_q8_0 // quants.c #define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 // repack.cpp diff --git a/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp b/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp index d775a0363858..5a2a7836a9ce 100644 --- a/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +++ b/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp @@ -266,6 +266,10 @@ static int ggml_backend_cpu_x86_score() { int score = 1; cpuid_x86 is; +#ifdef GGML_SSSE3 + if (!is.SSSE3()) { return 0; } + score += 1; +#endif #ifdef GGML_FMA if (!is.FMA()) { return 0; } score += 1; diff --git a/ggml/src/ggml-cpu/arch/x86/quants.c b/ggml/src/ggml-cpu/arch/x86/quants.c index bd572432f48f..882ea3848d08 100644 --- a/ggml/src/ggml-cpu/arch/x86/quants.c +++ b/ggml/src/ggml-cpu/arch/x86/quants.c @@ -558,6 +558,46 @@ static inline __m128i get_scale_shuffle(int i) { # define GGML_DPBUSD_256 _mm256_dpbusd_avx_epi32 #endif +#if defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2) +static inline __m128i mul_sum_i8_pairs_sse2(const __m128i x, const __m128i y) { + const __m128i sx = _mm_cmpgt_epi8(_mm_setzero_si128(), x); + const __m128i sy = _mm_cmpgt_epi8(_mm_setzero_si128(), y); + const __m128i lo = _mm_madd_epi16(_mm_unpacklo_epi8(x, sx), _mm_unpacklo_epi8(y, sy)); + const __m128i hi = _mm_madd_epi16(_mm_unpackhi_epi8(x, sx), _mm_unpackhi_epi8(y, sy)); + return _mm_add_epi32(lo, hi); +} + +static inline int hsum_i32_4_sse2(__m128i x) { + x = _mm_add_epi32(x, _mm_srli_si128(x, 8)); + x = _mm_add_epi32(x, _mm_srli_si128(x, 4)); + return _mm_cvtsi128_si32(x); +} + +static inline __m128i decode_trits_sse2(__m128i * x) { + const __m128i triple = _mm_add_epi16(*x, _mm_add_epi16(*x, *x)); + *x = _mm_and_si128(triple, _mm_set1_epi16(255)); + return _mm_sub_epi16(_mm_srli_epi16(triple, 8), _mm_set1_epi16(1)); +} +#endif + +#if defined(__SSSE3__) +static inline __m128i mul_sum_ternary_pairs_ssse3(const __m128i codes, const __m128i y) { + // Subtract the code offset without negating -128 activations. + const __m128i dot = _mm_maddubs_epi16(codes, y); + const __m128i sum = _mm_maddubs_epi16(_mm_set1_epi8(1), y); + return _mm_madd_epi16(_mm_sub_epi16(dot, sum), _mm_set1_epi16(1)); +} + +static inline __m128i decode_trits_ssse3(__m128i * x) { + // Extract floor(3*x/256) and keep the low byte for the next trit. + const __m128i biased = _mm_xor_si128(*x, _mm_set1_epi8(-128)); + const __m128i ge1 = _mm_cmpgt_epi8(biased, _mm_set1_epi8(85 - 128)); + const __m128i ge2 = _mm_cmpgt_epi8(biased, _mm_set1_epi8(170 - 128)); + *x = _mm_add_epi8(*x, _mm_add_epi8(*x, *x)); + return _mm_sub_epi8(_mm_setzero_si128(), _mm_add_epi8(ge1, ge2)); +} +#endif + void ggml_vec_dot_pq2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { const int qk = QK_PQ2_0; const int nb = n / qk; @@ -619,6 +659,35 @@ void ggml_vec_dot_pq2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo acc = _mm256_fmadd_ps(_mm256_set1_ps(d0), acc_block, acc); } sumf = hsum_float_8(acc); +#elif defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2) + const __m128i mask = _mm_set1_epi8(3); + for (int i = 0; i < nb; i++) { + const float d0 = GGML_CPU_FP16_TO_FP32(x[i].d); + float sumi = 0.0f; + for (int k = 0; k < 4; k++) { + const block_q8_0 * GGML_RESTRICT yb = &y[i * 4 + k]; + const __m128i qs = _mm_loadl_epi64((const __m128i *) &x[i].qs[k * 8]); + const __m128i q0 = _mm_and_si128(qs, mask); + const __m128i q1 = _mm_and_si128(_mm_srli_epi16(qs, 2), mask); + const __m128i q2 = _mm_and_si128(_mm_srli_epi16(qs, 4), mask); + const __m128i q3 = _mm_and_si128(_mm_srli_epi16(qs, 6), mask); + const __m128i q01 = _mm_unpacklo_epi8(q0, q1); + const __m128i q23 = _mm_unpacklo_epi8(q2, q3); + const __m128i lo = _mm_unpacklo_epi16(q01, q23); + const __m128i hi = _mm_unpackhi_epi16(q01, q23); +#if defined(__SSSE3__) + const __m128i dot0 = mul_sum_ternary_pairs_ssse3(lo, _mm_loadu_si128((const __m128i *) &yb->qs[0])); + const __m128i dot1 = mul_sum_ternary_pairs_ssse3(hi, _mm_loadu_si128((const __m128i *) &yb->qs[16])); +#else + const __m128i ones = _mm_set1_epi8(1); + const __m128i dot0 = mul_sum_i8_pairs_sse2(_mm_sub_epi8(lo, ones), _mm_loadu_si128((const __m128i *) &yb->qs[0])); + const __m128i dot1 = mul_sum_i8_pairs_sse2(_mm_sub_epi8(hi, ones), _mm_loadu_si128((const __m128i *) &yb->qs[16])); +#endif + const int dot = hsum_i32_4_sse2(_mm_add_epi32(dot0, dot1)); + sumi += GGML_CPU_FP16_TO_FP32(yb->d) * dot; + } + sumf += d0 * sumi; + } #else for (int i = 0; i < nb; i++) { const float d0 = GGML_CPU_FP16_TO_FP32(x[i].d); @@ -645,6 +714,84 @@ void ggml_vec_dot_pq2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo *s = sumf; } +void ggml_vec_dot_ptq1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +#if defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2) + assert(n % QK_PTQ1_0 == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bs); + UNUSED(bx); + UNUSED(by); + + const block_ptq1_0 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + const __m128i zero = _mm_setzero_si128(); + float sumf = 0.0f; + for (int i = 0; i < n / QK_PTQ1_0; ++i) { + const block_q8_0 * yb = &y[i * 4]; + __m128i sums[4] = {zero, zero, zero, zero}; + const __m128i qs = _mm_loadu_si128((const __m128i *) x[i].qs); +#if defined(__SSSE3__) + __m128i packed = qs; +#else + __m128i lo = _mm_unpacklo_epi8(qs, zero); + __m128i hi = _mm_unpackhi_epi8(qs, zero); +#endif + + // The first 16 packed bytes decode to five groups of 16 weights. + for (int j = 0; j < 5; ++j) { + const __m128i qy = _mm_loadu_si128((const __m128i *) &yb[j / 2].qs[(j % 2) * 16]); +#if defined(__SSSE3__) + const __m128i dot = mul_sum_ternary_pairs_ssse3(decode_trits_ssse3(&packed), qy); + sums[j / 2] = _mm_add_epi32(sums[j / 2], dot); +#else + const __m128i ql = decode_trits_sse2(&lo); + const __m128i qh = decode_trits_sse2(&hi); + const __m128i sy = _mm_cmpgt_epi8(zero, qy); + const __m128i dot0 = _mm_madd_epi16(ql, _mm_unpacklo_epi8(qy, sy)); + const __m128i dot1 = _mm_madd_epi16(qh, _mm_unpackhi_epi8(qy, sy)); + sums[j / 2] = _mm_add_epi32(sums[j / 2], _mm_add_epi32(dot0, dot1)); +#endif + } + + // The next eight bytes hold weights 80..119; qh holds weights 120..127. + __m128i tail = _mm_loadl_epi64((const __m128i *) &x[i].qs[16]); +#if !defined(__SSSE3__) + tail = _mm_unpacklo_epi8(tail, zero); +#endif + for (int j = 0; j < 5; ++j) { + const int offset = 80 + j * 8; + const __m128i qy = _mm_loadl_epi64((const __m128i *) &yb[offset / 32].qs[offset % 32]); +#if defined(__SSSE3__) + const __m128i dot = mul_sum_ternary_pairs_ssse3(decode_trits_ssse3(&tail), qy); +#else + const __m128i q = decode_trits_sse2(&tail); + const __m128i sy = _mm_cmpgt_epi8(zero, qy); + const __m128i dot = _mm_madd_epi16(q, _mm_unpacklo_epi8(qy, sy)); +#endif + sums[offset / 32] = _mm_add_epi32(sums[offset / 32], dot); + } + uint16_t qh; + memcpy(&qh, x[i].qh, sizeof(qh)); + tail = _mm_unpacklo_epi8(_mm_set1_epi16((int16_t) qh), zero); + tail = _mm_and_si128(_mm_mullo_epi16(tail, _mm_setr_epi16(1, 1, 3, 3, 9, 9, 27, 27)), _mm_set1_epi16(255)); + const __m128i q = decode_trits_sse2(&tail); + const __m128i qy = _mm_loadl_epi64((const __m128i *) &yb[3].qs[24]); + const __m128i sy = _mm_cmpgt_epi8(zero, qy); + sums[3] = _mm_add_epi32(sums[3], _mm_madd_epi16(q, _mm_unpacklo_epi8(qy, sy))); + + float sumi = 0.0f; + for (int k = 0; k < 4; ++k) { + sumi += GGML_CPU_FP16_TO_FP32(yb[k].d) * hsum_i32_4_sse2(sums[k]); + } + sumf += GGML_CPU_FP16_TO_FP32(x[i].d) * sumi; + } + *s = sumf; +#else + ggml_vec_dot_ptq1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); +#endif +} + void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { const int qk = QK1_0; const int nb = n / qk; diff --git a/ggml/src/ggml-cpu/ggml-cpu-impl.h b/ggml/src/ggml-cpu/ggml-cpu-impl.h index 5d1ca5ffcc36..7708b36e4e1d 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-impl.h +++ b/ggml/src/ggml-cpu/ggml-cpu-impl.h @@ -52,8 +52,8 @@ struct ggml_compute_params { #endif #endif -// __SSE3__ and __SSSE3__ are not defined in MSVC, but SSE3/SSSE3 are present when AVX/AVX2/AVX512 are available -#if defined(_MSC_VER) && (defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)) +// MSVC does not define SSE3/SSSE3 feature macros. +#if defined(_MSC_VER) && (defined(GGML_SSSE3) || defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)) #ifndef __SSE3__ #define __SSE3__ #endif From 9deb15b83d3b0426b8a28cd33ce8702587264f35 Mon Sep 17 00:00:00 2001 From: Scott J Guyton Date: Tue, 22 Sep 2026 23:44:06 -0700 Subject: [PATCH 2/3] test-quantize-fns: exact vec_dot check for PTQ1_0 and PQ2_0 The existing test_vec_dot_q() compares against a tolerance, which is the right call for lossy formats but too loose to catch an unpack bug in a ternary kernel: a wrong trit ordering still lands inside the error bound. Add a check that is exact instead. Weights and activations are driven over 256 bit patterns per format, and every scale is a power of two, so dequantize-then-dot and the packed vec_dot must agree bit for bit. Any mismatch in trit order, plane split, or the -1 offset shows up as a hard failure rather than a slightly larger error. Covers both block counts (1 and 3) so the multi-block loop is exercised. Passes on SSE2, SSSE3 and AVX2 builds, and on a dual Xeon E5645 where neither AVX path is available. --- tests/test-quantize-fns.cpp | 53 +++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/test-quantize-fns.cpp b/tests/test-quantize-fns.cpp index 4f77fb9701c7..b98418c4a8e6 100644 --- a/tests/test-quantize-fns.cpp +++ b/tests/test-quantize-fns.cpp @@ -2,6 +2,7 @@ #include "ggml.h" #include "ggml-cpu.h" +#include "../ggml/src/ggml-quants.h" #undef NDEBUG #include @@ -202,6 +203,57 @@ static int test_vec_dot_q(bool verbose) { return num_failed; } +static int test_vec_dot_ternary(bool verbose) { + int num_failed = 0; + for (ggml_type type : {GGML_TYPE_PQ2_0, GGML_TYPE_PTQ1_0}) { + const auto * traits = ggml_get_type_traits(type); + const auto * cpu = ggml_get_type_traits_cpu(type); + for (int nb : {1, 3}) { + const int n = nb * 128; + std::vector pq(nb); + std::vector ptq(nb); + std::vector q8(nb * 4); + std::vector x(n), y(n); + const void * weights = type == GGML_TYPE_PQ2_0 ? (const void *) pq.data() : (const void *) ptq.data(); + for (int pattern = 0; pattern < 256; ++pattern) { + for (int i = 0; i < nb; ++i) { + pq[i].d = ptq[i].d = ggml_fp32_to_fp16(0.25f * (i + 1)); + for (size_t j = 0; j < sizeof(pq[i].qs); ++j) { + pq[i].qs[j] = (uint8_t) (pattern + 17*j + i); + } + for (size_t j = 0; j < sizeof(ptq[i].qs); ++j) { + ptq[i].qs[j] = (uint8_t) (pattern + 17*j + i); + } + for (size_t j = 0; j < sizeof(ptq[i].qh); ++j) { + ptq[i].qh[j] = (uint8_t) (pattern + 37*j + i); + } + } + for (int i = 0; i < nb * 4; ++i) { + q8[i].d = ggml_fp32_to_fp16(0.125f * (i % 4 + 1)); + for (int j = 0; j < QK8_0; ++j) { + q8[i].qs[j] = (int8_t) ((pattern + 13*j + i) % 256 - 128); + } + } + traits->to_float(weights, x.data(), n); + ggml_get_type_traits(GGML_TYPE_Q8_0)->to_float(q8.data(), y.data(), n); + const float ref = dot_product(x.data(), y.data(), n); + float result = INFINITY; + cpu->vec_dot(n, &result, 0, weights, 0, q8.data(), 0, 1); + // Power-of-two scales keep this comparison exact. + const bool failed = result != ref; + num_failed += failed; + if (failed) { + printf("%5s packed dot nb=%d pattern=%d: FAILED (ref=%f got=%f)\n", ggml_type_name(type), nb, pattern, ref, result); + } + } + } + } + if (num_failed || verbose) { + printf("ternary packed dot products: %s (%d failures)\n", RESULT_STR[num_failed != 0], num_failed); + } + return num_failed; +} + int main(int argc, char * argv[]) { bool verbose = false; @@ -223,6 +275,7 @@ int main(int argc, char * argv[]) { num_failed += test_vec_dot_f32(verbose); num_failed += test_vec_dot_q(verbose); + num_failed += test_vec_dot_ternary(verbose); if (num_failed || verbose) { printf("%d tests failed\n", num_failed); From aed0a6345c707fb0c68be4eb6743d171e5d1425d Mon Sep 17 00:00:00 2001 From: Scott J Guyton Date: Wed, 23 Sep 2026 12:04:40 -0700 Subject: [PATCH 3/3] x86: follow PQ2_0 to the Q8_K activation path PQ2_0 moved to Q8_K activations, so ggml_vec_dot_pq2_0_q8_0 is no longer what the traits table dispatches and the SSE tier added earlier in this branch was dead code for normal inference. Add the same 128-bit tier to ggml_vec_dot_pq2_0_q8_K instead, below the AVX2 path. Structure mirrors the AVX2 kernel at half the width: the four sub-block dots of a 128-weight block accumulate in int32 and the block scale is applied once, rather than per sub-block. The existing unpack already emits byte b's bit-pair j at element 4b+j, which is the order the Q8_K activations are in, so no repermutation is needed. CPUs with AVX2 keep the upstream kernel; this only changes what runs below it, where PQ2_0 was falling back to the scalar loop. Also fix the ternary test to build the right activation type per format (Q8_K for PQ2_0, Q8_0 for PTQ1_0) and expand Q8_K by hand, since it is an activation-only type with no to_float. Verified the exact comparison still catches errors: injecting an off-by-one in the accumulator fails all 512 PQ2_0 cases. i5-12600K, AVX disabled at compile time, 8 threads, isolated A/B (same binary, only libggml-cpu.so swapped, alternated, two pairs): Ternary Bonsai 1.7B PQ2_0 pp128 tg32 scalar 15.04/15.12 9.97/9.83 SSSE3 83.81/82.98 44.91/45.21 4.5x tg. Higher than the 3.3x the Q8_0 path gave, because one activation scale per 256 removes the per-sub-block float chain. test-quantize-fns passes on SSE2, SSSE3 and AVX2 builds. --- ggml/src/ggml-cpu/arch/x86/quants.c | 32 ++++++++++++++++++++++++++++ tests/test-quantize-fns.cpp | 33 ++++++++++++++++++++++++++--- 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/x86/quants.c b/ggml/src/ggml-cpu/arch/x86/quants.c index 882ea3848d08..0f07c0aaeeef 100644 --- a/ggml/src/ggml-cpu/arch/x86/quants.c +++ b/ggml/src/ggml-cpu/arch/x86/quants.c @@ -4398,6 +4398,38 @@ void ggml_vec_dot_pq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo acc = _mm256_fmadd_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(x[i].d) * yb->d), _mm256_cvtepi32_ps(s32), acc); } sumf = hsum_float_8(acc); +#elif defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2) + // Same structure as the AVX2 path at half the width: the four sub-block dots of a + // 128-weight block accumulate in int32 and the block scale is applied once. The unpack + // puts byte b's bit-pair j at element 4b+j, which is the order q8 is already in. + const __m128i mask = _mm_set1_epi8(3); + for (int i = 0; i < nb; i++) { + const block_q8_K * GGML_RESTRICT yb = &y[i >> 1]; + const int8_t * GGML_RESTRICT q8 = yb->qs + 128 * (i & 1); + __m128i acc32 = _mm_setzero_si128(); + for (int k = 0; k < 4; k++) { + const __m128i qs = _mm_loadl_epi64((const __m128i *) &x[i].qs[8 * k]); + const __m128i q0 = _mm_and_si128(qs, mask); + const __m128i q1 = _mm_and_si128(_mm_srli_epi16(qs, 2), mask); + const __m128i q2 = _mm_and_si128(_mm_srli_epi16(qs, 4), mask); + const __m128i q3 = _mm_and_si128(_mm_srli_epi16(qs, 6), mask); + const __m128i q01 = _mm_unpacklo_epi8(q0, q1); + const __m128i q23 = _mm_unpacklo_epi8(q2, q3); + const __m128i lo = _mm_unpacklo_epi16(q01, q23); + const __m128i hi = _mm_unpackhi_epi16(q01, q23); + const __m128i y0 = _mm_loadu_si128((const __m128i *) (q8 + 32 * k)); + const __m128i y1 = _mm_loadu_si128((const __m128i *) (q8 + 32 * k + 16)); +#if defined(__SSSE3__) + acc32 = _mm_add_epi32(acc32, mul_sum_ternary_pairs_ssse3(lo, y0)); + acc32 = _mm_add_epi32(acc32, mul_sum_ternary_pairs_ssse3(hi, y1)); +#else + const __m128i ones = _mm_set1_epi8(1); + acc32 = _mm_add_epi32(acc32, mul_sum_i8_pairs_sse2(_mm_sub_epi8(lo, ones), y0)); + acc32 = _mm_add_epi32(acc32, mul_sum_i8_pairs_sse2(_mm_sub_epi8(hi, ones), y1)); +#endif + } + sumf += (GGML_CPU_FP16_TO_FP32(x[i].d) * yb->d) * (float) hsum_i32_4_sse2(acc32); + } #else ggml_vec_dot_pq2_0_q8_K_generic(n, &sumf, bs, vx, bx, vy, by, nrc); #endif diff --git a/tests/test-quantize-fns.cpp b/tests/test-quantize-fns.cpp index b98418c4a8e6..4172f316ef38 100644 --- a/tests/test-quantize-fns.cpp +++ b/tests/test-quantize-fns.cpp @@ -208,11 +208,17 @@ static int test_vec_dot_ternary(bool verbose) { for (ggml_type type : {GGML_TYPE_PQ2_0, GGML_TYPE_PTQ1_0}) { const auto * traits = ggml_get_type_traits(type); const auto * cpu = ggml_get_type_traits_cpu(type); - for (int nb : {1, 3}) { + // PQ2_0 dots against Q8_K (one float scale per 256), PTQ1_0 against Q8_0 (one fp16 + // scale per 32), so the activation side is built per format. PQ2_0 needs whole Q8_K + // blocks, i.e. an even number of 128-weight blocks. + const bool q8k = type == GGML_TYPE_PQ2_0; + const ggml_type ytype = q8k ? GGML_TYPE_Q8_K : GGML_TYPE_Q8_0; + for (int nb : q8k ? std::vector{2, 4} : std::vector{1, 3}) { const int n = nb * 128; std::vector pq(nb); std::vector ptq(nb); std::vector q8(nb * 4); + std::vector q8k_blocks(nb / 2 + 1); std::vector x(n), y(n); const void * weights = type == GGML_TYPE_PQ2_0 ? (const void *) pq.data() : (const void *) ptq.data(); for (int pattern = 0; pattern < 256; ++pattern) { @@ -234,11 +240,32 @@ static int test_vec_dot_ternary(bool verbose) { q8[i].qs[j] = (int8_t) ((pattern + 13*j + i) % 256 - 128); } } + for (size_t i = 0; i < q8k_blocks.size(); ++i) { + q8k_blocks[i].d = 0.125f * (i % 4 + 1); + for (int j = 0; j < QK_K; ++j) { + q8k_blocks[i].qs[j] = (int8_t) ((pattern + 13*j + i) % 256 - 128); + } + // bsums is unused by the PQ2_0 dot but keep it consistent. + for (int j = 0; j < QK_K/16; ++j) { + int16_t s = 0; + for (int t = 0; t < 16; ++t) s += q8k_blocks[i].qs[j*16 + t]; + q8k_blocks[i].bsums[j] = s; + } + } + const void * acts = q8k ? (const void *) q8k_blocks.data() : (const void *) q8.data(); traits->to_float(weights, x.data(), n); - ggml_get_type_traits(GGML_TYPE_Q8_0)->to_float(q8.data(), y.data(), n); + if (q8k) { + // Q8_K is an activation-only type and has no to_float, so expand it here. + for (int j = 0; j < n; ++j) { + const block_q8_K & b = q8k_blocks[j / QK_K]; + y[j] = b.d * (float) b.qs[j % QK_K]; + } + } else { + ggml_get_type_traits(ytype)->to_float(acts, y.data(), n); + } const float ref = dot_product(x.data(), y.data(), n); float result = INFINITY; - cpu->vec_dot(n, &result, 0, weights, 0, q8.data(), 0, 1); + cpu->vec_dot(n, &result, 0, weights, 0, acts, 0, 1); // Power-of-two scales keep this comparison exact. const bool failed = result != ref; num_failed += failed;