diff --git a/docs/build.md b/docs/build.md index ed48e7a05ec4..6b58575eeb80 100644 --- a/docs/build.md +++ b/docs/build.md @@ -92,6 +92,29 @@ cmake --build build --config Release - **Fedora / RHEL / Rocky / Alma:** `sudo dnf install openssl-devel` - **Arch / Manjaro:** `sudo pacman -S openssl` +### x86 CPU instruction sets + +To build for x86-64 CPUs with SSSE3 and no AVX requirement, use a fresh build directory: + +```bash +cmake -B build-ssse3 -DGGML_NATIVE=OFF -DGGML_SSSE3=ON \ + -DGGML_SSE42=OFF -DGGML_AVX=OFF -DGGML_AVX2=OFF \ + -DGGML_FMA=OFF -DGGML_F16C=OFF -DGGML_BMI2=OFF +cmake --build build-ssse3 --config Release -j 8 +``` + +SSSE3 is a separate extension from SSE3. Set `GGML_SSSE3=OFF` in this configuration for the x86-64 SSE2 baseline. Avoid `-march=native` in custom compiler flags when targeting older CPUs. + +For automatic selection at startup, build the CPU variants as shared backends: + +```bash +cmake -B build-auto -DGGML_NATIVE=OFF -DBUILD_SHARED_LIBS=ON \ + -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON +cmake --build build-auto --config Release -j 8 +``` + +On x86-64, this includes SSE2, SSSE3, SSE4.2, and AVX-family variants. Keep the backend libraries with the executable when distributing the build; the loader selects the highest-ranked variant supported by the CPU. + ## BLAS Build Building the program with BLAS support may lead to some performance improvements in prompt processing using batch sizes higher than 32 (the default is 512). Using BLAS doesn't affect the generation performance. There are currently several different BLAS implementations available for build and use: diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index aa1bef9484de..817b362deed3 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -151,6 +151,7 @@ message(DEBUG "INS_ENB : ${INS_ENB}") option(GGML_CPU_HBM "ggml: use memkind for CPU HBM" OFF) option(GGML_CPU_REPACK "ggml: use runtime weight conversion of Q4_0 to Q4_X_X" ON) option(GGML_CPU_KLEIDIAI "ggml: use KleidiAI optimized kernels if applicable" OFF) +option(GGML_SSSE3 "ggml: enable SSSE3" OFF) option(GGML_SSE42 "ggml: enable SSE 4.2" ${INS_ENB}) option(GGML_AVX "ggml: enable AVX" ${INS_ENB}) option(GGML_AVX_VNNI "ggml: enable AVX-VNNI" OFF) diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txt index 96535b49fa84..9fef45dfdb7f 100644 --- a/ggml/src/CMakeLists.txt +++ b/ggml/src/CMakeLists.txt @@ -441,7 +441,7 @@ function(ggml_add_cpu_backend_variant tag_name) # other: OPENMP LLAMAFILE CPU_HBM if (GGML_SYSTEM_ARCH STREQUAL "x86") foreach (feat NATIVE - SSE42 + SSSE3 SSE42 AVX AVX2 BMI2 AVX_VNNI FMA F16C AVX512 AVX512_VBMI AVX512_VNNI AVX512_BF16 AMX_TILE AMX_INT8 AMX_BF16) @@ -490,6 +490,7 @@ if (GGML_CPU_ALL_VARIANTS) endif() if (GGML_SYSTEM_ARCH STREQUAL "x86") ggml_add_cpu_backend_variant(x64) + ggml_add_cpu_backend_variant(ssse3 SSSE3) ggml_add_cpu_backend_variant(sse42 SSE42) ggml_add_cpu_backend_variant(sandybridge SSE42 AVX) if (NOT MSVC) diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index e16ac996a4a9..a13247f334e4 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -294,6 +294,12 @@ function(ggml_add_cpu_backend_variant_impl tag_name) list(APPEND ARCH_FLAGS /arch:SSE4.2) list(APPEND ARCH_DEFINITIONS GGML_SSE42) endif() + if (GGML_SSSE3) + list(APPEND ARCH_DEFINITIONS GGML_SSSE3) + if (CMAKE_C_COMPILER_ID STREQUAL "Clang") + list(APPEND ARCH_FLAGS -mssse3) + endif() + endif() if (GGML_AVX_VNNI) list(APPEND ARCH_DEFINITIONS __AVXVNNI__ GGML_AVX_VNNI) endif() @@ -305,6 +311,10 @@ function(ggml_add_cpu_backend_variant_impl tag_name) if (GGML_NATIVE) list(APPEND ARCH_FLAGS -march=native) else () + if (GGML_SSSE3) + list(APPEND ARCH_FLAGS -mssse3) + list(APPEND ARCH_DEFINITIONS GGML_SSSE3) + endif() if (GGML_SSE42) list(APPEND ARCH_FLAGS -msse4.2) list(APPEND ARCH_DEFINITIONS GGML_SSE42) diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h index e116558224b4..de3592160294 100644 --- a/ggml/src/ggml-cpu/arch-fallback.h +++ b/ggml/src/ggml-cpu/arch-fallback.h @@ -98,8 +98,6 @@ #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K #define ggml_gemm_pq2_0_4x8_q8_0_generic ggml_gemm_pq2_0_4x8_q8_0 #elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64) -// PTQ1_0 currently has only the generic vec_dot; alias it here until a SIMD version lands -#define ggml_vec_dot_ptq1_0_q8_0_generic ggml_vec_dot_ptq1_0_q8_0 // quants.c #define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 // repack.cpp diff --git a/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp b/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp index d775a0363858..5a2a7836a9ce 100644 --- a/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +++ b/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp @@ -266,6 +266,10 @@ static int ggml_backend_cpu_x86_score() { int score = 1; cpuid_x86 is; +#ifdef GGML_SSSE3 + if (!is.SSSE3()) { return 0; } + score += 1; +#endif #ifdef GGML_FMA if (!is.FMA()) { return 0; } score += 1; diff --git a/ggml/src/ggml-cpu/arch/x86/quants.c b/ggml/src/ggml-cpu/arch/x86/quants.c index bd572432f48f..0f07c0aaeeef 100644 --- a/ggml/src/ggml-cpu/arch/x86/quants.c +++ b/ggml/src/ggml-cpu/arch/x86/quants.c @@ -558,6 +558,46 @@ static inline __m128i get_scale_shuffle(int i) { # define GGML_DPBUSD_256 _mm256_dpbusd_avx_epi32 #endif +#if defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2) +static inline __m128i mul_sum_i8_pairs_sse2(const __m128i x, const __m128i y) { + const __m128i sx = _mm_cmpgt_epi8(_mm_setzero_si128(), x); + const __m128i sy = _mm_cmpgt_epi8(_mm_setzero_si128(), y); + const __m128i lo = _mm_madd_epi16(_mm_unpacklo_epi8(x, sx), _mm_unpacklo_epi8(y, sy)); + const __m128i hi = _mm_madd_epi16(_mm_unpackhi_epi8(x, sx), _mm_unpackhi_epi8(y, sy)); + return _mm_add_epi32(lo, hi); +} + +static inline int hsum_i32_4_sse2(__m128i x) { + x = _mm_add_epi32(x, _mm_srli_si128(x, 8)); + x = _mm_add_epi32(x, _mm_srli_si128(x, 4)); + return _mm_cvtsi128_si32(x); +} + +static inline __m128i decode_trits_sse2(__m128i * x) { + const __m128i triple = _mm_add_epi16(*x, _mm_add_epi16(*x, *x)); + *x = _mm_and_si128(triple, _mm_set1_epi16(255)); + return _mm_sub_epi16(_mm_srli_epi16(triple, 8), _mm_set1_epi16(1)); +} +#endif + +#if defined(__SSSE3__) +static inline __m128i mul_sum_ternary_pairs_ssse3(const __m128i codes, const __m128i y) { + // Subtract the code offset without negating -128 activations. + const __m128i dot = _mm_maddubs_epi16(codes, y); + const __m128i sum = _mm_maddubs_epi16(_mm_set1_epi8(1), y); + return _mm_madd_epi16(_mm_sub_epi16(dot, sum), _mm_set1_epi16(1)); +} + +static inline __m128i decode_trits_ssse3(__m128i * x) { + // Extract floor(3*x/256) and keep the low byte for the next trit. + const __m128i biased = _mm_xor_si128(*x, _mm_set1_epi8(-128)); + const __m128i ge1 = _mm_cmpgt_epi8(biased, _mm_set1_epi8(85 - 128)); + const __m128i ge2 = _mm_cmpgt_epi8(biased, _mm_set1_epi8(170 - 128)); + *x = _mm_add_epi8(*x, _mm_add_epi8(*x, *x)); + return _mm_sub_epi8(_mm_setzero_si128(), _mm_add_epi8(ge1, ge2)); +} +#endif + void ggml_vec_dot_pq2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { const int qk = QK_PQ2_0; const int nb = n / qk; @@ -619,6 +659,35 @@ void ggml_vec_dot_pq2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo acc = _mm256_fmadd_ps(_mm256_set1_ps(d0), acc_block, acc); } sumf = hsum_float_8(acc); +#elif defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2) + const __m128i mask = _mm_set1_epi8(3); + for (int i = 0; i < nb; i++) { + const float d0 = GGML_CPU_FP16_TO_FP32(x[i].d); + float sumi = 0.0f; + for (int k = 0; k < 4; k++) { + const block_q8_0 * GGML_RESTRICT yb = &y[i * 4 + k]; + const __m128i qs = _mm_loadl_epi64((const __m128i *) &x[i].qs[k * 8]); + const __m128i q0 = _mm_and_si128(qs, mask); + const __m128i q1 = _mm_and_si128(_mm_srli_epi16(qs, 2), mask); + const __m128i q2 = _mm_and_si128(_mm_srli_epi16(qs, 4), mask); + const __m128i q3 = _mm_and_si128(_mm_srli_epi16(qs, 6), mask); + const __m128i q01 = _mm_unpacklo_epi8(q0, q1); + const __m128i q23 = _mm_unpacklo_epi8(q2, q3); + const __m128i lo = _mm_unpacklo_epi16(q01, q23); + const __m128i hi = _mm_unpackhi_epi16(q01, q23); +#if defined(__SSSE3__) + const __m128i dot0 = mul_sum_ternary_pairs_ssse3(lo, _mm_loadu_si128((const __m128i *) &yb->qs[0])); + const __m128i dot1 = mul_sum_ternary_pairs_ssse3(hi, _mm_loadu_si128((const __m128i *) &yb->qs[16])); +#else + const __m128i ones = _mm_set1_epi8(1); + const __m128i dot0 = mul_sum_i8_pairs_sse2(_mm_sub_epi8(lo, ones), _mm_loadu_si128((const __m128i *) &yb->qs[0])); + const __m128i dot1 = mul_sum_i8_pairs_sse2(_mm_sub_epi8(hi, ones), _mm_loadu_si128((const __m128i *) &yb->qs[16])); +#endif + const int dot = hsum_i32_4_sse2(_mm_add_epi32(dot0, dot1)); + sumi += GGML_CPU_FP16_TO_FP32(yb->d) * dot; + } + sumf += d0 * sumi; + } #else for (int i = 0; i < nb; i++) { const float d0 = GGML_CPU_FP16_TO_FP32(x[i].d); @@ -645,6 +714,84 @@ void ggml_vec_dot_pq2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo *s = sumf; } +void ggml_vec_dot_ptq1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +#if defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2) + assert(n % QK_PTQ1_0 == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bs); + UNUSED(bx); + UNUSED(by); + + const block_ptq1_0 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + const __m128i zero = _mm_setzero_si128(); + float sumf = 0.0f; + for (int i = 0; i < n / QK_PTQ1_0; ++i) { + const block_q8_0 * yb = &y[i * 4]; + __m128i sums[4] = {zero, zero, zero, zero}; + const __m128i qs = _mm_loadu_si128((const __m128i *) x[i].qs); +#if defined(__SSSE3__) + __m128i packed = qs; +#else + __m128i lo = _mm_unpacklo_epi8(qs, zero); + __m128i hi = _mm_unpackhi_epi8(qs, zero); +#endif + + // The first 16 packed bytes decode to five groups of 16 weights. + for (int j = 0; j < 5; ++j) { + const __m128i qy = _mm_loadu_si128((const __m128i *) &yb[j / 2].qs[(j % 2) * 16]); +#if defined(__SSSE3__) + const __m128i dot = mul_sum_ternary_pairs_ssse3(decode_trits_ssse3(&packed), qy); + sums[j / 2] = _mm_add_epi32(sums[j / 2], dot); +#else + const __m128i ql = decode_trits_sse2(&lo); + const __m128i qh = decode_trits_sse2(&hi); + const __m128i sy = _mm_cmpgt_epi8(zero, qy); + const __m128i dot0 = _mm_madd_epi16(ql, _mm_unpacklo_epi8(qy, sy)); + const __m128i dot1 = _mm_madd_epi16(qh, _mm_unpackhi_epi8(qy, sy)); + sums[j / 2] = _mm_add_epi32(sums[j / 2], _mm_add_epi32(dot0, dot1)); +#endif + } + + // The next eight bytes hold weights 80..119; qh holds weights 120..127. + __m128i tail = _mm_loadl_epi64((const __m128i *) &x[i].qs[16]); +#if !defined(__SSSE3__) + tail = _mm_unpacklo_epi8(tail, zero); +#endif + for (int j = 0; j < 5; ++j) { + const int offset = 80 + j * 8; + const __m128i qy = _mm_loadl_epi64((const __m128i *) &yb[offset / 32].qs[offset % 32]); +#if defined(__SSSE3__) + const __m128i dot = mul_sum_ternary_pairs_ssse3(decode_trits_ssse3(&tail), qy); +#else + const __m128i q = decode_trits_sse2(&tail); + const __m128i sy = _mm_cmpgt_epi8(zero, qy); + const __m128i dot = _mm_madd_epi16(q, _mm_unpacklo_epi8(qy, sy)); +#endif + sums[offset / 32] = _mm_add_epi32(sums[offset / 32], dot); + } + uint16_t qh; + memcpy(&qh, x[i].qh, sizeof(qh)); + tail = _mm_unpacklo_epi8(_mm_set1_epi16((int16_t) qh), zero); + tail = _mm_and_si128(_mm_mullo_epi16(tail, _mm_setr_epi16(1, 1, 3, 3, 9, 9, 27, 27)), _mm_set1_epi16(255)); + const __m128i q = decode_trits_sse2(&tail); + const __m128i qy = _mm_loadl_epi64((const __m128i *) &yb[3].qs[24]); + const __m128i sy = _mm_cmpgt_epi8(zero, qy); + sums[3] = _mm_add_epi32(sums[3], _mm_madd_epi16(q, _mm_unpacklo_epi8(qy, sy))); + + float sumi = 0.0f; + for (int k = 0; k < 4; ++k) { + sumi += GGML_CPU_FP16_TO_FP32(yb[k].d) * hsum_i32_4_sse2(sums[k]); + } + sumf += GGML_CPU_FP16_TO_FP32(x[i].d) * sumi; + } + *s = sumf; +#else + ggml_vec_dot_ptq1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); +#endif +} + void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { const int qk = QK1_0; const int nb = n / qk; @@ -4251,6 +4398,38 @@ void ggml_vec_dot_pq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo acc = _mm256_fmadd_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(x[i].d) * yb->d), _mm256_cvtepi32_ps(s32), acc); } sumf = hsum_float_8(acc); +#elif defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2) + // Same structure as the AVX2 path at half the width: the four sub-block dots of a + // 128-weight block accumulate in int32 and the block scale is applied once. The unpack + // puts byte b's bit-pair j at element 4b+j, which is the order q8 is already in. + const __m128i mask = _mm_set1_epi8(3); + for (int i = 0; i < nb; i++) { + const block_q8_K * GGML_RESTRICT yb = &y[i >> 1]; + const int8_t * GGML_RESTRICT q8 = yb->qs + 128 * (i & 1); + __m128i acc32 = _mm_setzero_si128(); + for (int k = 0; k < 4; k++) { + const __m128i qs = _mm_loadl_epi64((const __m128i *) &x[i].qs[8 * k]); + const __m128i q0 = _mm_and_si128(qs, mask); + const __m128i q1 = _mm_and_si128(_mm_srli_epi16(qs, 2), mask); + const __m128i q2 = _mm_and_si128(_mm_srli_epi16(qs, 4), mask); + const __m128i q3 = _mm_and_si128(_mm_srli_epi16(qs, 6), mask); + const __m128i q01 = _mm_unpacklo_epi8(q0, q1); + const __m128i q23 = _mm_unpacklo_epi8(q2, q3); + const __m128i lo = _mm_unpacklo_epi16(q01, q23); + const __m128i hi = _mm_unpackhi_epi16(q01, q23); + const __m128i y0 = _mm_loadu_si128((const __m128i *) (q8 + 32 * k)); + const __m128i y1 = _mm_loadu_si128((const __m128i *) (q8 + 32 * k + 16)); +#if defined(__SSSE3__) + acc32 = _mm_add_epi32(acc32, mul_sum_ternary_pairs_ssse3(lo, y0)); + acc32 = _mm_add_epi32(acc32, mul_sum_ternary_pairs_ssse3(hi, y1)); +#else + const __m128i ones = _mm_set1_epi8(1); + acc32 = _mm_add_epi32(acc32, mul_sum_i8_pairs_sse2(_mm_sub_epi8(lo, ones), y0)); + acc32 = _mm_add_epi32(acc32, mul_sum_i8_pairs_sse2(_mm_sub_epi8(hi, ones), y1)); +#endif + } + sumf += (GGML_CPU_FP16_TO_FP32(x[i].d) * yb->d) * (float) hsum_i32_4_sse2(acc32); + } #else ggml_vec_dot_pq2_0_q8_K_generic(n, &sumf, bs, vx, bx, vy, by, nrc); #endif diff --git a/ggml/src/ggml-cpu/ggml-cpu-impl.h b/ggml/src/ggml-cpu/ggml-cpu-impl.h index 5d1ca5ffcc36..7708b36e4e1d 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-impl.h +++ b/ggml/src/ggml-cpu/ggml-cpu-impl.h @@ -52,8 +52,8 @@ struct ggml_compute_params { #endif #endif -// __SSE3__ and __SSSE3__ are not defined in MSVC, but SSE3/SSSE3 are present when AVX/AVX2/AVX512 are available -#if defined(_MSC_VER) && (defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)) +// MSVC does not define SSE3/SSSE3 feature macros. +#if defined(_MSC_VER) && (defined(GGML_SSSE3) || defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)) #ifndef __SSE3__ #define __SSE3__ #endif diff --git a/tests/test-quantize-fns.cpp b/tests/test-quantize-fns.cpp index 4f77fb9701c7..4172f316ef38 100644 --- a/tests/test-quantize-fns.cpp +++ b/tests/test-quantize-fns.cpp @@ -2,6 +2,7 @@ #include "ggml.h" #include "ggml-cpu.h" +#include "../ggml/src/ggml-quants.h" #undef NDEBUG #include @@ -202,6 +203,84 @@ static int test_vec_dot_q(bool verbose) { return num_failed; } +static int test_vec_dot_ternary(bool verbose) { + int num_failed = 0; + for (ggml_type type : {GGML_TYPE_PQ2_0, GGML_TYPE_PTQ1_0}) { + const auto * traits = ggml_get_type_traits(type); + const auto * cpu = ggml_get_type_traits_cpu(type); + // PQ2_0 dots against Q8_K (one float scale per 256), PTQ1_0 against Q8_0 (one fp16 + // scale per 32), so the activation side is built per format. PQ2_0 needs whole Q8_K + // blocks, i.e. an even number of 128-weight blocks. + const bool q8k = type == GGML_TYPE_PQ2_0; + const ggml_type ytype = q8k ? GGML_TYPE_Q8_K : GGML_TYPE_Q8_0; + for (int nb : q8k ? std::vector{2, 4} : std::vector{1, 3}) { + const int n = nb * 128; + std::vector pq(nb); + std::vector ptq(nb); + std::vector q8(nb * 4); + std::vector q8k_blocks(nb / 2 + 1); + std::vector x(n), y(n); + const void * weights = type == GGML_TYPE_PQ2_0 ? (const void *) pq.data() : (const void *) ptq.data(); + for (int pattern = 0; pattern < 256; ++pattern) { + for (int i = 0; i < nb; ++i) { + pq[i].d = ptq[i].d = ggml_fp32_to_fp16(0.25f * (i + 1)); + for (size_t j = 0; j < sizeof(pq[i].qs); ++j) { + pq[i].qs[j] = (uint8_t) (pattern + 17*j + i); + } + for (size_t j = 0; j < sizeof(ptq[i].qs); ++j) { + ptq[i].qs[j] = (uint8_t) (pattern + 17*j + i); + } + for (size_t j = 0; j < sizeof(ptq[i].qh); ++j) { + ptq[i].qh[j] = (uint8_t) (pattern + 37*j + i); + } + } + for (int i = 0; i < nb * 4; ++i) { + q8[i].d = ggml_fp32_to_fp16(0.125f * (i % 4 + 1)); + for (int j = 0; j < QK8_0; ++j) { + q8[i].qs[j] = (int8_t) ((pattern + 13*j + i) % 256 - 128); + } + } + for (size_t i = 0; i < q8k_blocks.size(); ++i) { + q8k_blocks[i].d = 0.125f * (i % 4 + 1); + for (int j = 0; j < QK_K; ++j) { + q8k_blocks[i].qs[j] = (int8_t) ((pattern + 13*j + i) % 256 - 128); + } + // bsums is unused by the PQ2_0 dot but keep it consistent. + for (int j = 0; j < QK_K/16; ++j) { + int16_t s = 0; + for (int t = 0; t < 16; ++t) s += q8k_blocks[i].qs[j*16 + t]; + q8k_blocks[i].bsums[j] = s; + } + } + const void * acts = q8k ? (const void *) q8k_blocks.data() : (const void *) q8.data(); + traits->to_float(weights, x.data(), n); + if (q8k) { + // Q8_K is an activation-only type and has no to_float, so expand it here. + for (int j = 0; j < n; ++j) { + const block_q8_K & b = q8k_blocks[j / QK_K]; + y[j] = b.d * (float) b.qs[j % QK_K]; + } + } else { + ggml_get_type_traits(ytype)->to_float(acts, y.data(), n); + } + const float ref = dot_product(x.data(), y.data(), n); + float result = INFINITY; + cpu->vec_dot(n, &result, 0, weights, 0, acts, 0, 1); + // Power-of-two scales keep this comparison exact. + const bool failed = result != ref; + num_failed += failed; + if (failed) { + printf("%5s packed dot nb=%d pattern=%d: FAILED (ref=%f got=%f)\n", ggml_type_name(type), nb, pattern, ref, result); + } + } + } + } + if (num_failed || verbose) { + printf("ternary packed dot products: %s (%d failures)\n", RESULT_STR[num_failed != 0], num_failed); + } + return num_failed; +} + int main(int argc, char * argv[]) { bool verbose = false; @@ -223,6 +302,7 @@ int main(int argc, char * argv[]) { num_failed += test_vec_dot_f32(verbose); num_failed += test_vec_dot_q(verbose); + num_failed += test_vec_dot_ternary(verbose); if (num_failed || verbose) { printf("%d tests failed\n", num_failed);